Node Webapp Server

main.js

require('./classes/gamelog.js')
require('./classes/proto.js');
require('./classes/service-locator.js');
fs = require('fs');
var context = require("./classes/context.js");
var cluster = require('cluster');
var os = require("os");
var hostname = os.hostname();
//console.log("hostname is " + hostname!);
var IS_MASTER_SHUTTING_DOWN = false;
var rc = require("./classes/request-context.js");
var numCPUs = 12;
var listenPort = 61337;
var deployment = "prod";
ServiceLocator.AppDeployment = deployment;

var domain = require('domain');
var _create = domain.create;

domain.create = function() {
    var d = _create.apply(domain, arguments);
    var parent_data = (process.domain && process.domain.data) || {};
    d.data = Object.create(parent_data);
    return d;
};



function startServer(deployment, version){
    ServiceLocator.ContextProvider = context;
    ServiceLocator.Process = process;
    ServiceLocator.IsMasterProcess = cluster.isMaster;
    ServiceLocator.Cluster = cluster;
    ServiceLocator.Ipc = require("./classes/ipc.js").instance;
    if(!cluster.isMaster) {
        ServiceLocator.WorkerId = cluster.worker.id;
        process.on('message', ServiceLocator.Ipc.HandleMessageOnWorker);
    }
    ServiceLocator.Constants = require("./classes/constants.js");
    ServiceLocator.init(deployment, version, function(success, errorMessage){
            if(!success){
                throw new Error("Failed to initialize services " + errorMessage);
            }
            if (cluster.isMaster) {
                require("./content-verify.js");
                var env = {};
                GameLog.info('Honorbound server ' + displayVersion + " starting with " + numCPUs + " workers in deployment " + deployment);

                for (var i = 0; i < numCPUs; i++) {
                    worker = cluster.fork(env);
                }

                cluster.on('exit', function(worker, code, signal) {
                    if(signal > 0){
                        GameLog.error('WORKER ' + worker.process.pid + ' died restarting');
                        worker = cluster.fork(env);
                        worker.on('message', ServiceLocator.Ipc.HandleMessageOnMaster);
                    } else{
                        GameLog.error('WORKER reported dead - but didnt ask for a restart');
                    }
                });

                cluster.on('disconnect', function(worker) {
                    GameLog.error('WORKER ' + worker.process.pid + ' disconnected');
                    if(!IS_MASTER_SHUTTING_DOWN){
                        worker = cluster.fork(env);
                        GameLog.error('REFORKING WORKER ' + worker.process.pid);
                        worker.on('message', ServiceLocator.Ipc.HandleMessageOnMaster);
                    }
                });

            }
            if(!cluster.isMaster) {
                ServiceLocator.WorkerId = cluster.worker.id;
                var server = ServiceLocator.Restify.createServer( {formatters: {
                    'text/html': function(req, res, body){
                        return body;
                    },
                    'application/x-gzip': function(req, res, body){
                        return body;
                    }
                }});
                ServiceLocator.Server = server;

                function handleHealthCheck(req, res, next){
                    // Handle LB Healthcheck ping
                    req.on('data', function (chunk) {
                    });
                    req.on('end', function () {
                        res.send("OK");
                    });
                }
                function handleBatch(req, res, next){
                    var context = RequestContext.Parse(req);
                    process.domain.data.RequestContext = context;
                    ServiceLocator.BatchProcessor.processBatch(req, res, false, context);
                }

                server.post('/batch', handleBatch);
                server.get('/healthCheck', handleHealthCheck);
                server.post('/postTest', handlePostTest);
                server.pre(function(req, res, next) {
                    var requestDomain = domain.create();

                    requestDomain.on("error", function(err){
                        GameLog.warn("REQUEST EXCEPTION: " + err.stack);
                        try {
                            if(numCPUs > 1){
                                GameLog.warn("Graceful shutdown starting");
                                // make sure we close down within 30 seconds
                                var killtimer = setTimeout(function() {
                                    ServiceLocator.Cluster.worker.destroy(0);
                                }, 30000);
                                // But don't keep the process open just for that!
                                killtimer.unref();

                                // stop taking new requests.
                                server.close();

                                // Let the master know we're dead.  This will trigger a
                                // 'disconnect' in the cluster master, and then it will fork
                                // a new worker.
                                ServiceLocator.Cluster.worker.disconnect();

                                // try to send an error to the request that triggered the problem
                                res.statusCode = 500;
                                res.setHeader('content-type', 'text/plain');
                                res.end('Oops, there was a problem!');
                            } else{
                                GameLog.warn("We are only running one one worker, reforking");
                                res.statusCode = 500;
                                res.setHeader('content-type', 'text/plain');
                                res.end('Oops, there was a problem!');
                                ServiceLocator.Cluster.worker.destroy(1);
                            }
                        } catch(err2){
                            ServiceLocator.Cluster.worker.destroy(1);
                        }
                    });
                    requestDomain.add(req);
                    requestDomain.add(res);
                    requestDomain.run(function(){
                        next();
                    });
                });


                function startServerIfReady(){

                    if(ServiceLocator.RuntimeSettings.SettingsLoaded){
                        server.listen(listenPort, function() {
                            var maxLifeSpan = 60*60*1000; // 1 Hour
                            var variance = 10 * 10 * 1000; // 10 minutes;
                            ServiceLocator.Constants.WORKER_EXPIRY_TIME = new Date().getTime() + ((maxLifeSpan - variance) + Math.floor(variance * 2 * Math.random()));
                            ServiceLocator.Constants.WORKER_START_TIME = new Date().getTime();
                            ServiceLocator.Constants.WORKER_REQUEST_STATE = "ready";
                            GameLog.system('Worker thread ' + cluster.worker.id + ' initialized and is listening at ' + server.url);
                            ServiceLocator.VersionControl = require("./classes/version-control.js").instance;
                            ServiceLocator.VersionControl.deployment = ServiceLocator.Constants.Deployment;
                            ServiceLocator.VersionControl.ensureInit();
                        });
                    } else{
                        GameLog.system('Worker thread ' + cluster.worker.id + ' stalled and is retrying');
                        setTimeout(startServerIfReady, 250);
                    }

                }
                ServiceLocator.RuntimeSettings.RequestFromMasterThread();
                startServerIfReady();
            } else{
                Object.keys(cluster.workers).forEach(function(id) {
                    cluster.workers[id].on('message', ServiceLocator.Ipc.HandleMessageOnMaster);
                });
                GameLog.log("Master process ready, starting async processor");
                setTimeout(asyncProcess, 500);
            }
        }
    );
}

fs.readFile(__dirname + "/version.txt", 'utf8', function (err,data) {
    var version = "unknown";
    if (!err) {
        var versionParts = data.split('_');
        version = versionParts[1];
    }
    fs.readFile(__dirname + "/env.properties", 'utf8', function (error, data) {
        if(error){
            //console.log("Env definition doesn't exist starting server in inferred context");
            startServer(deployment, version);
        } else{
            //console.log("Env definition exists parsing.");
            var envValues = data.split('\n');
            for(var i = 0; i < envValues.length; i++){
                var line = envValues[i];
                var lineParts = line.split('=');
                if(lineParts.length != 2){
                    continue;
                }
                var key = lineParts[0].trim();
                var val = lineParts[1].trim();
                if(key == "serverEnvironment"){
                    deployment = val;
                }
            }
            ServiceLocator.AppDeployment = deployment;
            startServer(deployment, version, displayVersion);
        }

    });

});

if (cluster.isMaster) {
    var signals = ['SIGINT', 'SIGTERM', 'SIGQUIT'];
    for (i in signals) {
        process.on(signals[i], function() {
            IS_MASTER_SHUTTING_DOWN = true;
            Object.keys(cluster.workers).forEach(function(id) {
                cluster.workers[id].destroy(0);
            });
            process.exit();
        })
    }
}

function asyncProcess(){
   setTimeout(asyncProcess, 500);
   try{
       ServiceLocator.Ipc.SendMessageToAllWorkers("HealthCheck");
       ServiceLocator.RuntimeTool.CheckForNewVersion();
       ServiceLocator.Swrve.CheckProcessQueue();
   }catch(err){
       console.log("caught an error proccessing queue " + err.message);
   }
}