handle errors in the server's workers

lib/hk-util.js

@@ -802,7 +802,9 @@ HK.initializeIndexWorkers = function (Env, config, _cb) {
             //console.log(res);
             response.handle(res.txid, [res.error, res.value]);
         });
-        worker.on('exit', function () {
+
+        var substituteWorker = Util.once(function () {
+            Env.Log.info("SUBSTITUTE_INDEX_WORKER", '');
             var idx = workers.indexOf(worker);
             if (idx !== -1) {
                 workers.splice(idx, 1);
@@ -815,6 +817,15 @@ HK.initializeIndexWorkers = function (Env, config, _cb) {
                 workers.push(w);
             });
         });
+
+        worker.on('exit', substituteWorker);
+        worker.on('close', substituteWorker);
+        worker.on('error', function (err) {
+            substituteWorker();
+            Env.log.error("INDEX_WORKER_ERROR", {
+                error: err,
+            });
+        });
     };
 
     var workerIndex = 0;
@@ -950,8 +961,9 @@ HK.initializeValidationWorkers = function (Env) {
             //console.log(+new Date(), "Received verification response");
             response.handle(res.txid, [res.error, res.value]);
         });
-        // Spawn a new process in one ends
-        worker.on('exit', function () {
+
+        var substituteWorker = Util.once( function () {
+            Env.Log.info("SUBSTITUTE_VALIDATION_WORKER", '');
             var idx = workers.indexOf(worker);
             if (idx !== -1) {
                 workers.splice(idx, 1);
@@ -961,6 +973,16 @@ HK.initializeValidationWorkers = function (Env) {
             workers.push(w);
             initWorker(w);
         });
+
+        // Spawn a new process in one ends
+        worker.on('exit', substituteWorker);
+        worker.on('close', substituteWorker);
+        worker.on('error', function (err) {
+            substituteWorker();
+            Env.Log.error('VALIDATION_WORKER_ERROR', {
+                error: err,
+            });
+        });
     };
     workers.forEach(initWorker);