Merge pull request #44 from sophie0730/develop

Develop

server/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "test": "mocha",
     "build": "esbuild pulseTracker.js --bundle --outfile=out.mjs --platform=node --target=node18.18 --format=esm  --banner:js='import { createRequire as topLevelCreateRequire } from \"module\"; const require = topLevelCreateRequire(import.meta.url);'",
-    "package": "bun build ./pulsetracker.js --compile --target node --outfile=./pulsetracker"
+    "package": "bun build ./pulseTracker.js --compile --target node --outfile=./pulsetracker"
   },
   "pre-commit": [
     "lint"

server/pulseTracker.js

@@ -1,5 +1,40 @@
 /* eslint-disable no-undef */
 /* eslint-disable no-unused-vars */
 import * as server from './app.js';
-import * as storeWorker from './workers/store.js';
-import * as alertWorker from './workers/alert.js';
+import { storeTimeout } from './utils/yml-util.js';
+
+const storeWorkerPath = './workers/store.js';
+const alertWorkerPath = './workers/alert.js';
+
+function createWorker(scriptPath) {
+  const worker = Bun.spawn(['bun', scriptPath], {
+    onExit(proc, exitCode, signalCode, error) {
+      console.log(`child process exit, code: ${exitCode}, signal: ${signalCode}`);
+      console.log('child process restarting...');
+      createWorker(scriptPath);
+    },
+    ipc(message) {
+      if (message.type === 'heartbeat') {
+        worker.lastHeartbeat = Date.now();
+      }
+    },
+    stdio: ['inherit', 'inherit', 'inherit'],
+  });
+
+  worker.lastHeartbeat = Date.now();
+  return worker;
+}
+
+const storeWorker = createWorker(storeWorkerPath);
+const alertWorker = createWorker(alertWorkerPath);
+
+setInterval(() => {
+  if (Date.now() - storeWorker.lastHeartbeat > storeTimeout * 1000 + 10000) {
+    console.log('Heartbeat from store worker is lost. Restarting...');
+    storeWorker.kill();
+  }
+  if (Date.now() - alertWorker.lastHeartbeat > storeTimeout * 1000 + 10000) {
+    console.log('Heartbeat from alert worker is lost. Restarting...');
+    alertWorker.kill();
+  }
+}, 5000);

server/workers/alert.js

@@ -20,3 +20,9 @@ function scheduleCheckAlerts() {
 }
 
 scheduleCheckAlerts();
+
+function sendHeartbeat() {
+  process.send({ type: 'heartbeat' });
+}
+
+setInterval(sendHeartbeat, TIMEOUT);

server/workers/store.js

@@ -17,3 +17,9 @@ function scheduleOperation(operation) {
 
 scheduleOperation(storeExporterMetrices);
 scheduleOperation(storeExporterStatus);
+
+function sendHeartbeat() {
+  process.send({ type: 'heartbeat' });
+}
+
+setInterval(sendHeartbeat, TIMEOUT);