test: initial integration testing

.gitignore

@@ -10,5 +10,4 @@ docs.json
 log/*
 .ipynb_checkpoints/
 tmp
-experiment.json
-supervisord.conf
+experiment.json

test/integration/Dockerfile

@@ -0,0 +1,6 @@
+FROM integration-test-base
+COPY supervisord.conf /usr/local/etc/supervisord.conf
+COPY supervisord-shutdown.sh /home/apowers/supervisord-shutdown.sh
+
+# run server
+CMD ["supervisord", "-c", "/usr/local/etc/supervisord.conf"]

test/integration/integration-test.ipynb

@@ -0,0 +1,216 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# NHAI Experiment integration testing (2021-03-08)\n",
+    "\n",
+    "Testing module loading and breakpoints"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": [
+     "exact"
+    ]
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "testing magic\n"
+     ]
+    }
+   ],
+   "source": [
+    "%echo testing magic"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "scrolled": true,
+    "tags": [
+     "noerror"
+    ]
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "RedisGraph [class Graph]\n",
+      "Component: action-selection\n",
+      "[ added magic: '%break' which will call function 'shellBreak' ]\n",
+      "[ added magic: '%bk' which will call function 'shellBreak' ]\n",
+      "[ added magic: '%continue' which will call function 'shellContinue' ]\n",
+      "[ added magic: '%cont' which will call function 'shellContinue' ]\n",
+      "[ added magic: '%step' which will call function 'shellStep' ]\n",
+      "[ added magic: '%version' which will call function 'shellVersion' ]\n",
+      "[ added magic: '%watch' which will call function 'shellWatch' ]\n",
+      "Component: vision\n",
+      "Component: delta\n",
+      "Component: str\n",
+      "Component: dex\n",
+      "Component: con\n",
+      "Component: int\n",
+      "Component: wis\n",
+      "Component: char\n",
+      "Component: cap\n",
+      "Component: energy\n",
+      "Component: energy-max\n",
+      "Component: xp\n",
+      "Component: ac\n",
+      "Component: hunger\n",
+      "Component: hp\n",
+      "Component: hp-max\n",
+      "Component: cond\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{\n",
+       "  runNethack: [AsyncFunction: runNethack],\n",
+       "  crl: {\n",
+       "    init: [AsyncFunction: init],\n",
+       "    Action: [class Action],\n",
+       "    ActionEvent: [class ActionEvent extends EventBase],\n",
+       "    ActionSelection: [class ActionSelection extends Component],\n",
+       "    Breakpoint: [class Breakpoint extends EventFilter],\n",
+       "    Component: [class Component],\n",
+       "    Config: [class Config],\n",
+       "    Context: [class Context extends Component],\n",
+       "    Edge: [class Edge extends TransientObject],\n",
+       "    EventBase: [class EventBase],\n",
+       "    EventBusBase: [class EventBusBase extends EventEmitter],\n",
+       "    EventFilter: [class EventFilter],\n",
+       "    EventListener: [class EventListener],\n",
+       "    FeatureExtractor: [class FeatureExtractor extends Component],\n",
+       "    GraphDb: [class GraphDb],\n",
+       "    GraphEdge: undefined,\n",
+       "    GraphNode: undefined,\n",
+       "    HtmlTemplate: [class HtmlTemplate],\n",
+       "    Intrinsic: [class Intrinsic extends Component],\n",
+       "    Jupyter: [class Jupyter],\n",
+       "    Log: [class Log],\n",
+       "    Node: [class Node extends TransientObject],\n",
+       "    Perception: [class Perception extends Component],\n",
+       "    PerceptionEvent: [class PerceptionEvent extends EventBase],\n",
+       "    PerceptionModule: undefined,\n",
+       "    Pipeline: [class Pipeline],\n",
+       "    PipelineStage: [class PipelineStage],\n",
+       "    Significance: [class Significance extends Component],\n",
+       "    Schema: [class Schema],\n",
+       "    SignificanceEvent: [class SignificanceEvent extends EventBase],\n",
+       "    StatusImage: [class StatusImage],\n",
+       "    Synchronize: [class Synchronize],\n",
+       "    Utility: [class Utility],\n",
+       "    Trace: [class Trace],\n",
+       "    TransientObject: [class TransientObject] { cache: Map(0) {} }\n",
+       "  }\n",
+       "}"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "let {runNethack, crl: {Config}} = require(\"../../main\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": [
+     "noerror"
+    ]
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Map(23) {\n",
+       "  'app-name' => 'nhai',\n",
+       "  'app-version' => '0.0.0',\n",
+       "  'debug-break-on-entry' => true,\n",
+       "  'debug-sync-environment' => true,\n",
+       "  'environment-synchronous' => true,\n",
+       "  'environment-sync-watchdog-timeout' => 90000,\n",
+       "  'environment-async-time' => 100,\n",
+       "  'graphdb-name' => 'nhai',\n",
+       "  'html-template-dir' => '/Users/ampower/Projects/personal/nhai/assets/hbs',\n",
+       "  'locale' => 'default',\n",
+       "  'log-level' => 'debug',\n",
+       "  'log-patch-console' => true,\n",
+       "  'log-start-msg' => false,\n",
+       "  'log-force-color' => true,\n",
+       "  'log-file-enabled' => false,\n",
+       "  'log-file-prefix' => 'nhai-',\n",
+       "  'log-file-suffix' => '.log',\n",
+       "  'log-file-path' => '.',\n",
+       "  'random-seed' => 'goodluck!',\n",
+       "  'redisgraph-server' => '127.0.0.1',\n",
+       "  'redisgraph-port' => 6379,\n",
+       "  'redisgraph-options' => undefined,\n",
+       "  'schema-dir' => '/Users/ampower/Projects/personal/nhai/assets/schema'\n",
+       "}"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "Config.load({\n",
+    "    \"debug-break-on-entry\": true,\n",
+    "    \"log-level\": \"debug\",\n",
+    "    \"environment-sync-watchdog-timeout\": 90000\n",
+    "})"
+   ]
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Tags",
+  "kernelspec": {
+   "display_name": "Javascript (Node.js)",
+   "language": "javascript",
+   "name": "javascript"
+  },
+  "language_info": {
+   "file_extension": ".js",
+   "mimetype": "application/javascript",
+   "name": "javascript",
+   "version": "14.16.0"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {
+    "height": "calc(100% - 180px)",
+    "left": "10px",
+    "top": "150px",
+    "width": "358.391px"
+   },
+   "toc_section_display": true,
+   "toc_window_display": true
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

test/integration/supervisord-shutdown.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# event manager for supervisord: shuts down the daemon when one of it's processes exits
+# see also: http://supervisord.org/events.html
+
+printf "READY\n";
+
+while read line; do
+  echo "Processing Event: $line" >&2;
+  kill -3 $(cat "/var/run/supervisord.pid")
+done < /dev/stdin

test/integration/supervisord.conf

@@ -0,0 +1,13 @@
+[include]
+files=/usr/local/etc/supervisord.base.conf
+
+[program:papermill]
+command=papermill /home/apowers/integration-test.ipynb /home/apowers/integration-output.ipynb
+priority=100
+startsecs=0
+autostart=true
+autorestart=false
+
+[eventlistener:processes]
+command=/home/apowers/supervisord-shutdown.sh
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL

test/integration/test.js

@@ -0,0 +1,93 @@
+const {assert} = require("chai");
+const fs = require("fs");
+const path = require("path");
+const {spawn} = require("child_process");
+const tmp = require("tmp");
+
+const jupyterInputPath = path.resolve(__dirname, "integration-test.ipynb");
+const expectedResults = readJson(jupyterInputPath);
+let testResults;
+console.log("CWD", process.cwd());
+
+// validate the outputs of the integration test
+describe("integration testing", function() {
+    before(async function() {
+        this.slow(5 * 60 * 1000);
+        this.timeout(5 * 60 * 1000);
+
+        // build main nhai docker image
+        await spawnAsync("docker build --build-arg JUPYTER_FILE=test/integration/integration-test.ipynb --tag integration-test-base .");
+        // build integration testing nhai docker image
+        await spawnAsync("docker build --tag integration-test-image test/integration");
+        // remove old container to prevent name conflicts
+        await spawnAsync("docker container rm integration-test-container");
+        // run integration tests
+        await spawnAsync("docker run --name integration-test-container integration-test-image supervisord -c /usr/local/etc/supervisord.conf");
+        // copy over test results
+        let tmpName = tmp.tmpNameSync();
+        console.log("tmp name", tmpName);
+        await spawnAsync(`docker cp integration-test-container:/home/apowers/integration-output.ipynb ${tmpName}`);
+        testResults = readJson(tmpName);
+        fs.rmSync(tmpName);
+    });
+
+    it("has same number of tests", function() {
+        console.log("expectedResults.cells.length", expectedResults.cells.length);
+        console.log("testResults.cells.length", testResults.cells.length);
+        assert.strictEqual(expectedResults.cells.length, testResults.cells.length);
+    });
+
+    it("tests cells", async function() {
+        expectedResults.cells.forEach((c, i) => cellTest(i));
+    });
+});
+
+function cellTest(cellNum) {
+    process.stdout.write(`Testing Cell ${cellNum}: `);
+    let expectedCell = expectedResults.cells[cellNum];
+    let testResultCell = testResults.cells[cellNum];
+
+    if (expectedCell.cell_type !== "code") {
+        console.log("Skipped (Not Code).");
+        return;
+    }
+
+    if (expectedCell.metadata.tags.includes("exact")) {
+        console.log("Exact Match.");
+        assert.deepEqual(expectedCell.outputs, testResultCell.outputs);
+    }
+
+    if (expectedCell.metadata.tags.includes("noerror")) {
+        console.log("No Error.");
+        // TODO: look for error
+    }
+}
+
+function spawnAsync(str) {
+    console.log("RUNNING:", str);
+    let args = str.split(" ");
+    let cmd = args.shift();
+
+    if (typeof cmd !== "string") {
+        throw new Error("expected 'cmd' to be string, got", cmd);
+    }
+
+    let opts = {
+        stdio: "inherit",
+    };
+
+    return new Promise((resolve, reject) => {
+        spawn(cmd, args, opts).on("close", (code) => {
+            if (code === 0) {
+                return resolve(code);
+            }
+
+            reject(code);
+        });
+    });
+}
+
+function readJson(file) {
+    let contents = fs.readFileSync(file, {encoding: "utf8"});
+    return JSON.parse(contents);
+}