build: add experiment management to gulp and plop

.dockerignore

@@ -0,0 +1,13 @@
+node_modules
+docs
+.DS_Store
+npm-debug.log
+coverage
+.nyc_output
+*~
+docs.json
+.nhairc.js
+log/*
+.ipynb_checkpoints
+./tmp
+.github

.gitignore

@@ -9,3 +9,6 @@ docs.json
 .nhairc.js
 log/*
 .ipynb_checkpoints/
+tmp
+experiment.json
+supervisord.conf

Dockerfile

@@ -0,0 +1,11 @@
+FROM ghcr.io/apowers313/nhai-base:latest
+
+COPY ./package.json /home/apowers
+COPY ./package-lock.json /home/apowers
+RUN npm install
+ARG JUPYTER_FILE=unset
+COPY $JUPYTER_FILE /home/apowers
+COPY . /home/apowers
+
+#COPY ./supervisord.conf /usr/local/etc/supervisord.conf
+#CMD ["supervisord", "-c", "/usr/local/etc/supervisord.conf"]

assets/plop/plopfile.js

@@ -1,36 +1,35 @@
-function getDate() {
-    let numericDateTime = {
-        year: "numeric",
-        month: "2-digit",
-        day: "2-digit",
-        // hour: "2-digit",
-        // minute: "2-digit",
-        // second: "2-digit",
-        // hour12: false,
-        // timeZoneName: "short",
-    };
-    let d = new Intl.DateTimeFormat("default", numericDateTime)
-        .formatToParts()
-        .reduce((acc, v) => Object({... acc, [v.type]: v.value}), {});
-
-    console.log("date", d);
-    return `${d.year}-${d.month}-${d.day}`;
+const path = require("path");
+const fs = require("fs");
+const {spawn} = require("child_process");
+const projectDir = path.resolve(__dirname, "../..");
+const expConfFile = path.resolve(projectDir, "experiment.json");
+let expConf;
+try {
+    expConf = require(expConfFile);
+} catch (e) {
+    expConf = {active: false, answers: {}, date: {}};
 }
-let date = getDate();
 
 // eslint-disable-next-line jsdoc/require-jsdoc
 module.exports = function(plop) {
-    plop.setGenerator("jupyter", {
-        description: "sets up a Jupyter experiment for NHAI",
+    plop.setGenerator("experiment", {
+        description: "sets up an experiment for NHAI",
         prompts: [
-            // experiment description
-            // {
-            //     type: "input",
-            //     name: "date",
-            //     message: "date of the experiment",
-            //     default: "2020-03-06",
-            //     when: true,
-            // },
+            // continue experiment?
+            {
+                type: "confirm",
+                name: "continue",
+                message: `would you like to continue the experiment "${expConf.answers.title}" (${expConf.date.dashDate})?`,
+                default: true,
+                when: expConf.active,
+            },
+            {
+                type: "confirm",
+                name: "push",
+                message: `would you like to archive the docker container "${expConf.dockerContainer}?"`,
+                default: true,
+                when: (a) => (expConf.active && !a.continue),
+            },
             {
                 type: "input",
                 name: "title",
@@ -46,28 +45,174 @@ module.exports = function(plop) {
 
                     return true;
                 },
+                when: (a) => !a.continue,
             },
             {
                 type: "input",
                 name: "desc",
                 message: "a long description of the experiment",
                 default: "TODO",
-
-            },
-        ],
-        actions: [
-            // copy notebook file
-            {
-                type: "add",
-                data: {date},
-                path: "{{date}}_{{snakeCase title}}.ipynb",
-                templateFile: "template-notebook.ipynb",
-            },
-            // TODO: start Jupyter
-            function(answers) {
-                console.log("ANSWERS:", answers);
-                console.log("getDate", getDate());
+                when: (a) => !a.continue,
             },
         ],
+        actions: function(answers) {
+            let actions = [];
+
+            if (answers.continue) {
+                return restoreExperiment(answers);
+            }
+
+            if (expConf.active) {
+                actions = archiveExperiment(answers);
+            }
+
+            return actions.concat(createExperiment(answers));
+        },
+    });
+
+    plop.setGenerator("archive", {
+        descrption: "saves the current experiment in an external repository",
+        prompts: [],
+        actions: function() {
+            return archiveExperiment({push: true}, false);
+        },
     });
 };
+
+function createExperiment(answers) {
+    let actions = [];
+    let title = answers.title.toLowerCase().replace(/ /g, "-");
+    let newJupyterFile = `${date.dashDate}_${title}.ipynb`;
+
+    // let newJupyterFile = path.resolve(projectDir, "experiments", `${date.dashDate}_${title}.ipynb`);
+    // process.env.NHAI_JUPYTER_FILE = newJupyterFile;
+
+    // create experiment.json
+    actions.push(() => {
+        console.log("creating experiment...");
+        expConf.date = date;
+        expConf.active = true;
+        expConf.jupyterFile = newJupyterFile;
+        expConf.answers = answers;
+        expConf.dockerContainer = `nhai-container-${date.compactDate}-${title}`;
+        expConf.dockerImage = `nhai-image-${date.compactDate}`;
+        saveConf();
+    });
+
+    // convert plopfile to Jupyter file
+    actions.push({
+        type: "add",
+        data: {date: `${date.dashDate}`},
+        path: `../../${newJupyterFile}`,
+        templateFile: "template-notebook.ipynb",
+    });
+
+    // build docker image
+    actions.push(async() => {
+        console.log("running build...");
+        await spawnAsync(... `docker build --build-arg JUPYTER_FILE=./${newJupyterFile} --tag ${expConf.dockerImage} .`.split(" "));
+        fs.rmSync(`./${newJupyterFile}`);
+    });
+
+    // run docker image
+    actions.push(async() => {
+        console.log("running image...");
+        return spawnAsync(... `docker run -p 6379:6379 -p 8080:8080 -p 8888:8888 -it --name ${expConf.dockerContainer} ${expConf.dockerImage}`.split(" "));
+    });
+
+    return actions;
+}
+
+function restoreExperiment(answers) {
+    let actions = [];
+
+    // docker container start
+    actions.push(async() => {
+        return spawnAsync(... `docker container start -ai ${expConf.dockerContainer}`.split(" "));
+    });
+
+    return actions;
+}
+
+function archiveExperiment(answers, deactivate = true) {
+    let actions = [];
+
+    // set experiment.json to inactive
+    if (deactivate) {
+        actions.push(() => {
+            expConf.active = false;
+            saveConf();
+        });
+    }
+
+    // extract Jupyter file from docker
+    actions.push(async() => {
+        console.log("copying file...");
+        let from = `${expConf.dockerContainer}:/home/apowers/${path.basename(expConf.jupyterFile)}`;
+        let to = path.resolve(projectDir, "./experiments");
+        return spawnAsync(... `docker cp ${from} ${to}`.split(" "));
+    });
+
+    if (answers.push) {
+        // push Docker container to repo
+        actions.push(async() => {
+            console.log("storing docker...");
+            return spawnAsync(... `docker push ${expConf.dockerContainer}`.split(" "));
+        });
+    }
+
+    return actions;
+}
+
+function saveConf() {
+    fs.writeFileSync(expConfFile, JSON.stringify(expConf, null, 4), {encoding: "utf8"});
+}
+
+function spawnAsync(cmd, ... args) {
+    if (typeof cmd !== "string") {
+        throw new Error("expected 'cmd' to be string, got", cmd);
+    }
+
+    let opts = {
+        stdio: "inherit",
+    };
+
+    // // XXX TODO -- this is for debugging only
+    // args.unshift(cmd);
+    // cmd = "echo";
+    console.log("CMD:", cmd);
+    console.log("ARGS:", args);
+
+    return new Promise((resolve, reject) => {
+        spawn(cmd, args, opts).on("close", (code) => {
+            if (code === 0) {
+                return resolve(code);
+            }
+
+            reject(code);
+        });
+    });
+}
+
+function getDate() {
+    let numericDateTime = {
+        year: "numeric",
+        month: "2-digit",
+        day: "2-digit",
+        hour: "2-digit",
+        minute: "2-digit",
+        second: "2-digit",
+        hour12: false,
+        timeZoneName: "short",
+    };
+    let d = new Intl.DateTimeFormat("default", numericDateTime)
+        .formatToParts()
+        .reduce((acc, v) => Object({... acc, [v.type]: v.value}), {});
+
+    d.dashDate = `${d.year}-${d.month}-${d.day}`;
+    d.compactDate = `${d.year}${d.month}${d.day}`;
+
+    return d;
+}
+let date = getDate();
+

gulpfile.js

@@ -20,6 +20,7 @@ const markdown = ["**/*.md"];
 const documentation = [... sources, ... markdown, ... css];
 const all = [... js, ... support];
 const jsDocConfig = require("./.jsdoc-conf.json");
+const nodePlop = require("node-plop");
 
 /* ************
  * TESTING
@@ -167,6 +168,31 @@ function audit(done) {
     let aud = spawn(cmd, args, opts).on("close", done);
 }
 
+/* ************
+ * EXPERIMENT
+ **************/
+async function experiment() {
+    return doPlop("experiment");
+}
+
+async function experimentSave() {
+    return doPlop("archive");
+}
+
+async function doPlop(cmd) {
+    const plop = nodePlop("./assets/plop/plopfile.js");
+    const exp = plop.getGenerator(cmd);
+    console.log("exp", exp);
+    let answers = await exp.runPrompts();
+    let res = await exp.runActions(answers);
+    console.log("res", res);
+}
+
+/* ************
+ * INTEGRATION TESTING
+ **************/
+async function integration() {}
+
 module.exports = {
     audit,
     test,
@@ -181,4 +207,9 @@ module.exports = {
     "dev:coverage": watchCoverage,
     "dev:docs": watchDocs,
     "dev:main": watchMain,
+    experiment,
+    "exp": experiment,
+    "exp:run": experiment,
+    "exp:save": experimentSave,
+    integration,
 };

package.json

@@ -56,6 +56,7 @@
     "magicpatch-test": "^1.1.2",
     "mocha": "^8.2.1",
     "mockery": "^2.1.0",
+    "node-plop": "^0.26.2",
     "npm-watch": "^0.7.0",
     "nyc": "^15.1.0",
     "pinst": "^2.1.1",