feat(scaffold-agent): add tool to scaffold new agents (#6)

.prettierrc

@@ -0,0 +1,6 @@
+{
+  "printWidth": 100,
+  "arrowParens": "avoid",
+  "singleQuote": true,
+  "trailingComma": "all"
+}

packages/scaffold-agent/.gitignore

@@ -0,0 +1 @@
+agent

packages/scaffold-agent/README.md

@@ -0,0 +1,4 @@
+
+```console
+./bin/scaffold-agent.js <v1-project> <destination-folder>
+```

packages/scaffold-agent/bin/scaffold-agent.js

@@ -0,0 +1,18 @@
+#! /usr/bin/env node
+
+const { program } = require('commander');
+const { generateProject } = require('../src/main');
+
+program
+  .name('scaffold-agent')
+  .description('CLI to generate a modern agent from a legacy one')
+  .version(require('../package.json').version)
+  .argument('<projectFolder>', 'Path to the project folder')
+  .argument('<destinationFolder>', 'Path to the destination folder')
+
+program.parse(process.argv);
+
+generateProject(program.args[0], program.args[1]).catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

packages/scaffold-agent/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@forestadmin-experimental/scaffold-agent",
+  "version": "0.0.1",
+  "license": "GPL-3.0",
+  "bin": "./bin/scaffold-agent.js",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@forestadmin/agent": "^1.11.0",
+    "@forestadmin/datasource-customizer": "^1.9.0",
+    "@forestadmin/datasource-sql": "^1.6.7",
+    "@forestadmin/datasource-toolkit": "^1.5.0",
+    "commander": "^11.0.0",
+    "dotenv": "^16.3.1",
+    "ejs": "^3.1.9",
+    "inflection": "^2.0.1",
+    "pg": "^8.11.0",
+    "prettier": "^2.8.8"
+  }
+}

packages/scaffold-agent/src/loaders.js

@@ -0,0 +1,75 @@
+const { readFileSync } = require("node:fs");
+const { join } = require("node:path");
+const { createSqlDataSource } = require("@forestadmin/datasource-sql");
+const { SchemaGenerator } = require("@forestadmin/agent");
+const { DataSourceCustomizer } = require("@forestadmin/datasource-customizer");
+const dotenv = require("dotenv");
+
+function loadOldSchema(projectFolder) {
+  try {
+    const oldSchemaPath = join(projectFolder, ".forestadmin-schema.json");
+    return JSON.parse(readFileSync(oldSchemaPath, "utf8"));
+  } catch {
+    console.error(
+      "Could not find a .forestadmin-schema.json file in the project folder."
+    );
+    process.exit(1);
+  }
+}
+
+function loadEnv(projectFolder) {
+  const mandatoryEntries = [
+    "DATABASE_URL",
+    "FOREST_ENV_SECRET",
+    "FOREST_AUTH_SECRET",
+  ];
+
+  try {
+    const dotEnvPath = join(projectFolder, ".env");
+    const env = dotenv.parse(readFileSync(dotEnvPath, "utf8"));
+
+    for (const mandatoryEntry of mandatoryEntries) {
+      if (!env[mandatoryEntry]) {
+        console.error(`Could not find a ${mandatoryEntry} in the .env file.`);
+        process.exit(1);
+      }
+    }
+
+    return env;
+  } catch {
+    console.error("Could not find a .env file in the project folder.");
+    process.exit(1);
+  }
+}
+
+async function loadNewSchema(databaseUrl) {
+  try {
+    const factory = createSqlDataSource({
+      uri: databaseUrl,
+      sslMode: "preferred",
+    });
+    const sql = await factory(() => { });
+    const customizer = new DataSourceCustomizer().addDataSource(
+      async () => sql
+    );
+    const newSchema = await SchemaGenerator.buildSchema(
+      await customizer.getDataSource(),
+      null
+    );
+
+    await sql.sequelize.close();
+
+    return newSchema;
+  } catch {
+    console.error(
+      "Failed to connect to the database. Please check your DATABASE_URL environment variable."
+    );
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  loadEnv,
+  loadOldSchema,
+  loadNewSchema,
+};

packages/scaffold-agent/src/main.js

@@ -0,0 +1,122 @@
+const { pluralize } = require('inflection');
+const { join } = require('node:path');
+
+const { render } = require('./utils/file');
+const { toDashCase, toSnakeCase } = require('./utils/string');
+const { hasCustomizationFile } = require('./template-helpers/_shared');
+const { loadEnv, loadNewSchema, loadOldSchema } = require('./loaders');
+
+function getCollectionsByIntegration(oldSchema) {
+  const collections = {};
+  for (const collection of oldSchema.collections) {
+    let folder = 'base';
+    if (collection.isVirtual) folder = collection.integration ?? 'others';
+
+    collections[folder] ??= [];
+    collections[folder].push(collection);
+  }
+
+  return collections;
+}
+
+function linkOldSchemaAndNewSchema(oldSchema, newSchema) {
+  // Link schema together
+  for (const oldCollection of oldSchema.collections) {
+    // Find mathing collection in new schema
+    const newCollection =
+      newSchema.collections.find(newCollection => newCollection.name === oldCollection.name) ??
+      newSchema.collections.find(
+        newCollection => newCollection.name === pluralize(oldCollection.name),
+      ) ??
+      newSchema.collections.find(
+        newCollection => newCollection.name === toSnakeCase(oldCollection.name),
+      ) ??
+      newSchema.collections.find(
+        newCollection => newCollection.name === toSnakeCase(pluralize(oldCollection.name)),
+      );
+
+    if (newCollection) {
+      // Save links
+      oldCollection.newCollection = newCollection;
+      newCollection.oldCollection = oldCollection;
+
+      // Do the same for fields
+      for (const oldField of oldCollection.fields) {
+        // Find matching field in new schema
+        let newField =
+          newCollection.fields.find(newField => newField.field === oldField.field) ??
+          newCollection.fields.find(newField => newField.field === toSnakeCase(oldField.field));
+
+        if (!newField) {
+          const newFieldCandidates = newCollection.fields.filter(newField =>
+            newField.field.startsWith(`${oldField.field}_through_`),
+          );
+
+          if (newFieldCandidates.length) {
+            newField = newFieldCandidates[0];
+            if (newFieldCandidates.length > 1) {
+              oldField.newFieldCandidates = newFieldCandidates;
+            }
+          }
+        }
+
+        if (newField) {
+          oldField.newField = newField;
+          newField.oldField = oldField;
+        }
+      }
+    }
+  }
+}
+
+function writeFiles(destPath, env, newSchema, collectionsByIntegration) {
+  const variables = { collectionsByIntegration, newSchema, env };
+
+  render('package', join(destPath, 'package.json'), variables, false);
+  render('tsconfig', join(destPath, 'tsconfig.json'), variables, false);
+  render('env', join(destPath, '.env'), variables, false);
+  render('main', join(destPath, 'src/main.ts'), variables);
+  render('typings', join(destPath, 'src/typings.ts'), variables);
+
+  Object.entries(collectionsByIntegration).forEach(([integration, oldCollections]) => {
+    const integrationVariables = { ...variables, integration, oldCollections };
+    const dataSourceFolder = `src/datasources/${toDashCase(integration)}`;
+
+    if (integration === 'base') {
+      const filepath = join(destPath, `${dataSourceFolder}/index.ts`);
+      render('datasource-base', filepath, integrationVariables);
+    } else {
+      const filepath = join(destPath, `${dataSourceFolder}/index.ts`);
+      render('datasource-index', filepath, integrationVariables);
+    }
+
+    for (const oldCollection of oldCollections) {
+      const filename = toDashCase(oldCollection.name);
+      const collectionVars = { ...integrationVariables, oldCollection };
+
+      if (hasCustomizationFile(oldCollection)) {
+        const filepath = join(destPath, `src/customizations/${filename}.ts`);
+        render('customization', filepath, collectionVars);
+      }
+
+      if (integration !== 'base') {
+        const filepath = join(destPath, `${dataSourceFolder}/${filename}.ts`);
+        render('datasource-collection', filepath, collectionVars);
+      }
+    }
+  });
+}
+
+async function generateProject(projectFolder, destPath) {
+  const env = loadEnv(projectFolder);
+  const oldSchema = loadOldSchema(projectFolder);
+  const newSchema = await loadNewSchema(env.DATABASE_URL);
+
+  linkOldSchemaAndNewSchema(oldSchema, newSchema);
+
+  const collectionsByIntegration = getCollectionsByIntegration(oldSchema);
+
+  writeFiles(destPath, env, newSchema, collectionsByIntegration);
+}
+
+module.exports = { generateProject };

packages/scaffold-agent/src/template-helpers/_shared.js

@@ -0,0 +1,67 @@
+function hasCustomizationFile(collection) {
+  const hasActions = collection.actions.length > 0;
+  const hasFields = collection.fields.some((field) => field.isVirtual);
+  const hasSegments = collection.segments.length > 0;
+  const isSmart = collection.isVirtual;
+
+  return (
+    (!isSmart && (hasActions || hasFields || hasSegments)) ||
+    (isSmart && (hasActions || hasSegments))
+  );
+}
+
+function convertType(type) {
+  if (typeof type === "string") {
+    // Casing is not consistent in the schema of forest-express
+    return type[0].toUpperCase() + type.slice(1).toLocaleLowerCase();
+  }
+
+  if (Array.isArray(type)) {
+    // [{ field: 'name', type: 'String' }, { field: 'age', type: 'Number' }]
+    if (
+      type.every((t) => typeof t === "object" && "field" in t && "type" in t)
+    ) {
+      return Object.fromEntries(
+        type.map(({ field, type }) => [field, convertType(type)])
+      );
+    }
+
+    // ['String', 'Number']
+    return type.map(convertType);
+  }
+
+  // { name: 'String', age: 'Number' }
+  if (typeof type === "object") {
+    return Object.fromEntries(
+      Object.entries(type).map(([k, v]) => [k, convertType(v)])
+    );
+  }
+
+  return null;
+}
+
+function stringify(value) {
+  function replacer(key, value) {
+    if (typeof value === "function") {
+      return `$$$$${value.toString()}$$$$`;
+    }
+
+    if (value instanceof Set) {
+      const values = JSON.stringify(Array.from(value)).replace(/"/g, "'");
+
+      return `$$$$new Set(${values})$$$$`;
+    }
+
+    return value;
+  }
+
+  return JSON.stringify(value, replacer)
+    .replace(/\$\$\$\$"/g, "")
+    .replace(/"\$\$\$\$/g, "");
+}
+
+module.exports = {
+  convertType,
+  hasCustomizationFile,
+  stringify,
+};

packages/scaffold-agent/src/template-helpers/customization.js

@@ -0,0 +1,44 @@
+const { convertType } = require("./_shared");
+
+function computeActionForm(action) {
+  return action.fields.map((field) => {
+    const newField = { label: field.field, type: field.type };
+    if (field.description) newField.description = field.description;
+    if (field.isRequired) newField.isRequired = true;
+
+    if (field.defaultValue !== null) {
+      newField.defaultValue = field.defaultValue;
+    }
+
+    if (field.reference !== null) {
+      newField.type = "Collection";
+      newField.collectionName = field.reference.split(".")[0];
+    }
+
+    if (newField.type === "Enum") {
+      newField.enumValues = field.enums;
+    }
+
+    return newField;
+  });
+}
+
+function getValueOfType(type) {
+  const ourType = convertType(type);
+
+  if (ourType === "Number") return 0;
+  if (ourType === "Boolean") return true;
+  if (ourType === "Date") return new Date().toISOString();
+  if (ourType === "Enum") return field.enums[0];
+  if (ourType === "String") return "<sample>";
+  if (ourType === 'Json') return {};
+  if (Array.isArray(ourType)) return [getValueOfTypes(ourType[0])];
+  if (typeof ourType === "object")
+    return Object.fromEntries(
+      Object.entries(ourType).map(([k, v]) => [k, getValueOfType(v)])
+    );
+
+  return null;
+}
+
+module.exports = { computeActionForm, getValueOfType, convertType };