api/test: replace test.js with test-ci.js

.github/workflows/test-services.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
       - id: checkServices
-        run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test-ci get-services)" >> "$GITHUB_OUTPUT"
+        run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test get-services)" >> "$GITHUB_OUTPUT"
 
   test-services:
     needs: check-services
@@ -30,4 +30,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-      - run: pnpm i --frozen-lockfile && node api/src/util/test-ci run-tests-for ${{ matrix.service }}
+      - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}

api/src/misc/run-test.js

@@ -41,4 +41,4 @@ export async function runTest(url, params, expect) {
     if (result.body.status === 'tunnel') {
         // TODO: stream testing
     }
-}
+}

api/src/util/test.js

@@ -1,84 +1,129 @@
-import "dotenv/config";
+import path from "node:path";
 
-import { services } from "../processing/service-config.js";
-import { extract } from "../processing/url.js";
-import match from "../processing/match.js";
-import { loadJSON } from "../misc/load-from-fs.js";
-import { normalizeRequest } from "../processing/request.js";
 import { env } from "../config.js";
+import { runTest } from "../misc/run-test.js";
+import { loadJSON } from "../misc/load-from-fs.js";
+import { Red, Bright } from "../misc/console-text.js";
+import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 
-env.apiURL = 'http://localhost:9000'
-let tests = loadJSON('./src/util/tests.json');
-
-let noTest = [];
-let failed = [];
-let success = 0;
-
-function addToFail(service, testName, url, status, response) {
-    failed.push({
-        service: service,
-        name: testName,
-        url: url,
-        status: status,
-        response: response
-    })
-}
-for (let i in services) {
-    if (tests[i]) {
-        console.log(`\nRunning tests for ${i}...\n`)
-        for (let k = 0; k < tests[i].length; k++) {
-            let test = tests[i][k];
-
-            console.log(`Running test ${k+1}: ${test.name}`);
-            console.log('params:');
-            let params = {...{url: test.url}, ...test.params};
-            console.log(params);
-
-            let chck = await normalizeRequest(params);
-            if (chck.success) {
-                chck = chck.data;
-
-                const parsed = extract(chck.url);
-                if (parsed === null) {
-                    throw `Invalid URL: ${chck.url}`
-                }
+import { services } from "../processing/service-config.js";
 
-                let j = await match({
-                    host: parsed.host,
-                    patternMatch: parsed.patternMatch,
-                    params: chck,
-                });
-                console.log('\nReceived:');
-                console.log(j)
-                if (j.status === test.expected.code && j.body.status === test.expected.status) {
-                    console.log("\n✅ Success.\n");
-                    success++
-                } else {
-                    console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`);
-                    addToFail(i, test.name, test.url, j.body.status, j)
-                }
-            } else {
-                console.log("\n❌ couldn't validate the request JSON.\n");
-                addToFail(i, test.name, test.url, "unknown", {})
+const getTestPath = service => path.join('./src/util/tests/', `./${service}.json`);
+const getTests = (service) => loadJSON(getTestPath(service));
+
+// services that are known to frequently fail due to external
+// factors (e.g. rate limiting)
+const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube']);
+
+const runTestsFor = async (service) => {
+    const tests = getTests(service);
+    let softFails = 0, fails = 0;
+
+    if (!tests) {
+        throw "no such service: " + service;
+    }
+
+    for (const test of tests) {
+        const { name, url, params, expected } = test;
+        const canFail = test.canFail || finnicky.has(service);
+
+        try {
+            await runTest(url, params, expected);
+            console.log(`${service}/${name}: ok`);
+
+        } catch(e) {
+            softFails += !canFail;
+            fails++;
+
+            let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+            if (canFail && process.env.GITHUB_ACTION) {
+                console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
             }
+
+            console.error(`${service}/${name}: ${failText}`);
+            const errorString = e.toString().split('\n');
+            let c = '┃';
+            errorString.forEach((line, index) => {
+                line = line.replace('!=', Red('!='));
+
+                if (index === errorString.length - 1) {
+                    c = '┗';
+                }
+
+                console.error(`   ${c}`, line);
+            });
         }
-        console.log("\n\n")
-    } else {
-        console.warn(`No tests found for ${i}.`);
-        noTest.push(i)
     }
-}
 
-console.log(`✅ ${success} tests succeeded.`);
-console.log(`❌ ${failed.length} tests failed.`);
-console.log(`❔ ${noTest.length} services weren't tested.`);
+    return { fails, softFails };
+}
 
-if (failed.length > 0) {
-    console.log(`\nFailed tests:`);
-    console.log(failed)
+const printHeader = (service, padLen) => {
+    const padding = padLen - service.length;
+    service = service.padEnd(1 + service.length + padding, ' ');
+    console.log(service + '='.repeat(50));
 }
 
-if (noTest.length > 0) {
-    console.log(`\nMissing tests:`);
-    console.log(noTest)
+const action = process.argv[2];
+switch (action) {
+    case "get-services":
+        const fromConfig = Object.keys(services);
+
+        const missingTests = fromConfig.filter(
+            service => {
+                const tests = getTests(service);
+                return !tests || tests.length === 0
+            }
+        );
+
+        if (missingTests.length) {
+            console.error('services have no tests:', missingTests);
+            process.exitCode = 1;
+            break;
+        }
+
+        console.log(JSON.stringify(fromConfig));
+        break;
+
+    case "run-tests-for":
+        env.streamLifespan = 10000;
+        env.apiURL = 'http://x/';
+        randomizeCiphers();
+
+        try {
+            const { softFails } = await runTestsFor(process.argv[3]);
+            process.exitCode = Number(!!softFails);
+        } catch(e) {
+            console.error(e);
+            process.exitCode = 1;
+            break;
+        }
+
+        break;
+    default:
+        const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
+        const failCounters = {};
+
+        env.streamLifespan = 10000;
+        env.apiURL = 'http://x/';
+        randomizeCiphers();
+
+        for (const service in services) {
+            printHeader(service, maxHeaderLen);
+            const { fails, softFails } = await runTestsFor(service);
+            failCounters[service] = fails;
+            console.log();
+
+            if (!process.exitCode && softFails)
+                process.exitCode = 1;
+        }
+
+        console.log('='.repeat(50 + maxHeaderLen));
+        console.log(
+            Bright('total fails:'),
+            Object.values(failCounters).reduce((a, b) => a + b)
+        );
+        for (const [ service, fails ] of Object.entries(failCounters)) {
+            if (fails) console.log(`${Bright(service)} fails: ${fails}`);
+        }
 }