fix timestamps and complete npm collection

TBD54566975 · Jun 2, 2024 · 94d0305 · 94d0305
1 parent 5b843da
commit 94d0305
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 60 deletions.
diff --git a/metrics-collector/src/index.ts b/metrics-collector/src/index.ts
@@ -5,7 +5,7 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 
 import { collectGhMetrics } from "./gh-metrics";
-import { collectNpmMetrics } from "./npm-metrics";
+import { collectNpmMetrics, saveNpmMetrics } from "./npm-metrics";
 import { collectSonatypeMetrics } from "./sonatype-metrics";
 
 const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true";
@@ -35,20 +35,26 @@ const argv = yargs(hideBin(process.argv)).options({
 }).argv as Arguments;
 
 async function main() {
-  const collectGh = argv["collect-gh"];
   const collectNpm = argv["collect-npm"];
-  const collectSonatype = argv["collect-sonatype"];
-
-  const noArgs = !collectGh && !collectNpm && !collectSonatype;
+  if (collectNpm) {
+    await collectNpmMetrics();
+  }
 
-  if (collectGh || noArgs) {
-    await collectGhMetrics(isLocalPersistence);
+  const collectSonatype = argv["collect-sonatype"];
+  if (collectSonatype) {
+    await collectSonatypeMetrics(isLocalPersistence);
   }
-  if (collectNpm || noArgs) {
-    await collectNpmMetrics(isLocalPersistence);
+
+  const collectGh = argv["collect-gh"];
+  if (collectGh) {
+    await collectGhMetrics();
   }
-  if (collectSonatype || noArgs) {
-    await collectSonatypeMetrics(isLocalPersistence);
+
+  const localCollection = !collectGh && !collectNpm && !collectSonatype;
+  if (localCollection) {
+    await saveNpmMetrics();
+    // await saveGhMetrics();
+    // await saveSonatypeMetrics();
   }
 }
 

diff --git a/metrics-collector/src/npm-metrics.ts b/metrics-collector/src/npm-metrics.ts
@@ -1,7 +1,7 @@
 import * as fs from "fs";
 import * as path from "path";
 import { createObjectCsvWriter } from "csv-writer";
-import { readJsonFile, writeJsonFile } from "./utils";
+import { getYesterdayDate, readJsonFile, writeJsonFile } from "./utils";
 import { postMetric } from "./post-metric";
 
 // Define the npm packages to collect metrics for
@@ -19,10 +19,34 @@ const npmPackages = [
 const dataFilePath = path.join(process.cwd(), "npm_metrics.json");
 const csvFilePath = path.join(process.cwd(), "npm_metrics.csv");
 
-async function collectNpmMetrics(
-  isLocalPersistence: boolean = false,
-  metricDate?: string
-) {
+// Push collected metrics to the metrics service
+export async function collectNpmMetrics() {
+  // Total downloads are everything until yesterday
+  const yesterdayDate = getYesterdayDate();
+
+  for (const pkg of npmPackages) {
+    const { downloads: totalDownloads } = await getNpmDownloadCount(pkg);
+
+    // Collect daily downloads too
+    const { downloads: dailyDownloads } = await getNpmDownloadCount(
+      pkg,
+      false,
+      yesterdayDate
+    );
+
+    postNpmMetrics({
+      pkg,
+      metricDate: new Date(yesterdayDate),
+      totalDownloads: totalDownloads,
+      dailyDownloads: dailyDownloads,
+    });
+  }
+
+  console.info("NPM metrics collected successfully");
+}
+
+// Save collected total downloads and last 30d downloads to a local file
+export async function saveNpmMetrics() {
   const timestamp = new Date().toISOString();
 
   const metrics = [];
@@ -32,61 +56,59 @@ async function collectNpmMetrics(
       getNpmDownloadCount(pkg, true),
       getNpmDownloadCount(pkg),
     ]);
-    let metricDateDownloads;
-    if (metricDate) {
-      metricDateDownloads = await getNpmDownloadCount(pkg, false, metricDate);
-    }
     metrics.push({
       pkg,
       timestamp,
       publishedAt: totalDownloads.start,
       totalDownloads: totalDownloads.downloads,
       lastMonthDownloads: lastMonthDownloads.downloads,
-      metricDate,
-      metricDateDownloads: metricDateDownloads?.downloads,
     });
   }
 
   console.info("NPM metrics collected successfully", { metrics });
 
-  if (isLocalPersistence) {
-    const npmMetrics = readJsonFile(dataFilePath);
-    for (const metric of metrics) {
-      if (!npmMetrics[metric.pkg]) {
-        npmMetrics[metric.pkg] = [];
-      }
-      npmMetrics[metric.pkg].push(metric);
+  const npmMetrics = readJsonFile(dataFilePath);
+  for (const metric of metrics) {
+    if (!npmMetrics[metric.pkg]) {
+      npmMetrics[metric.pkg] = [];
     }
-    writeJsonFile(dataFilePath, npmMetrics);
-    await writeMetricsToCsv(csvFilePath, npmMetrics);
-    console.log(
-      "NPM metrics have been successfully saved to npm_metrics.json and npm_metrics.csv"
-    );
-  } else {
-    await postNpmMetrics(metrics);
+    npmMetrics[metric.pkg].push(metric);
   }
+  writeJsonFile(dataFilePath, npmMetrics);
+  await writeMetricsToCsv(csvFilePath, npmMetrics);
+  console.log(
+    "NPM metrics have been successfully saved to npm_metrics.json and npm_metrics.csv"
+  );
 
   return metrics;
 }
 
-async function postNpmMetrics(metrics: any) {
-  for (const metric of metrics) {
-    const labels = {
-      package: metric.pkg,
-    };
-
-    if (metric.metricDate) {
-      const metricDate = new Date(metric.metricDate);
-      await postMetric(
-        "npm_downloads",
-        metric.metricDateDownloads || 0,
-        labels,
-        metricDate
-      );
-    }
-
-    await postMetric("npm_total_downloads", metric.totalDownloads, labels);
+async function postNpmMetrics(metric: {
+  pkg: string;
+  metricDate: Date;
+  totalDownloads: number;
+  dailyDownloads?: number;
+}) {
+  const labels = {
+    package: metric.pkg,
+  };
+
+  // Push daily downloads if present
+  if (metric.dailyDownloads !== undefined) {
+    await postMetric(
+      "npm_downloads",
+      metric.dailyDownloads,
+      labels,
+      metric.metricDate
+    );
   }
+
+  await postMetric(
+    "npm_total_downloads",
+    metric.totalDownloads,
+    labels,
+    metric.metricDate
+  );
 }
 
 async function getNpmDownloadCount(
@@ -148,5 +170,3 @@ async function writeMetricsToCsv(
 
   await csvWriter.writeRecords(records);
 }
-
-export { collectNpmMetrics };
diff --git a/metrics-collector/src/service-app.ts b/metrics-collector/src/service-app.ts
@@ -34,12 +34,13 @@ const initDb = async () => {
         metric_name VARCHAR(255) NOT NULL,
         value DOUBLE PRECISION NOT NULL,
         labels JSONB,
-        timestamp TIMESTAMPTZ NOT NULL DEFAULT now()
+        metric_timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
+        created_at TIMESTAMPTZ NOT NULL DEFAULT now()
     );
 
     CREATE INDEX IF NOT EXISTS idx_metric_name ON metrics(metric_name);
-    CREATE INDEX IF NOT EXISTS idx_timestamp ON metrics(timestamp);
-    CREATE INDEX IF NOT EXISTS idx_labels ON metrics USING GIN(labels);
+    CREATE INDEX IF NOT EXISTS idx_metric_timestamp ON metrics(metric_timestamp);
+    CREATE INDEX IF NOT EXISTS idx_metric_labels ON metrics USING GIN(labels);
     `;
   await pool.query(createTableQuery);
   console.log("Database initialized");
@@ -84,7 +85,7 @@ const seedDb = async () => {
   );
 
   const insertQuery = `
-    INSERT INTO metrics (metric_name, value, labels, timestamp) VALUES ($1, $2, $3, $4)
+    INSERT INTO metrics (metric_name, value, labels, metric_timestamp) VALUES ($1, $2, $3, $4)
     `;
 
   for (const val of values) {
@@ -114,7 +115,7 @@ app.post("/api/v1/metrics", async (req: any, res: any) => {
 
   try {
     await pool.query(
-      "INSERT INTO metrics (metric_name, value, labels, timestamp) VALUES ($1, $2, $3, $4)",
+      "INSERT INTO metrics (metric_name, value, labels, metric_timestamp) VALUES ($1, $2, $3, $4)",
       [
         metricName,
         value,

diff --git a/metrics-collector/src/utils.ts b/metrics-collector/src/utils.ts
@@ -14,3 +14,9 @@ export function readJsonFile(filePath: string): any {
 export function writeJsonFile(filePath: string, data: any): void {
   fs.writeFileSync(filePath, JSON.stringify(data, null, 2), "utf8");
 }
+
+export const getYesterdayDate = () => {
+  const yesterday = new Date();
+  yesterday.setDate(yesterday.getDate() - 1);
+  return yesterday.toISOString().split("T")[0];
+};