fix timestamps and complete npm collection

metrics-collector/src/index.ts

@@ -7,13 +7,15 @@ import { hideBin } from "yargs/helpers";
 import { collectGhMetrics } from "./gh-metrics";
 import { collectNpmMetrics, saveNpmMetrics } from "./npm-metrics";
 import { collectSonatypeMetrics } from "./sonatype-metrics";
+import { getYesterdayDate } from "./utils";
 
 const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true";
 
 interface Arguments {
   "collect-gh": boolean;
   "collect-npm": boolean;
   "collect-sonatype": boolean;
+  "initial-load-from": string;
 }
 
 const argv = yargs(hideBin(process.argv)).options({
@@ -32,12 +34,38 @@ const argv = yargs(hideBin(process.argv)).options({
     description: "Collect Sonatype metrics",
     default: false,
   },
+  "initial-load-from": {
+    type: "string",
+    description:
+      "Execute initial load of metrics for all selected sources from this date in format YYYY-MM-DD",
+    default: "",
+  },
 }).argv as Arguments;
 
 async function main() {
+  const initialLoadFrom = argv["initial-load-from"];
+  // validate the date format is YYYY-MM-DD
+  if (initialLoadFrom && !/^\d{4}-\d{2}-\d{2}$/.test(initialLoadFrom)) {
+    throw new Error("Invalid date format. Please use YYYY-MM-DD");
+  }
+
+  // by default the metric date is yesterday, because stats services
+  // usually provide data for everything until the previous day
+  const metricDateStr = getYesterdayDate();
+
+  const metricDate = new Date(metricDateStr);
+  const initialLoadFromDate = initialLoadFrom
+    ? new Date(initialLoadFrom)
+    : undefined;
+
   const collectNpm = argv["collect-npm"];
   if (collectNpm) {
-    await collectNpmMetrics();
+    console.info(`\n\n============\n\n>>> Collecting metrics for NPM...`);
+    if (initialLoadFromDate) {
+      await initialLoad(initialLoadFromDate, metricDate, collectNpmMetrics);
+    } else {
+      await collectNpmMetrics(metricDateStr);
+    }
   }
 
   const collectSonatype = argv["collect-sonatype"];
@@ -58,4 +86,18 @@ async function main() {
   }
 }
 
+async function initialLoad(
+  initialLoadFromDate: Date,
+  initialLoadToDate: Date,
+  collectMetrics: (metricDate: string) => Promise<void>
+) {
+  let date = initialLoadFromDate;
+  while (date <= initialLoadToDate) {
+    const dateStr = date.toISOString().split("T")[0];
+    console.log(`\n\n>>> Collecting metrics for date: ${dateStr}`);
+    await collectMetrics(dateStr);
+    date.setDate(date.getDate() + 1);
+  }
+}
+
 main();

metrics-collector/src/npm-metrics.ts

@@ -20,23 +20,24 @@ const dataFilePath = path.join(process.cwd(), "npm_metrics.json");
 const csvFilePath = path.join(process.cwd(), "npm_metrics.csv");
 
 // Push collected metrics to the metrics service
-export async function collectNpmMetrics() {
-  // Total downloads are everything until yesterday
-  const yesterdayDate = getYesterdayDate();
-
+export async function collectNpmMetrics(metricDate: string) {
   for (const pkg of npmPackages) {
-    const { downloads: totalDownloads } = await getNpmDownloadCount(pkg);
+    const { downloads: totalDownloads } = await getNpmDownloadCount(
+      pkg,
+      false,
+      { begin: "1970-01-01", end: metricDate }
+    );
 
     // Collect daily downloads too
     const { downloads: dailyDownloads } = await getNpmDownloadCount(
       pkg,
       false,
-      yesterdayDate
+      { begin: metricDate, end: metricDate }
     );
 
     postNpmMetrics({
       pkg,
-      metricDate: new Date(yesterdayDate),
+      metricDate: new Date(metricDate),
       totalDownloads: totalDownloads,
       dailyDownloads: dailyDownloads,
     });
@@ -114,14 +115,14 @@ async function postNpmMetrics(metric: {
 async function getNpmDownloadCount(
   packageName: string,
   onlyLastMonth?: boolean,
-  singleDay?: string
+  dateRange?: { begin: string; end: string }
 ): Promise<{ downloads: number; start: string }> {
   try {
     let url = `https://api.npmjs.org/downloads/point`;
     if (onlyLastMonth) {
       url += `/last-month/${packageName}`;
-    } else if (singleDay) {
-      url += `/${singleDay}/${packageName}`;
+    } else if (dateRange) {
+      url += `/${dateRange.begin}:${dateRange.end}/${packageName}`;
     } else {
       url += `/1970-01-01:2100-01-01/${packageName}`;
     }