diff --git a/metrics-collector/package.json b/metrics-collector/package.json index 2af71ad..40b5b53 100644 --- a/metrics-collector/package.json +++ b/metrics-collector/package.json @@ -18,6 +18,7 @@ "@octokit/types": "^13.5.0", "@types/node": "^20.12.12", "csv-writer": "^1.6.0", + "date-fns": "^3.6.0", "dotenv": "^16.4.5", "express": "^4.19.2", "faker": "^6.6.6", diff --git a/metrics-collector/pnpm-lock.yaml b/metrics-collector/pnpm-lock.yaml index 665bfc5..7f73815 100644 --- a/metrics-collector/pnpm-lock.yaml +++ b/metrics-collector/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: csv-writer: specifier: ^1.6.0 version: 1.6.0 + date-fns: + specifier: ^3.6.0 + version: 3.6.0 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -222,6 +225,9 @@ packages: csv-writer@1.6.0: resolution: {integrity: sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==} + date-fns@3.6.0: + resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} + debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -774,6 +780,8 @@ snapshots: csv-writer@1.6.0: {} + date-fns@3.6.0: {} + debug@2.6.9: dependencies: ms: 2.0.0 diff --git a/metrics-collector/src/index.ts b/metrics-collector/src/index.ts index c05b5bc..5688a63 100644 --- a/metrics-collector/src/index.ts +++ b/metrics-collector/src/index.ts @@ -136,12 +136,6 @@ async function initialLoad( `Initial load from ${initialLoadFromDate} to ${initialLoadToDate} with date ${date}` ); - // if (monthlyInterval) { - // // Change the date to the first day of the month - // date.setDate(0); - // } - // console.info(`Date after setting to first day of the month: ${date}`); - while (date <= initialLoadToDate) { const dateStr = date.toISOString().split("T")[0]; console.log(`\n\n>>> Collecting metric ${metricName} for date: ${dateStr}`); diff --git a/metrics-collector/src/sonatype-metrics.ts b/metrics-collector/src/sonatype-metrics.ts index 47df99c..14078e2 100644 --- a/metrics-collector/src/sonatype-metrics.ts +++ b/metrics-collector/src/sonatype-metrics.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import { createObjectCsvWriter } from "csv-writer"; +import { parse, format, subMonths } from "date-fns"; import { fetchWithRetry, readJsonFile, writeJsonFile } from "./utils"; import { postMetric } from "./post-metric"; @@ -27,24 +28,28 @@ export async function collectSonatypeMetrics(metricDate: string) { continue; // TODO: add parameterized filter } + const reportPeriod = getLastMonthPeriod(metricDate); + const reportPeriodWithoutHyphen = reportPeriod.replace("-", ""); + const rawDownloads = await getArtifactStats( projectId, groupId, artifact, "raw", - metricDate + reportPeriodWithoutHyphen ); const uniqueIPs = await getArtifactStats( projectId, groupId, artifact, "ip", - metricDate + reportPeriodWithoutHyphen ); await postSonatypeMavenMetrics({ artifact, metricDate: new Date(metricDate), + reportPeriod, rawDownloads: rawDownloads.total, uniqueIPs: uniqueIPs.total, }); @@ -56,12 +61,14 @@ export async function collectSonatypeMetrics(metricDate: string) { async function postSonatypeMavenMetrics(metric: { artifact: string; metricDate: Date; + reportPeriod: string; rawDownloads: number; uniqueIPs: number; }) { console.info("posting sonatype metric", { metric }); const labels = { artifact: metric.artifact, + reportPeriod: metric.reportPeriod, }; await postMetric( @@ -195,11 +202,9 @@ async function getArtifactStats( groupId: string, artifactId: string, type: string, - fromDate?: string + reportPeriod?: string ): Promise<{ total: number }> { - const from = fromDate - ? convertDateToLastYearMonth(fromDate) - : getLastMonthDate(); + const from = reportPeriod ?? getLastMonthDate(); console.info( `Fetching ${type} stats for artifact ${artifactId} from ${from}...` ); @@ -275,12 +280,8 @@ function getLastMonthDate() { return `${lastMonthYear}${String(lastMonth).padStart(2, "0")}`; } -// function to convert YYYY-MM-DD to YYYYMM -function convertDateToLastYearMonth(date: string) { - console.info(`Converting date ${date} to last year month`); - const lastMonth = new Date(date); - // reduce 1 month, JS will automatically adjust the year if needed - lastMonth.setMonth(lastMonth.getMonth() - 1); - const [year, month] = lastMonth.toISOString().split("T")[0].split("-"); - return `${year}${month}`; +function getLastMonthPeriod(date: string): string { + const parsedDate = parse(date, "yyyy-MM-dd", new Date()); + const previousMonth = subMonths(parsedDate, 1); + return format(previousMonth, "yyyy-MM"); } diff --git a/metrics-collector/src/utils.ts b/metrics-collector/src/utils.ts index 59f1c15..d8ded79 100644 --- a/metrics-collector/src/utils.ts +++ b/metrics-collector/src/utils.ts @@ -1,5 +1,5 @@ import * as fs from "fs"; -import * as path from "path"; +import { subDays, format } from "date-fns"; // Read JSON data from the file export function readJsonFile(filePath: string): any { @@ -16,9 +16,8 @@ export function writeJsonFile(filePath: string, data: any): void { } export const getYesterdayDate = () => { - const yesterday = new Date(); - yesterday.setDate(yesterday.getDate() - 1); - return yesterday.toISOString().split("T")[0]; + const yesterday = subDays(new Date(), 1); + return format(yesterday, "yyyy-MM-dd"); }; interface FetchWithRetryOptions { @@ -32,7 +31,7 @@ export async function fetchWithRetry( options: RequestInit & FetchWithRetryOptions = {} ): Promise { const { - maxRetries = 3, + maxRetries = 9, retryDelay = 1000, timeout = 10000, ...fetchOptions