New: Ingest downloads for Firefox extension

function-download-data/function.json

@@ -0,0 +1,11 @@
+{
+  "bindings": [
+    {
+      "name": "processDailyDownloads",
+      "type": "timerTrigger",
+      "direction": "in",
+      "schedule": "0 0 11 * * *"
+    }
+  ],
+  "scriptFile": "../dist/src/download-data/index.js"
+}

src/download-data/analytics.ts

@@ -0,0 +1,32 @@
+import * as applicationinsights from 'applicationinsights';
+
+const instrumentationKey = process.env.RESULT_INSTRUMENTATION_KEY!; // eslint-disable-line no-process-env
+
+applicationinsights.setup(instrumentationKey).start();
+
+const appInsightsClient = applicationinsights.defaultClient;
+
+/**
+ * Send an event to Application Insights.
+ * @param name - Event name.
+ * @param properties - Properties to track.
+ */
+export const trackEvent = (name: string, properties: any = {}) => {
+    appInsightsClient.trackEvent({
+        name,
+        properties
+    });
+};
+
+/**
+ * Send the remaining data to Application Insights.
+ */
+export const sendPendingData = (): Promise<void> => {
+    return new Promise((resolve) => {
+        appInsightsClient.flush({
+            callback: () => {
+                return resolve();
+            }
+        });
+    });
+};

src/download-data/api.ts

@@ -0,0 +1,64 @@
+import { get } from 'got';
+import { f12DownloadEvent, AIResponseQuery, Browser, DownloadData, FirefoxDownloads } from './types';
+
+
+const getDateString = (date: Date) => {
+    return date.toISOString().split('T')[0].replace(/-/g, '');
+};
+
+const cleanQuery = (query: string) => {
+    return query.replace(/\r?\n/gm, '') // remove new lines
+        .replace(/\s{2,}/gm, ' '); // remove extra spaces.
+};
+
+/* eslint-disable no-process-env */
+const downloadDataEndpoint = cleanQuery(`https://api.applicationinsights.io/v1/apps/${process.env.APPID}/query?
+query=customEvents
+      | where name == '${f12DownloadEvent}'
+      | sort by todatetime(customDimensions["date"]) desc
+      | project customDimensions
+      | take 1`);
+
+export const getFirefoxDownloadData = async (startDate: Date, endDate: Date) => {
+    const start = getDateString(startDate);
+    const end = getDateString(endDate);
+    const url = `https://addons.mozilla.org/en-US/firefox/addon/webhint/statistics/downloads-day-${start}-${end}.json`;
+    const data: FirefoxDownloads[] = (await get(url, { json: true })).body;
+
+    return data.map((entry) => {
+        return {
+            browser: 'Firefox',
+            count: entry.count,
+            date: new Date(entry.date)
+        } as DownloadData;
+    });
+};
+
+/**
+ * Return the latest download data we have in Application Insights
+ */
+export const getLatestDownloads = async (browser: Browser): Promise<DownloadData | null> => {
+    const aiResponse: AIResponseQuery = (await get(downloadDataEndpoint, {
+        headers: { 'x-api-key': process.env.X_API_KEY }, // eslint-disable-line no-process-env
+        json: true,
+        retry: { retries: 5 }
+    })).body;
+
+    const table = aiResponse.tables[0];
+    const customDimensionsIndex = table.columns.findIndex((column) => {
+        return column.name === 'customDimensions';
+    });
+
+    const row = table.rows[0];
+
+    if (!row) {
+        return null;
+    }
+
+    const downloadData: DownloadData = JSON.parse(row[customDimensionsIndex]!);
+
+    // `downloadData.date` is already normalized to UTC.
+    downloadData.date = new Date(downloadData.date);
+
+    return downloadData;
+};

src/download-data/index.ts

@@ -0,0 +1,38 @@
+import { AzureFunction, Context } from '@azure/functions';
+
+import { getLatestDownloads, getFirefoxDownloadData } from './api';
+import { getISODateString } from './utils';
+import { trackEvent, sendPendingData } from './analytics';
+import { f12DownloadEvent } from './types';
+
+const downloadData: AzureFunction = async function (context: Context, myTimer: any): Promise<void> {
+    const latestDownloads = await getLatestDownloads('Firefox');
+    const end = new Date(getISODateString());
+    let start: Date;
+
+    if (latestDownloads) {
+        start = new Date(latestDownloads.date);
+        start.setUTCDate(start.getUTCDate() + 1);
+    } else {
+        start = new Date(end);
+        start.setUTCDate(start.getUTCDate() - 90);
+    }
+
+    if (end.getTime() - start.getTime() <= 0) {
+        context.log('No data to update.');
+
+        return;
+    }
+
+    const downloadData = await getFirefoxDownloadData(start, end);
+
+    for (const data of downloadData.reverse()) {
+        if (data.date.getTime() >= start.getTime()) {
+            trackEvent(f12DownloadEvent, data);
+        }
+    }
+
+    await sendPendingData();
+};
+
+export default downloadData;

src/download-data/types.ts

@@ -0,0 +1,31 @@
+export const f12DownloadEvent = 'f12-downloads';
+
+export type Browser = 'Chrome' | 'Edge' | 'Firefox';
+
+export type DownloadData = {
+    browser: Browser;
+    count: number;
+    date: Date;
+}
+
+export type FirefoxDownloads = {
+    count: number;
+    date: string;
+};
+
+type AIResponseColumn = {
+    name: string;
+    type: string;
+}
+
+type AIResponseRow = (string | null)[];
+
+type AIResponseTable = {
+    columns: AIResponseColumn[];
+    name: string;
+    rows: AIResponseRow[];
+}
+
+export type AIResponseQuery = {
+    tables: AIResponseTable[];
+}

src/download-data/utils.ts

@@ -0,0 +1,14 @@
+/**
+ * Calculate the ISO string given a date at the 0 hours, 0 minutes and 0 seconds.
+ * @param {number} dateInMS - Date in milliseconds to convert.
+ */
+export const getISODateString = (dateInMS: number = Date.now()) => {
+    const date = new Date(dateInMS);
+
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+
+    return date.toISOString();
+};

tests/download-data/api.ts

@@ -0,0 +1,82 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import anyTest, { TestInterface } from 'ava';
+import * as sinon from 'sinon';
+import * as proxyquire from 'proxyquire';
+
+import { AIResponseQuery, FirefoxDownloads } from '../../src/download-data/types';
+
+const firefoxDownloadsResponse: FirefoxDownloads = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures', 'firefox-downloads.json'), 'utf-8')); // eslint-disable-line no-sync
+const downloadsResponse: AIResponseQuery = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures', 'downloads.json'), 'utf-8')); // eslint-disable-line no-sync
+const downloadsEmptyResponse: AIResponseQuery = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures', 'downloads-empty.json'), 'utf-8')); // eslint-disable-line no-sync
+
+type Request = {
+    get: (url: string) => Promise<any>;
+}
+
+type APIContext = {
+    sandbox: sinon.SinonSandbox;
+    request: Request;
+}
+
+const test = anyTest as TestInterface<APIContext>;
+
+const loadScript = (context: APIContext) => {
+    return proxyquire('../../src/download-data/api', { got: context.request });
+};
+
+test.beforeEach((t) => {
+    t.context.sandbox = sinon.createSandbox();
+
+    t.context.request = {
+        get(url: string) {
+            return null as any;
+        }
+    };
+});
+
+test.afterEach((t) => {
+    t.context.sandbox.restore();
+});
+
+test(`'getFirefoxDownloadData' will request the data in just one query`, async (t) => {
+    const requestStub = sinon.stub(t.context.request, 'get').resolves({ body: firefoxDownloadsResponse });
+
+    const { getFirefoxDownloadData } = loadScript(t.context);
+
+    const start = new Date('2020-02-07');
+    const end = new Date('2020-02-09');
+
+    const result = await getFirefoxDownloadData(start, end);
+
+    t.true(requestStub.calledOnce);
+    t.is(requestStub.firstCall.args[0].split('/').pop(), `downloads-day-20200207-20200209.json`);
+    t.is(result.length, 3);
+    t.is(result[0].browser, 'Firefox');
+    t.is(result[0].count, 10);
+    t.deepEqual(result[0].date, end);
+});
+
+test(`'getLatestDownloads' will return null if there is no data in App Insight`, async (t) => {
+    sinon.stub(t.context.request, 'get').resolves({ body: downloadsEmptyResponse });
+
+    const { getLatestDownloads } = loadScript(t.context);
+
+    const now = new Date();
+
+    const result = await getLatestDownloads(now);
+
+    t.is(result, null);
+});
+
+test(`'getLatestDownloads' will request the data in just one query`, async (t) => {
+    const requestGetStub = sinon.stub(t.context.request, 'get').resolves({ body: downloadsResponse });
+
+    const { getLatestDownloads } = loadScript(t.context);
+
+    const now = new Date();
+
+    await getLatestDownloads(now);
+
+    t.true(requestGetStub.calledOnce);
+});

tests/download-data/fixtures/downloads-empty.json

@@ -0,0 +1,14 @@
+{
+    "tables": [
+        {
+            "name": "PrimaryResult",
+            "columns": [
+                {
+                    "name": "customDimensions",
+                    "type": "dynamic"
+                }
+            ],
+            "rows": []
+        }
+    ]
+}

tests/download-data/fixtures/downloads.json

@@ -0,0 +1,18 @@
+{
+    "tables": [
+        {
+            "name": "PrimaryResult",
+            "columns": [
+                {
+                    "name": "customDimensions",
+                    "type": "dynamic"
+                }
+            ],
+            "rows": [
+                [
+                    "{\"date\":\"2019-10-10T00:00:00.0000000Z\",\"oneDay\":\"5\",\"twoDays\":\"4\"}"
+                ]
+            ]
+        }
+    ]
+}

tests/download-data/fixtures/firefox-downloads.json

@@ -0,0 +1,17 @@
+[
+    {
+        "count": 10,
+        "date": "2020-02-09",
+        "end": "2020-02-09"
+    },
+    {
+        "count": 17,
+        "date": "2020-02-08",
+        "end": "2020-02-08"
+    },
+    {
+        "count": 16,
+        "date": "2020-02-07",
+        "end": "2020-02-07"
+    }
+]