minor updates

.github/workflows/cicd.yaml

@@ -25,6 +25,10 @@ jobs:
           python-version: 3.11
       - name: install requirements
         run: just setup
+      - name: ingest docs data
+        run: just ingest-docs
+        env:
+          AZURE_STORAGE_SECRET: ${{ secrets.AZURE_STORAGE_SECRET }}
       - name: ingest pypi data
         run: just ingest-pypi
         env:

config.toml

@@ -29,11 +29,15 @@ endpoints = [
     "issues",
     "contributors",
 ]
+[goat]
+account_name = "notonedrive"
+directory = "goat-export"
+blob_name = "goatcounter-export-ibis.csv.gz"
 [motherduck]
 database="metrics"
 exclude_tables = ["penguins"]
 #exclude_tables = ["penguins", "stars", "issues", "pulls", "forks", "watchers", "commits", "downloads"]
-ci_tables = ["stars", "issues", "pulls", "forks", "watchers", "commits"]
+#ci_tables=["jobs", "analysis", "workflows"]
 [azure]
 account_name = "notonedrive"
 directory = "metrics-backup"

justfile

@@ -35,8 +35,8 @@ ingest-ci:
     @python src/ingest_ci.py
 
 # ingest docs data
-#ingest-docs:
-#    @python src/ingest_docs.py
+ingest-docs:
+    @python src/ingest_docs.py
 
 # load a backup
 load-backup:
@@ -77,19 +77,26 @@ goat:
         --header "Authorization: Bearer $GOAT_TOKEN" \
         "$api/export"
 
-ingest-docs:
-    #!/bin/bash +x
-    api="https://ibis.goatcounter.com/api/v0/export/791705491/download"
-    curl -v -X POST \
-        --header 'Content-Type: application/json' \
-        --header "Authorization: Bearer $GOAT_TOKEN" \
-        "$api"
-
+#ingest-docs:
+#    #!/bin/bash +x
+#    api="https://ibis.goatcounter.com/api/v0/export/791705491/download"
+#    curl -v -X POST \
+#        --header 'Content-Type: application/json' \
+#        --header "Authorization: Bearer $GOAT_TOKEN" \
+#        "$api"
+#
 #load-backup:
 #    az storage azcopy blob download \
 #        --account-name notonedrive \
-#        --account-key $AZURE_STORAGE_KEY \
+#        --account-key $AZURE_STORAGE_SECRET \
 #        -c metrics-backup \
 #        -s cache.ddb \
 #        -d cache.ddb
 
+upload-goat:
+    az storage azcopy blob upload \
+        --account-name notonedrive \
+        --account-key $AZURE_STORAGE_SECRET \
+        -c goat-export \
+        -s data/docs/*.csv.gz \
+        -d goatcounter-export-ibis.csv.gz

src/ingest_docs.py

@@ -1,71 +1,38 @@
 # imports
 import os
-import time
-import json
-import requests
+import toml
 
-import pandas as pd
 import logging as log
-import gzip
 
-from pathlib import Path
 from dotenv import load_dotenv
 
+from azure.storage.blob import BlobServiceClient as BSC
+
 # configure logger
 log.basicConfig(level=log.INFO)
 
-# load .env file
-load_dotenv()
-
-# setup for goatcounter
-api_token = os.getenv("GOAT_TOKEN")
-api_url = "https://ibis.goatcounter.com/api/v0"
-headers = {
-    "Authorization": f"Bearer {api_token}",
-    "Content-Type": "application/json",
-}
-
-
-def get_last_cursor():
-    path = Path("data/docs")
-    files = path.glob("goatcounter-export-*.csv.gz")
-    if not files:
-        return None
-    last_file = max(files, key=lambda p: p.stat().st_mtime)
-    # extract the cursor from the filename
-    return str(last_file.stem.split("-")[-2])
+# config.toml
+config = toml.load("config.toml")["goat"]
+transform_config = toml.load("config.toml")["transform"]
 
+account_name = config.get("account_name")
+directory = config.get("directory")
+blob_name = config.get("blob_name")
 
-def start_export(cursor=None):
-    data = {}
-    if cursor is not None:
-        data = {"start_from_hit_id": cursor}
-    r = requests.post(f"{api_url}/export", headers=headers, data=data)
-    breakpoint()
-    if r.status_code >= 200 and r.status_code < 300:
-        return r.json()["id"]
-    else:
-        log.error(f"Error starting export: {r.status_code} {r.text}")
-        return None
-
-
-def check_export(id):
-    r = requests.get(f"{api_url}/export/{id}", headers=headers)
-    return r.json()["finished_at"]
-
-
-def download_export(id):
-    r = requests.get(f"{api_url}/export/{id}/download", headers=headers)
-    filename = f"data/docs/goatcounter-export-{id}.csv.gz"
-    with open(filename, "wb") as f:
-        f.write(r.content)
+# load env vars
+load_dotenv()
+storage_token = os.getenv("AZURE_STORAGE_SECRET")
 
+# cloud config
+account_url = f"https://{account_name}.blob.core.windows.net/"
+credential = {"account_name": account_name, "account_key": storage_token}
 
-cursor = get_last_cursor()
-export_id = start_export(cursor)
+bsc = BSC(account_url=account_url, credential=credential)
+bs = bsc.get_blob_client(directory, "goatcounter-export-ibis.csv.gz")
 
-while True:
-    time.sleep(1)
-    if check_export(export_id):
-        download_export(export_id)
-        break
+# download blob
+log.info("Downloading blob...")
+os.makedirs("data/docs", exist_ok=True)
+with open(f"data/docs/{blob_name}", "wb") as f:
+    data = bs.download_blob()
+    data.readinto(f)

transform.py

@@ -109,12 +109,11 @@
 log.info("processing downloads...")
 con.create_table("downloads", downloads, overwrite=True)
 
-if not os.getenv("CI"):
-    docs = clean_data(con.read_csv("data/docs/*.csv*"))
-    docs = docs.relabel({"2_path": "path", "date": "timestamp"})
+docs = clean_data(con.read_csv("data/docs/*.csv*"))
+docs = docs.relabel({"2_path": "path", "date": "timestamp"})
 
-    log.info("processing docs...")
-    con.create_table("docs", docs, overwrite=True)
+log.info("processing docs...")
+con.create_table("docs", docs, overwrite=True)
 
 if config["ci_enabled"] and not os.getenv("CI"):
     ci_con = ibis.connect("duckdb://data/ci/ibis/raw.ddb")