Split reports by core project

.github/workflows/compile-data.yaml → .github/workflows/pipeline.yaml

@@ -1,10 +1,10 @@
-name: Compile data
+name: Run pipeline
 
 on:
   workflow_dispatch:
 
 jobs:
-  compile:
+  pipeline:
     runs-on: ubuntu-latest
 
     steps:
@@ -18,23 +18,16 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: "22"
-          cache: "yarn"
+          cache: yarn
+          cache-dependency-path: |
+            app/yarn.lock
+            data/yarn.lock
 
-      - name: Install /data packages
-        run: yarn install
-        working-directory: ./data
+      - name: Install packages
+        run: ./run.sh --install
 
-      - name: Install /app packages
-        run: yarn install
-        working-directory: ./app
-
-      - name: Install Playwright
-        run: yarn playwright
-        working-directory: ./data
-
-      - name: Compile report
-        run: yarn compile
-        working-directory: ./data
+      - name: Run pipeline
+        run: ./run.sh
 
       - name: Commit results
         uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/test.yaml

@@ -0,0 +1,30 @@
+name: Run tests
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - if: runner.debug == '1'
+        uses: mxschmitt/action-tmate@v3
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: yarn
+          cache-dependency-path: |
+            app/yarn.lock
+            data/yarn.lock
+
+      - name: Install packages
+        run: ./run.sh --install
+
+      - name: Run tests
+        run: ./run.sh --test

.gitignore

@@ -1 +1 @@
-.DS_Store
+.DS_Store

README.md

@@ -1,30 +1,57 @@
 # Requirements
 
-- Node v22+
-- Yarn v1 ("classic")
+- [Node](https://nodejs.org/) v22+
+- [Yarn](https://classic.yarnpkg.com/) v1 ("classic")
+
+# Terminology
+
+- `webapp` - Live, interactive website that provides access to all reports.
+- `report` - Info about a particular project/publication/whatever, _either_ as a webapp _page_ or a printed _PDF_.
 
 # Repo Content
 
 - `/app` - Live dashboard webapp made with Vue.
   Also used for rendering PDF reports.
-- `/data` - All other functionality involving ingesting, transforming, and storing data for the webapp and reports.
+- `/data` - All other functionality involving ingesting, transforming, and storing data.
   - `/api` - Types and functions for getting raw, as-is data from third-party APIs.
   - `/database` - Types, schemas, and functions for getting/setting data in the database.
-    And the raw SQLite database itself.
+    And the raw database itself†.
   - `/ingest` - Functions for scraping or requesting data from webpages or APIs and transforming it into a common db format.
+  - `/report` - Functions specific to making reports, e.g. printing, collating.
   - `/util` - Small-scope general purpose functions.
 
-# Process
+† For now, the database is a **public** SQLite database stored in this repo.
+The method of storing secret information is to-be-determined.
 
-The general process of compiling the data is as follows:
+# Pipeline
 
-1. Gather funding opportunity numbers (and related metadata) from NIH public documents.
-1. Transform and pare down that opportunity info into a common format.
-1. Add that transformed info to the database†.
-1. Get the next level of info dependent on that info (e.g. opportunity number -> grant numbers, grant numbers -> publication numbers) and repeat the previous steps.
-1. Pick out and collate high-level data from database.
-1. Run webapp to display and format data and print to PDF reports.
-1. Live webapp and PDF reports deploy publicly automatically.
+The automated steps in this repo are generally as follows:
 
-† For now, the database is a **public** SQLite database stored in this repo.
-The method of storing secret information is to-be-determined.
+1. "Ingest"
+   1. Gather funding opportunity numbers (and related metadata) from NIH public documents.
+   1. Transform and pare down opportunity info into a common format.
+   1. Add transformed info to the database.
+   1. Repeat previous steps in order of dependency (e.g. opportunity number -> grant numbers) to get all needed info.
+1. "Collate"
+   1. Pick out high-level info from database in format best for reports.
+1. "Print"
+   1. Run webapp to render each _page_ report to _PDF_ report.
+1. Live webapp and PDFs deploy publicly automatically.
+
+## Commands
+
+Use `run.sh` to conveniently run multiple commands across subdirectories.
+Examples:
+
+```shell
+# install all dependencies
+./run.sh --install-all
+
+# run pipeline
+./run.sh
+
+# run tests
+./run.sh --test
+```
+
+See script source for all available flags.

app/.env

@@ -1,2 +1,2 @@
-VITE_TITLE="CFDE Evaluation Core Dashboard"
+VITE_TITLE="CFDE Evaluation Core"
 VITE_DESCRIPTION="Evaluation Core dashboard to support the NIH Common Fund Data Ecosystem"

app/README.md

@@ -9,3 +9,4 @@
 | `yarn test:lint`  | Check linting and formatting |
 | `yarn test:types` | Check types                  |
 | `yarn test`       | Run all tests above          |
+| `yarn clean`      | Hard uninstall packages      |

app/index.html

@@ -4,6 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>%VITE_TITLE%</title>
+    <link rel="icon" type="image/png" sizes="320z320" href="icon.png" />
     <link rel="preconnect" href="https://fonts.googleapis.com" />
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
     <link

app/lint.sh

@@ -6,5 +6,5 @@ if [ "$FIX" ]; then
   prettierArgs="--write"
 fi
 
-bun eslint . --ext .ts,.vue $eslintArgs
-bun prettier **/*.{ts,vue,css,html,md} $prettierArgs --no-error-on-unmatched-pattern
+yarn eslint . --ext .ts,.vue $eslintArgs
+yarn prettier **/*.{ts,vue,css,html,md} $prettierArgs --no-error-on-unmatched-pattern

app/package.json

@@ -8,19 +8,22 @@
     "test:types": "tsc --noemit",
     "test": "yarn test:lint && yarn test:types",
     "preview": "vite preview",
-    "clean": "rm -rf node_modules yarn.lock && yarn cache clean && yarn install"
+    "clean": "rm -rf node_modules yarn.lock && yarn cache clean"
   },
   "dependencies": {
+    "@tanstack/vue-table": "^8.17.3",
+    "@vueuse/core": "^10.11.0",
     "echarts": "^5.5.0",
     "lodash": "^4.17.21",
-    "vue": "^3.4.27",
-    "vue-echarts": "^6.7.3"
+    "vue": "^3.4.29",
+    "vue-echarts": "^6.7.3",
+    "vue-router": "4.3.0"
   },
   "devDependencies": {
     "@ianvs/prettier-plugin-sort-imports": "^4.2.1",
     "@types/echarts": "^4.9.22",
     "@types/lodash": "^4.17.5",
-    "@types/node": "^20.14.2",
+    "@types/node": "^20.14.7",
     "@vitejs/plugin-vue": "^5.0.5",
     "@vue/eslint-config-prettier": "^9.0.0",
     "@vue/eslint-config-typescript": "^13.0.0",
@@ -32,9 +35,9 @@
     "prettier": "^3.3.2",
     "prettier-plugin-css-order": "^2.1.2",
     "prettier-plugin-jsdoc": "^1.3.0",
-    "typescript": "^5.4.0",
-    "vite": "^5.2.13",
+    "typescript": "^5.5.2",
+    "vite": "^5.3.1",
     "vite-svg-loader": "^5.1.0",
     "vue-tsc": "^2.0.21"
   }
-}
+}

app/public/404.html

@@ -0,0 +1,11 @@
+<html>
+  <body>
+    <script type="text/javascript">
+      /** redirect to root and pass page to router */
+      console.debug("Redirecting from:", window.location);
+      const { origin, pathname, search, hash } = window.location;
+      window.sessionStorage.redirect = pathname + search + hash;
+      window.location = "/";
+    </script>
+  </body>
+</html>

app/src/App.vue

@@ -1,34 +1,12 @@
 <template>
   <TheHeader />
-  <main class="charts">
-    <AppChart title="Opportunity Prefixes" :data="prefixes" />
-    <AppChart title="Opportunity Activity Codes" :data="activityCodes" />
+  <main>
+    <RouterView />
   </main>
   <TheFooter />
 </template>
 
 <script setup lang="ts">
-import AppChart from "@/components/AppChart.vue";
 import TheFooter from "@/components/TheFooter.vue";
 import TheHeader from "@/components/TheHeader.vue";
-import data from "./data.json";
-
-const prefixes = Object.fromEntries(
-  data.opportunities.prefixes.map(({ prefix, count }) => [prefix, count]),
-);
-const activityCodes = Object.fromEntries(
-  data.opportunities.activityCodes.map(({ activity_code, count }) => [
-    activity_code,
-    count,
-  ]),
-);
 </script>
-
-<style scoped>
-.charts {
-  display: grid;
-  grid-template-columns: repeat(auto-fit, minmax(min(500px, 100%), 1fr));
-  grid-auto-rows: minmax(200px, calc(11in * 0.35));
-  gap: 40px;
-}
-</style>

app/src/assets/README.md

@@ -0,0 +1,3 @@
+# Icons provided by...
+
+- Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2024 Fonticons, Inc.

app/src/components/AppButton.vue

@@ -0,0 +1,66 @@
+<template>
+  <component :is="component" class="button" :to="to" @click="click">
+    <slot />
+  </component>
+</template>
+
+<script setup lang="ts">
+import { computed } from "vue";
+import AppLink from "@/components/AppLink.vue";
+
+type Props = {
+  /** text to show */
+  text?: string;
+  /** icon to show */
+  icon?: string;
+  /** location to link to */
+  to?: string;
+  /** on click action */
+  click?: () => unknown;
+};
+
+const props = withDefaults(defineProps<Props>(), {
+  text: "",
+  icon: "",
+  to: "",
+  click: undefined,
+  download: undefined,
+});
+
+/** type of component to render */
+const component = computed(() => (props.to ? AppLink : "button"));
+
+type Slots = {
+  default: () => unknown;
+};
+
+defineSlots<Slots>();
+</script>
+
+<style scoped>
+.button {
+  display: inline-flex;
+  appearance: none;
+  align-items: center;
+  justify-content: center;
+  padding: 8px 15px;
+  gap: 10px;
+  border-radius: var(--rounded);
+  background: var(--theme);
+  color: white;
+  text-decoration: none;
+  transition: var(--fast);
+  transition-property: background;
+}
+
+.button:hover {
+  background: var(--light-gray);
+  color: var(--theme);
+}
+
+@media print {
+  .button {
+    display: none;
+  }
+}
+</style>