Merge pull request #57 from icefoganalytics/issue-54/dataset-visualiz…

…e-dataset-entry-search-and-download

Dataset Visualize Dataset Entry Search and Download

api/package.json

@@ -20,6 +20,7 @@
     "@faker-js/faker": "^8.4.0",
     "axios": "^1.6.5",
     "cls-hooked": "^4.2.2",
+    "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
@@ -29,6 +30,7 @@
     "jwks-rsa": "^3.1.0",
     "lodash": "^4.17.21",
     "luxon": "^3.4.4",
+    "papaparse": "^5.4.1",
     "qs": "^6.11.2",
     "sequelize": "^6.35.2",
     "slugify": "^1.6.6",
@@ -37,12 +39,14 @@
   },
   "devDependencies": {
     "@types/cls-hooked": "^4.3.8",
+    "@types/cookie-parser": "^1.4.7",
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",
     "@types/jest": "^29.5.11",
     "@types/jmespath": "^0.15.2",
     "@types/lodash": "^4.14.202",
     "@types/luxon": "^3.4.2",
+    "@types/papaparse": "^5.3.14",
     "@types/qs": "^6.9.11",
     "@types/supertest": "^6.0.2",
     "@typescript-eslint/eslint-plugin": "^6.21.0",

api/src/app.ts

@@ -1,4 +1,5 @@
 import express, { type Request, type Response } from "express"
+import cookieParser from "cookie-parser"
 import cors from "cors"
 import path from "path"
 import helmet from "helmet"
@@ -16,6 +17,7 @@ app.set("query parser", (params: string) => {
     strictNullHandling: true,
   })
 })
+app.use(cookieParser())
 app.use(express.json()) // for parsing application/json
 app.use(express.urlencoded({ extended: true })) // for parsing application/x-www-form-urlencoded
 app.use(

api/src/controllers/base-controller.ts

@@ -6,8 +6,14 @@ export type Actions = "index" | "show" | "new" | "edit" | "create" | "update" |
 
 type ControllerRequest = Request & {
   currentUser: User
+  format?: string
 }
 
+// Keep in sync with web/src/api/base-api.ts
+const MAX_PER_PAGE = 1000
+const MAX_PER_PAGE_EQUIVALENT = -1
+const DEFAULT_PER_PAGE = 10
+
 // See https://guides.rubyonrails.org/routing.html#crud-verbs-and-actions
 export class BaseController {
   protected request: ControllerRequest
@@ -61,23 +67,23 @@ export class BaseController {
     }
   }
 
-  index(): Promise<any> {
+  index(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  create(): Promise<any> {
+  create(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  show(): Promise<any> {
+  show(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  update(): Promise<any> {
+  update(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  destroy(): Promise<any> {
+  destroy(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
@@ -91,6 +97,10 @@ export class BaseController {
     return this.request.currentUser
   }
 
+  get format() {
+    return this.request.format
+  }
+
   get params() {
     return this.request.params
   }
@@ -101,8 +111,8 @@ export class BaseController {
 
   get pagination() {
     const page = parseInt(this.query.page?.toString() || "") || 1
-    const perPage = parseInt(this.query.perPage?.toString() || "") || 10
-    const limit = Math.max(10, Math.min(perPage, 1000)) // restrict max limit to 1000 for safety
+    const perPage = parseInt(this.query.perPage?.toString() || "") || DEFAULT_PER_PAGE
+    const limit = this.determineLimit(perPage)
     const offset = (page - 1) * limit
     return {
       page,
@@ -111,6 +121,15 @@ export class BaseController {
       offset,
     }
   }
+
+  private determineLimit(perPage: number) {
+    if (perPage === MAX_PER_PAGE_EQUIVALENT) {
+      return MAX_PER_PAGE
+    }
+
+
+    return Math.min(perPage, MAX_PER_PAGE)
+  }
 }
 
 export default BaseController

api/src/controllers/dataset-entries-controller.ts

@@ -1,24 +1,58 @@
-import { WhereOptions } from "sequelize"
+import { ModelStatic, WhereOptions } from "sequelize"
+import { isEmpty } from "lodash"
+import { createReadStream } from "fs"
 
 import { DatasetEntry } from "@/models"
 import { DatasetEntriesPolicy } from "@/policies"
+import { CreateCsvService } from "@/services/dataset-entries"
 
 import BaseController from "@/controllers/base-controller"
 
+// TODO: consider moving this to a /datasets/:datasetId/entries route
+// so that we can use the datasetId to scope the entries and reduce query complexity
 export class DatasetEntriesController extends BaseController {
   async index() {
     const where = this.query.where as WhereOptions<DatasetEntry>
+    const searchToken = this.query.searchToken as string
 
     const scopedDatasetEntries = DatasetEntriesPolicy.applyScope(DatasetEntry, this.currentUser)
 
-    const totalCount = await scopedDatasetEntries.count({ where })
-    const datasetEntries = await scopedDatasetEntries.findAll({
-      where,
-      limit: this.pagination.limit,
-      offset: this.pagination.offset,
-    })
+    let filteredDatasetEntries = scopedDatasetEntries
+    if (!isEmpty(searchToken)) {
+      filteredDatasetEntries = scopedDatasetEntries.scope({ method: ["search", searchToken] })
+    }
 
-    return this.response.json({ datasetEntries, totalCount })
+    if (this.format === "csv") {
+      return this.respondWithCsv(filteredDatasetEntries, where)
+    } else {
+      const totalCount = await filteredDatasetEntries.count({ where })
+      const datasetEntries = await filteredDatasetEntries.findAll({
+        where,
+        limit: this.pagination.limit,
+        offset: this.pagination.offset,
+      })
+
+      return this.response.json({ datasetEntries, totalCount })
+    }
+  }
+
+  private async respondWithCsv(
+    datasetEntriesScope: ModelStatic<DatasetEntry>,
+    where: WhereOptions<DatasetEntry>
+  ) {
+    try {
+      const filePath = await CreateCsvService.perform(datasetEntriesScope, where)
+
+      this.response.header("Content-Type", "text/csv")
+      const date = new Date().toISOString().split("T")[0]
+      this.response.attachment(`Export, Dataset Entries, ${date}.csv`)
+
+      const fileStream = createReadStream(filePath)
+      fileStream.pipe(this.response)
+    } catch (error) {
+      console.error("Failed to generate CSV", error)
+      this.response.status(500).send(`Failed to generate CSV: ${error}`)
+    }
   }
 }
 

api/src/controllers/index.ts

@@ -8,6 +8,7 @@ export { DatasetsController } from "./datasets-controller"
 export { DatasetStewardshipsController } from "./dataset-stewardships-controller"
 export { TaggingsController } from "./taggings-controller"
 export { TagsController } from "./tags-controller"
+export { TemporaryAccessCookieController } from "./temporary-access-cookie-controller"
 export { UserGroupsController } from "./user-groups-controller"
 export { UsersController } from "./users-controller"
 export { VisualizationControlsController } from "./visualization-controls-controller"

api/src/controllers/temporary-access-cookie-controller.ts

@@ -0,0 +1,29 @@
+import { TEMPORARY_ACCESS_COOKIE_NAME } from "@/middlewares/temporary-access-cookie-hoist-middleware"
+
+import BaseController from "@/controllers/base-controller"
+
+export const TEMPORARY_ACCESS_COOKIE_EXPIRY = 60 * 1000 // 1 minute in milliseconds
+
+export class TemporaryAccessCookieController extends BaseController {
+  async create() {
+    const authHeader = this.request.headers.authorization
+    if (authHeader && authHeader.startsWith("Bearer ")) {
+      const token = authHeader.substring(7, authHeader.length)
+
+      const secure = process.env.NODE_ENV !== "development"
+      this.response.cookie(TEMPORARY_ACCESS_COOKIE_NAME, token, {
+        httpOnly: true,
+        secure,
+        sameSite: "strict",
+        maxAge: TEMPORARY_ACCESS_COOKIE_EXPIRY,
+      })
+      this.response.end()
+    } else {
+      this.response.status(401).send({
+        message: "Authorization token missing or improperly formatted",
+      })
+    }
+  }
+}
+
+export default TemporaryAccessCookieController

api/src/middlewares/path-format-middleware.ts

@@ -0,0 +1,25 @@
+import { Request, Response, NextFunction } from "express"
+
+export type RequestWithFormat = Request & {
+  format?: string
+}
+
+/**
+ * Adds support for /my/url.xxx on all routes, where xxx is a format
+ *
+ * TODO: also support /my/url?format=xxx style
+ */
+export default function pathFormatMiddleware(req: RequestWithFormat, res: Response, next: NextFunction) {
+  const formatRegex = /(.+)\.(\w+)$/
+  const match = req.path.match(formatRegex)
+
+  if (match) {
+    // Add the format as a parameter to req.params
+    req.format = match[2]
+
+    // Modify the URL path to strip off the format
+    req.url = match[1]
+  }
+
+  next()
+}

api/src/middlewares/temporary-access-cookie-hoist-middleware.ts

@@ -0,0 +1,18 @@
+import { Request, Response, NextFunction } from "express"
+
+export const TEMPORARY_ACCESS_COOKIE_NAME = "temporary_access_token"
+
+export function temporaryAccessCookieHoistMiddleware(
+  req: Request,
+  res: Response,
+  next: NextFunction
+) {
+  const temporaryAccessToken = req.cookies?.[TEMPORARY_ACCESS_COOKIE_NAME]
+  if (temporaryAccessToken && !req.headers.authorization) {
+    req.headers.authorization = `Bearer ${temporaryAccessToken}`
+  }
+
+  next()
+}
+
+export default temporaryAccessCookieHoistMiddleware