Merge pull request #113 from icefoganalytics/issue-42/build-dataset-s…

…earch-feature

Build Dataset Search Feature

_Design/Entity Relationship Diagrams.wsd

@@ -177,12 +177,13 @@ entity "user_groups" {
   * id : int <<PK>>
   --
   * parent_id : int <<FK>>
-  name: nvarchar(255)
-  type: nvarchar(255)
-  order: number
+  * type: nvarchar(255)
+  * name: nvarchar(255)
+  * order: number
+  * acronym: nvarchar(10)
   last_division_directory_sync_at: datetime2(0)
-  created_at : datetime2(0)
-  updated_at : datetime2(0)
+  * created_at : datetime2(0)
+  * updated_at : datetime2(0)
   deleted_at : datetime2(0)
 }
 

api/src/db/migrations/2024.06.10T17.08.32.add-acronym-to-user-groups.ts

@@ -0,0 +1,14 @@
+import { DataTypes } from "sequelize"
+
+import type { Migration } from "@/db/umzug"
+
+export const up: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.addColumn("user_groups", "acronym", {
+    type: DataTypes.STRING(10),
+    allowNull: true,
+  })
+}
+
+export const down: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.removeColumn("user_groups", "acronym")
+}

api/src/db/migrations/2024.06.10T18.01.34.backfill-user-groups-acronyms.ts

@@ -0,0 +1,26 @@
+import { DataTypes } from "sequelize"
+
+import type { Migration } from "@/db/umzug"
+import { UserGroup } from "@/models"
+import acronymize from "@/utils/acronymize"
+
+export const up: Migration = async ({ context: queryInterface }) => {
+  await UserGroup.findEach(async (userGroup) => {
+    const acronym = acronymize(userGroup.name)
+    await userGroup.update({
+      acronym,
+    })
+  })
+
+  await queryInterface.changeColumn("user_groups", "acronym", {
+    type: DataTypes.STRING(10),
+    allowNull: false,
+  })
+}
+
+export const down: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.changeColumn("user_groups", "acronym", {
+    type: DataTypes.STRING(10),
+    allowNull: true,
+  })
+}

api/src/models/dataset.ts

@@ -1,4 +1,4 @@
-import { isNil } from "lodash"
+import { isEmpty, isNil } from "lodash"
 import {
   Association,
   BelongsToCreateAssociationMixin,
@@ -41,6 +41,7 @@ import {
   datasetHasApprovedAccessRequestFor,
   datasetIsAccessibleViaOpenAccessGrantBy,
   datasetsAccessibleViaAccessGrantsBy,
+  datasetsSearch,
   mostPermissiveAccessGrantFor,
 } from "@/models/datasets"
 import VisualizationControl from "@/models/visualization-control"
@@ -353,6 +354,23 @@ Dataset.init(
       },
     ],
     scopes: {
+      search(searchToken: string) {
+        if (isEmpty(searchToken)) {
+          return {}
+        }
+
+        return {
+          where: {
+            id: {
+              [Op.in]: datasetsSearch(),
+            },
+          },
+          replacements: {
+            searchTokenWildcard: `%${searchToken}%`,
+            searchToken,
+          },
+        }
+      },
       accessibleViaAccessGrantsBy(user: User) {
         return {
           where: {

api/src/models/datasets/datasets-search.ts

@@ -0,0 +1,78 @@
+import { literal } from "sequelize"
+import { Literal } from "sequelize/types/utils"
+
+import { compactSql } from "@/utils/compact-sql"
+
+/**
+ * Requires replacements to be passed in to query.
+ * e.g. { replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }
+ */
+export function datasetsSearch(): Literal {
+  const matchingEntries = compactSql(/*sql*/ `
+    (
+      SELECT
+        DISTINCT datasets.id
+      FROM
+        datasets
+      WHERE
+        datasets.deleted_at IS NULL
+        AND (
+          LOWER(datasets.name) LIKE LOWER(:searchTokenWildcard)
+          OR LOWER(datasets.description) LIKE LOWER(:searchTokenWildcard)
+          OR EXISTS (
+            SELECT
+              1
+            FROM
+              taggings
+              INNER JOIN tags ON taggings.tag_id = tags.id
+              AND datasets.id = taggings.taggable_id
+              AND taggings.taggable_type = 'Dataset'
+              AND tags.deleted_at IS NULL
+              AND taggings.deleted_at IS NULL
+              AND LOWER(tags.name) LIKE LOWER(:searchTokenWildcard)
+          )
+          OR EXISTS (
+            SELECT
+              1
+            FROM
+              user_groups
+              INNER JOIN dataset_stewardships ON (
+                (
+                  dataset_stewardships.department_id = user_groups.id
+                  AND user_groups.type = 'department'
+                )
+                OR (
+                  dataset_stewardships.department_id IS NOT NULL
+                  AND dataset_stewardships.division_id = user_groups.id
+                  AND user_groups.type = 'division'
+                )
+                OR (
+                  dataset_stewardships.department_id IS NOT NULL
+                  AND dataset_stewardships.division_id IS NOT NULL
+                  AND dataset_stewardships.branch_id = user_groups.id
+                  AND user_groups.type = 'branch'
+                )
+                OR (
+                  dataset_stewardships.department_id IS NOT NULL
+                  AND dataset_stewardships.division_id IS NOT NULL
+                  AND dataset_stewardships.branch_id IS NOT NULL
+                  AND dataset_stewardships.unit_id = user_groups.id
+                  AND user_groups.type = 'unit'
+                )
+              )
+              AND datasets.id = dataset_stewardships.dataset_id
+              AND dataset_stewardships.deleted_at IS NULL
+              AND user_groups.deleted_at IS NULL
+              AND (
+                LOWER(user_groups.name) LIKE LOWER(:searchTokenWildcard)
+                OR user_groups.acronym = :searchToken
+              )
+          )
+        )
+      )
+  `)
+
+  return literal(matchingEntries)
+}
+
+export default datasetsSearch

api/src/models/datasets/index.ts

@@ -2,6 +2,7 @@ export { datasetHasApprovedAccessRequestFor } from "./dataset-has-approved-acces
 export { datasetIsAccessibleViaOpenAccessGrantBy } from "./dataset-is-accessible-via-open-access-grant-by"
 export { datasetsAccessibleViaAccessGrantsBy } from "./datasets-accessible-via-access-grants-by"
 export { datasetsAccessibleViaOwner } from "./datasets-accessible-via-owner"
+export { datasetsSearch } from "./datasets-search"
 export { datasetsWithApprovedAccessRequestsFor } from "./datasets-with-approved-access-requests-for"
 export { datasetsWithFieldExclusionsDisabled } from "./datasets-with-field-exclusions-disabled"
 export { datasetsWithPreviewDisabled } from "./datasets-with-preview-disabled"

api/src/models/tagging.ts

@@ -27,6 +27,8 @@ export enum TaggableTypes {
 }
 
 export class Tagging extends Model<InferAttributes<Tagging>, InferCreationAttributes<Tagging>> {
+  static readonly TaggableTypes = TaggableTypes
+
   declare id: CreationOptional<number>
   declare tagId: ForeignKey<Tag["id"]>
   declare taggableId: number

api/src/models/user-groups.ts

@@ -43,8 +43,9 @@ export class UserGroup extends BaseModel<
 
   declare id: CreationOptional<number>
   declare parentId: ForeignKey<UserGroup["id"]> | null
-  declare name: string
   declare type: string
+  declare name: string
+  declare acronym: string
   declare order: number
   declare lastDivisionDirectorySyncAt: Date | null
   declare createdAt: CreationOptional<Date>
@@ -275,6 +276,10 @@ UserGroup.init(
       type: DataTypes.STRING(255),
       allowNull: false,
     },
+    acronym: {
+      type: DataTypes.STRING(10),
+      allowNull: false,
+    },
     order: {
       type: DataTypes.INTEGER,
       allowNull: false,

api/src/services/user-groups/sync-service.ts

@@ -1,8 +1,10 @@
-import { UserGroup } from "@/models"
+import { isEmpty } from "lodash"
 
-import BaseService from "@/services/base-service"
+import acronymize from "@/utils/acronymize"
 import { yukonGovernmentIntegration } from "@/integrations"
+import { UserGroup } from "@/models"
 import { UserGroupTypes } from "@/models/user-groups"
+import BaseService from "@/services/base-service"
 
 export const DEFAULT_ORDER = -1
 
@@ -16,13 +18,18 @@ export class SyncService extends BaseService {
         const isDivision = branch === null && unit === null
         const isBranch = unit === null
 
+        const departmentName = this.cleanName(department)
+        if (isEmpty(departmentName)) continue
+
+        const departmentAcronym = acronymize(departmentName)
         const [userGroup1] = await UserGroup.findOrCreate({
           where: {
-            name: this.cleanName(department),
+            name: departmentName,
             type: UserGroupTypes.DEPARTMENT,
           },
           defaults: {
-            name: this.cleanName(department),
+            name: departmentName,
+            acronym: departmentAcronym,
             type: UserGroupTypes.DEPARTMENT,
             order: DEFAULT_ORDER,
           },
@@ -33,15 +40,20 @@ export class SyncService extends BaseService {
         }
 
         if (division !== null) {
+          const divisionName = this.cleanName(division)
+          if (isEmpty(divisionName)) continue
+
+          const divisionAcronym = acronymize(divisionName)
           const [userGroup2] = await UserGroup.findOrCreate({
             where: {
               parentId: userGroup1.id,
-              name: this.cleanName(division),
+              name: divisionName,
               type: UserGroupTypes.DIVISION,
             },
             defaults: {
               parentId: userGroup1.id,
-              name: this.cleanName(division),
+              name: divisionName,
+              acronym: divisionAcronym,
               type: UserGroupTypes.DIVISION,
               order: DEFAULT_ORDER,
             },
@@ -52,15 +64,20 @@ export class SyncService extends BaseService {
           }
 
           if (branch !== null) {
+            const branchName = this.cleanName(branch)
+            if (isEmpty(branchName)) continue
+
+            const branchAcronym = acronymize(branchName)
             const [userGroup3] = await UserGroup.findOrCreate({
               where: {
                 parentId: userGroup2.id,
-                name: this.cleanName(branch),
+                name: branchName,
                 type: UserGroupTypes.BRANCH,
               },
               defaults: {
                 parentId: userGroup2.id,
-                name: this.cleanName(branch),
+                name: branchName,
+                acronym: branchAcronym,
                 type: UserGroupTypes.BRANCH,
                 order: DEFAULT_ORDER,
               },
@@ -71,15 +88,20 @@ export class SyncService extends BaseService {
             }
 
             if (unit !== null) {
+              const unitName = this.cleanName(unit)
+              if (isEmpty(unitName)) continue
+
+              const unitAcronym = acronymize(unitName)
               await UserGroup.findOrCreate({
                 where: {
                   parentId: userGroup3.id,
-                  name: this.cleanName(unit),
+                  name: unitName,
                   type: UserGroupTypes.UNIT,
                 },
                 defaults: {
                   parentId: userGroup3.id,
-                  name: this.cleanName(unit),
+                  name: unitName,
+                  acronym: unitAcronym,
                   type: UserGroupTypes.UNIT,
                   order,
                 },

api/tests/factories/dataset-integration-factory.ts

@@ -16,7 +16,7 @@ function assertParamsHasDatasetId(
 
 export const datasetIntegrationFactory = BaseFactory.define<DatasetIntegration>(
   ({ sequence, params, onCreate }) => {
-    onCreate((datasetField) => datasetField.save())
+    onCreate((datasetIntegration) => datasetIntegration.save())
 
     assertParamsHasDatasetId(params)
 

api/tests/factories/dataset-stewardship-factory.ts

@@ -0,0 +1,57 @@
+import { DeepPartial } from "fishery"
+
+import { DatasetStewardship } from "@/models"
+import BaseFactory from "@/factories/base-factory"
+
+export const datasetStewardshipFactory = BaseFactory.define<DatasetStewardship>(
+  ({ sequence, params, onCreate }) => {
+    onCreate((datasetStewardship) => datasetStewardship.save())
+
+    assertParamsHasDatasetId(params)
+    assertParamsHasOnwerId(params)
+    assertParamsHasSupportId(params)
+    assertParamsHasDepartmentId(params)
+
+    return DatasetStewardship.build({
+      id: sequence,
+      datasetId: params.datasetId, // does not unbrand and cast datasetId to number
+      ownerId: params.ownerId,
+      supportId: params.supportId,
+      departmentId: params.departmentId,
+    })
+  }
+)
+
+export default datasetStewardshipFactory
+
+function assertParamsHasDatasetId(
+  params: DeepPartial<DatasetStewardship>
+): asserts params is DeepPartial<DatasetStewardship> & { datasetId: number } {
+  if (typeof params.datasetId !== "number") {
+    throw new Error("datasetId is must be a number")
+  }
+}
+
+function assertParamsHasOnwerId(
+  params: DeepPartial<DatasetStewardship>
+): asserts params is DeepPartial<DatasetStewardship> & { ownerId: number } {
+  if (typeof params.ownerId !== "number") {
+    throw new Error("ownerId is must be a number")
+  }
+}
+
+function assertParamsHasSupportId(
+  params: DeepPartial<DatasetStewardship>
+): asserts params is DeepPartial<DatasetStewardship> & { supportId: number } {
+  if (typeof params.supportId !== "number") {
+    throw new Error("supportId is must be a number")
+  }
+}
+
+function assertParamsHasDepartmentId(
+  params: DeepPartial<DatasetStewardship>
+): asserts params is DeepPartial<DatasetStewardship> & { departmentId: number } {
+  if (typeof params.departmentId !== "number") {
+    throw new Error("departmentId is must be a number")
+  }
+}

api/tests/factories/helpers.ts

@@ -1,8 +1,8 @@
 import { faker } from "@faker-js/faker"
 
 export function anytime() {
-  const hours = faker.number.int({ min: 0, max: 23 }).toString().padStart(2, '0')
-  const minutes = faker.number.int({ min: 0, max: 59 }).toString().padStart(2, '0')
+  const hours = faker.number.int({ min: 0, max: 23 }).toString().padStart(2, "0")
+  const minutes = faker.number.int({ min: 0, max: 59 }).toString().padStart(2, "0")
   const seconds = "00" // currently we aren't tracking time to the second
 
   return `${hours}:${minutes}:${seconds}`