Merge pull request #100 from icefoganalytics/issue-70/visualize-tab-l…

…imit-view-not-implemented-as-intended

Visualize Tab: Limit View Not Implemented As Intended

_Design/Entity Relationship Diagrams.wsd

@@ -34,6 +34,28 @@ entity "access_requests" {
   deleted_at : datetime2(0)
 }
 
+entity "dataset_entries" {
+  * id : int <<PK>>
+  --
+  * dataset_id : int <<FK>>
+  * raw_json_data : nvarchar(MAX)
+  * json_data : nvarchar(MAX)
+  created_at : datetime2(0)
+  updated_at : datetime2(0)
+  deleted_at : datetime2(0)
+}
+
+entity "dataset_entry_previews" {
+  * id : int <<PK>>
+  --
+  * dataset_id : int <<FK>>
+  * dataset_entry_id : int <<FK>>
+  * json_data : nvarchar(MAX)
+  created_at : datetime2(0)
+  updated_at : datetime2(0)
+  deleted_at : datetime2(0)
+}
+
 entity "dataset_fields" {
   * id : int <<PK>>
   --
@@ -43,7 +65,7 @@ entity "dataset_fields" {
   * data_type : nvarchar(100)
   description : nvarchar(1000)
   note : nvarchar(MAX)
-  is_excluded_from_search : bit
+  is_excluded_from_preview : bit
   created_at : datetime2(0)
   updated_at : datetime2(0)
   deleted_at : datetime2(0)
@@ -188,18 +210,20 @@ entity "visualization_controls" {
   --
   * dataset_id : int <<FK>>
   is_downloadable_as_csv : bit
-  has_search_row_limits : bit
-  search_row_limit_maximum : int
-  has_search_customizations : bit
-  has_fields_excluded_from_search : bit
+  has_preview_row_limit : bit
+  preview_row_limit : int
+  has_preview : bit
+  has_fields_excluded_from_preview : bit
   created_at : datetime2(0)
   updated_at : datetime2(0)
   deleted_at : datetime2(0)
 }
 
 ' Define relationships
 access_grants }o--|| access_requests : access_grant_id
+dataset_entries::id }o--|| dataset_entry_previews::dataset_entry_id
 datasets }o--|| access_grants : dataset_id
+datasets::id }o--|| dataset_entries::dataset_id
 datasets }o--|| dataset_fields : dataset_id
 datasets }o--|| dataset_integrations : dataset_id
 datasets }o--|| taggings : taggable_id, tagging_type = 'Dataset'

api/src/controllers/dataset-entry-previews-controller.ts

@@ -0,0 +1,36 @@
+import { WhereOptions } from "sequelize"
+import { isEmpty } from "lodash"
+
+import { DatasetEntryPreview } from "@/models"
+import { BaseScopeOptions } from "@/policies/base-policy"
+import { DatasetEntryPreviewsPolicy } from "@/policies"
+import BaseController from "@/controllers/base-controller"
+
+export class DatasetEntryPreviewsController extends BaseController {
+  async index() {
+    const where = this.query.where as WhereOptions<DatasetEntryPreview>
+    const filters = this.query.filters as Record<string, unknown>
+
+    const scopes: BaseScopeOptions[] = []
+    if (!isEmpty(filters)) {
+      Object.entries(filters).forEach(([key, value]) => {
+        scopes.push({ method: [key, value] })
+      })
+    }
+    const scopedDatasetEntryPreview = DatasetEntryPreviewsPolicy.applyScope(
+      scopes,
+      this.currentUser
+    )
+
+    const totalCount = await scopedDatasetEntryPreview.count({ where })
+    const datasetEntryPreviews = await scopedDatasetEntryPreview.findAll({
+      where,
+      limit: this.pagination.limit,
+      offset: this.pagination.offset,
+    })
+
+    return this.response.json({ datasetEntryPreviews, totalCount })
+  }
+}
+
+export default DatasetEntryPreviewsController

api/src/controllers/datasets-controller.ts

@@ -71,9 +71,14 @@ export class DatasetsController extends BaseController {
 
     try {
       const serializedDataset = ShowSerializer.perform(dataset, this.currentUser)
+      // TODO: consider developing a standard pattern for this?
+      const serializedPolicy = {
+        ...policy.toJSON(),
+        showUnlimited: policy.show({ unlimited: true }),
+      }
       return this.response.status(200).json({
         dataset: serializedDataset,
-        policy,
+        policy: serializedPolicy,
       })
     } catch (error) {
       return this.response.status(500).json({ message: `Dataset serialization failed: ${error}` })

api/src/controllers/download/datasets-controller.ts

@@ -56,10 +56,7 @@ export class DatasetsController extends BaseController {
         // for data rendering logic
         "file",
         "integration",
-        {
-          association: "fields",
-          where: { isExcludedFromSearch: false },
-        },
+        "fields",
         // for policy logic
         {
           association: "owner",

api/src/controllers/index.ts

@@ -2,6 +2,7 @@ export { AccessGrantsController } from "./access-grants-controller"
 export { AccessRequestsController } from "./access-requests-controller"
 export { CurrentUserController } from "./current-user-controller"
 export { DatasetEntriesController } from "./dataset-entries-controller"
+export { DatasetEntryPreviewsController } from "./dataset-entry-previews-controller"
 export { DatasetFieldsController } from "./dataset-fields-controller"
 export { DatasetIntegrationsController } from "./dataset-integrations-controller"
 export { DatasetsController } from "./datasets-controller"

api/src/controllers/visualization-controls-controller.ts

@@ -93,7 +93,7 @@ export class VisualizationControlsController extends BaseController {
             "accessRequests",
           ],
         },
-        "searchExcludedDatasetFields",
+        "previewExcludedDatasetFields",
       ],
     })
   }

api/src/db/db-client.ts

@@ -4,8 +4,8 @@ import { createNamespace } from "cls-hooked"
 import { DB_HOST, DB_USER, DB_PASS, DB_NAME, DB_PORT, NODE_ENV } from "@/config"
 import { monkeyPatchSequelizeErrorsForJest } from "@/db/utils/monkey-patch-sequelize-errors-for-jest"
 
-const namespace = createNamespace("sequelize-transaction-context")
-Sequelize.useCLS(namespace)
+export const transactionManager = createNamespace("transaction-manager")
+Sequelize.useCLS(transactionManager)
 
 if (DB_NAME === undefined) throw new Error("database name is unset.")
 if (DB_USER === undefined) throw new Error("database username is unset.")
@@ -26,7 +26,7 @@ export const SEQUELIZE_CONFIG: Options = {
     underscored: true,
     timestamps: true, // This is actually the default, but making it explicit for clarity.
     paranoid: true, // adds deleted_at column
-    whereMergeStrategy: 'and', // where fields will be merged using the and operator (instead of overwriting each other)
+    whereMergeStrategy: "and", // where fields will be merged using the and operator (instead of overwriting each other)
   },
 }
 

api/src/db/migrations/2024.05.16T23.36.54.update-visualization-controls-fields-to-operate-as-preview-modifications.ts

@@ -0,0 +1,94 @@
+import { DataTypes } from "sequelize"
+
+import type { Migration } from "@/db/umzug"
+
+export const up: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.addColumn("visualization_controls", "has_preview", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET has_preview = 0 -- instead of has_search_customizations for security reasons
+  `)
+  await queryInterface.removeColumn("visualization_controls", "has_search_customizations")
+
+  await queryInterface.addColumn("visualization_controls", "has_fields_excluded_from_preview", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: true,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET has_fields_excluded_from_preview = 1 -- instead of has_fields_excluded_from_search for security reasons
+  `)
+  await queryInterface.removeColumn("visualization_controls", "has_fields_excluded_from_search")
+
+  await queryInterface.addColumn("visualization_controls", "has_preview_row_limit", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: true,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET has_preview_row_limit = 1 -- instead of has_search_row_limits for security reasons
+  `)
+  await queryInterface.removeColumn("visualization_controls", "has_search_row_limits")
+
+  await queryInterface.addColumn("visualization_controls", "preview_row_limit", {
+    type: DataTypes.INTEGER,
+    allowNull: false,
+    defaultValue: 10,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET preview_row_limit = ISNULL(search_row_limit_maximum, 10)
+  `)
+  await queryInterface.removeColumn("visualization_controls", "search_row_limit_maximum")
+}
+
+export const down: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.addColumn("visualization_controls", "search_row_limit_maximum", {
+    type: DataTypes.INTEGER,
+    allowNull: true,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET search_row_limit_maximum = preview_row_limit
+  `)
+  await queryInterface.removeColumn("visualization_controls", "preview_row_limit")
+
+  await queryInterface.addColumn("visualization_controls", "has_search_row_limits", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET has_search_row_limits = has_preview_row_limit
+  `)
+  await queryInterface.removeColumn("visualization_controls", "has_preview_row_limit")
+
+  await queryInterface.addColumn("visualization_controls", "has_search_customizations", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET has_search_customizations = has_preview
+  `)
+  await queryInterface.removeColumn("visualization_controls", "has_preview")
+
+  await queryInterface.addColumn("visualization_controls", "has_fields_excluded_from_search", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE visualization_controls
+    SET has_fields_excluded_from_search = has_fields_excluded_from_preview
+  `)
+  await queryInterface.removeColumn("visualization_controls", "has_fields_excluded_from_preview")
+}

api/src/db/migrations/2024.05.17T00.15.32.update-dataset-field-exclusion-flag-to-refer-to-preview-not-search.ts

@@ -0,0 +1,29 @@
+import { DataTypes } from "sequelize"
+
+import type { Migration } from "@/db/umzug"
+
+export const up: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.addColumn("dataset_fields", "is_excluded_from_preview", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: true,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE dataset_fields
+    SET is_excluded_from_preview = 1 -- instead of is_excluded_from_search for security reasons
+  `)
+  await queryInterface.removeColumn("dataset_fields", "is_excluded_from_search")
+}
+
+export const down: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.addColumn("dataset_fields", "is_excluded_from_search", {
+    type: DataTypes.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  })
+  await queryInterface.sequelize.query(/* sql */ `
+    UPDATE dataset_fields
+    SET is_excluded_from_search = 0 -- instead of is_excluded_from_preview as conceptually different
+  `)
+  await queryInterface.removeColumn("dataset_fields", "is_excluded_from_preview")
+}

api/src/db/migrations/2024.05.17T20.38.17.create-dataset-entry-previews-table.ts

@@ -0,0 +1,63 @@
+import { DataTypes, Op } from "sequelize"
+
+import type { Migration } from "@/db/umzug"
+import { MssqlSimpleTypes } from "@/db/utils/mssql-simple-types"
+
+export const up: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.createTable("dataset_entry_previews", {
+    id: {
+      type: DataTypes.INTEGER,
+      primaryKey: true,
+      allowNull: false,
+      autoIncrement: true,
+    },
+    dataset_id: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      references: {
+        model: "datasets",
+        key: "id",
+      },
+    },
+    dataset_entry_id: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      references: {
+        model: "dataset_entries",
+        key: "id",
+      },
+    },
+    json_data: {
+      type: DataTypes.TEXT,
+      allowNull: false,
+    },
+    created_at: {
+      type: MssqlSimpleTypes.DATETIME2(0),
+      allowNull: false,
+      defaultValue: MssqlSimpleTypes.NOW,
+    },
+    updated_at: {
+      type: MssqlSimpleTypes.DATETIME2(0),
+      allowNull: false,
+      defaultValue: MssqlSimpleTypes.NOW,
+    },
+    deleted_at: {
+      type: MssqlSimpleTypes.DATETIME2(0),
+      allowNull: true,
+    },
+  })
+
+  await queryInterface.addIndex("dataset_entry_previews", ["dataset_id"])
+  await queryInterface.addIndex("dataset_entry_previews", ["dataset_entry_id"], {
+    unique: true,
+    where: {
+      deleted_at: {
+        [Op.is]: null,
+      },
+    },
+  })
+}
+
+export const down: Migration = async ({ context: queryInterface }) => {
+  await queryInterface.dropTable("dataset_entry_previews")
+}

api/src/db/utils/mssql-json-object-types.ts

@@ -0,0 +1,11 @@
+/**
+ * See https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16#return-value
+ */
+export enum JsonDataType {
+  NULL = 0,
+  STRING = 1,
+  NUMBER = 2,
+  BOOLEAN = 3,
+  ARRAY = 4,
+  OBJECT = 5,
+}