diff --git a/_Design/Entity Relationship Diagrams.png b/_Design/Entity Relationship Diagrams.png index cacd0cb4..b58343e0 100644 Binary files a/_Design/Entity Relationship Diagrams.png and b/_Design/Entity Relationship Diagrams.png differ diff --git a/_Design/Entity Relationship Diagrams.wsd b/_Design/Entity Relationship Diagrams.wsd index 8753a894..6ef13dee 100644 --- a/_Design/Entity Relationship Diagrams.wsd +++ b/_Design/Entity Relationship Diagrams.wsd @@ -34,6 +34,28 @@ entity "access_requests" { deleted_at : datetime2(0) } +entity "dataset_entries" { + * id : int <> + -- + * dataset_id : int <> + * raw_json_data : nvarchar(MAX) + * json_data : nvarchar(MAX) + created_at : datetime2(0) + updated_at : datetime2(0) + deleted_at : datetime2(0) +} + +entity "dataset_entry_previews" { + * id : int <> + -- + * dataset_id : int <> + * dataset_entry_id : int <> + * json_data : nvarchar(MAX) + created_at : datetime2(0) + updated_at : datetime2(0) + deleted_at : datetime2(0) +} + entity "dataset_fields" { * id : int <> -- @@ -43,7 +65,7 @@ entity "dataset_fields" { * data_type : nvarchar(100) description : nvarchar(1000) note : nvarchar(MAX) - is_excluded_from_search : bit + is_excluded_from_preview : bit created_at : datetime2(0) updated_at : datetime2(0) deleted_at : datetime2(0) @@ -188,10 +210,10 @@ entity "visualization_controls" { -- * dataset_id : int <> is_downloadable_as_csv : bit - has_search_row_limits : bit - search_row_limit_maximum : int - has_search_customizations : bit - has_fields_excluded_from_search : bit + has_preview_row_limit : bit + preview_row_limit : int + has_preview : bit + has_fields_excluded_from_preview : bit created_at : datetime2(0) updated_at : datetime2(0) deleted_at : datetime2(0) @@ -199,7 +221,9 @@ entity "visualization_controls" { ' Define relationships access_grants }o--|| access_requests : access_grant_id +dataset_entries::id }o--|| dataset_entry_previews::dataset_entry_id datasets }o--|| access_grants : dataset_id +datasets::id }o--|| dataset_entries::dataset_id datasets }o--|| dataset_fields : dataset_id datasets }o--|| dataset_integrations : dataset_id datasets }o--|| taggings : taggable_id, tagging_type = 'Dataset' diff --git a/api/src/controllers/dataset-entry-previews-controller.ts b/api/src/controllers/dataset-entry-previews-controller.ts new file mode 100644 index 00000000..1100fae0 --- /dev/null +++ b/api/src/controllers/dataset-entry-previews-controller.ts @@ -0,0 +1,36 @@ +import { WhereOptions } from "sequelize" +import { isEmpty } from "lodash" + +import { DatasetEntryPreview } from "@/models" +import { BaseScopeOptions } from "@/policies/base-policy" +import { DatasetEntryPreviewsPolicy } from "@/policies" +import BaseController from "@/controllers/base-controller" + +export class DatasetEntryPreviewsController extends BaseController { + async index() { + const where = this.query.where as WhereOptions + const filters = this.query.filters as Record + + const scopes: BaseScopeOptions[] = [] + if (!isEmpty(filters)) { + Object.entries(filters).forEach(([key, value]) => { + scopes.push({ method: [key, value] }) + }) + } + const scopedDatasetEntryPreview = DatasetEntryPreviewsPolicy.applyScope( + scopes, + this.currentUser + ) + + const totalCount = await scopedDatasetEntryPreview.count({ where }) + const datasetEntryPreviews = await scopedDatasetEntryPreview.findAll({ + where, + limit: this.pagination.limit, + offset: this.pagination.offset, + }) + + return this.response.json({ datasetEntryPreviews, totalCount }) + } +} + +export default DatasetEntryPreviewsController diff --git a/api/src/controllers/datasets-controller.ts b/api/src/controllers/datasets-controller.ts index 5732809d..4cecd043 100644 --- a/api/src/controllers/datasets-controller.ts +++ b/api/src/controllers/datasets-controller.ts @@ -71,9 +71,14 @@ export class DatasetsController extends BaseController { try { const serializedDataset = ShowSerializer.perform(dataset, this.currentUser) + // TODO: consider developing a standard pattern for this? + const serializedPolicy = { + ...policy.toJSON(), + showUnlimited: policy.show({ unlimited: true }), + } return this.response.status(200).json({ dataset: serializedDataset, - policy, + policy: serializedPolicy, }) } catch (error) { return this.response.status(500).json({ message: `Dataset serialization failed: ${error}` }) diff --git a/api/src/controllers/download/datasets-controller.ts b/api/src/controllers/download/datasets-controller.ts index 4b5816ce..1daadc01 100644 --- a/api/src/controllers/download/datasets-controller.ts +++ b/api/src/controllers/download/datasets-controller.ts @@ -56,10 +56,7 @@ export class DatasetsController extends BaseController { // for data rendering logic "file", "integration", - { - association: "fields", - where: { isExcludedFromSearch: false }, - }, + "fields", // for policy logic { association: "owner", diff --git a/api/src/controllers/index.ts b/api/src/controllers/index.ts index 55ab355f..75505ad1 100644 --- a/api/src/controllers/index.ts +++ b/api/src/controllers/index.ts @@ -2,6 +2,7 @@ export { AccessGrantsController } from "./access-grants-controller" export { AccessRequestsController } from "./access-requests-controller" export { CurrentUserController } from "./current-user-controller" export { DatasetEntriesController } from "./dataset-entries-controller" +export { DatasetEntryPreviewsController } from "./dataset-entry-previews-controller" export { DatasetFieldsController } from "./dataset-fields-controller" export { DatasetIntegrationsController } from "./dataset-integrations-controller" export { DatasetsController } from "./datasets-controller" diff --git a/api/src/controllers/visualization-controls-controller.ts b/api/src/controllers/visualization-controls-controller.ts index 52d98d14..176cd80f 100644 --- a/api/src/controllers/visualization-controls-controller.ts +++ b/api/src/controllers/visualization-controls-controller.ts @@ -93,7 +93,7 @@ export class VisualizationControlsController extends BaseController { "accessRequests", ], }, - "searchExcludedDatasetFields", + "previewExcludedDatasetFields", ], }) } diff --git a/api/src/db/db-client.ts b/api/src/db/db-client.ts index 18937fb0..9022f025 100644 --- a/api/src/db/db-client.ts +++ b/api/src/db/db-client.ts @@ -4,8 +4,8 @@ import { createNamespace } from "cls-hooked" import { DB_HOST, DB_USER, DB_PASS, DB_NAME, DB_PORT, NODE_ENV } from "@/config" import { monkeyPatchSequelizeErrorsForJest } from "@/db/utils/monkey-patch-sequelize-errors-for-jest" -const namespace = createNamespace("sequelize-transaction-context") -Sequelize.useCLS(namespace) +export const transactionManager = createNamespace("transaction-manager") +Sequelize.useCLS(transactionManager) if (DB_NAME === undefined) throw new Error("database name is unset.") if (DB_USER === undefined) throw new Error("database username is unset.") @@ -26,7 +26,7 @@ export const SEQUELIZE_CONFIG: Options = { underscored: true, timestamps: true, // This is actually the default, but making it explicit for clarity. paranoid: true, // adds deleted_at column - whereMergeStrategy: 'and', // where fields will be merged using the and operator (instead of overwriting each other) + whereMergeStrategy: "and", // where fields will be merged using the and operator (instead of overwriting each other) }, } diff --git a/api/src/db/migrations/2024.05.16T23.36.54.update-visualization-controls-fields-to-operate-as-preview-modifications.ts b/api/src/db/migrations/2024.05.16T23.36.54.update-visualization-controls-fields-to-operate-as-preview-modifications.ts new file mode 100644 index 00000000..fbec05e6 --- /dev/null +++ b/api/src/db/migrations/2024.05.16T23.36.54.update-visualization-controls-fields-to-operate-as-preview-modifications.ts @@ -0,0 +1,94 @@ +import { DataTypes } from "sequelize" + +import type { Migration } from "@/db/umzug" + +export const up: Migration = async ({ context: queryInterface }) => { + await queryInterface.addColumn("visualization_controls", "has_preview", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET has_preview = 0 -- instead of has_search_customizations for security reasons + `) + await queryInterface.removeColumn("visualization_controls", "has_search_customizations") + + await queryInterface.addColumn("visualization_controls", "has_fields_excluded_from_preview", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: true, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET has_fields_excluded_from_preview = 1 -- instead of has_fields_excluded_from_search for security reasons + `) + await queryInterface.removeColumn("visualization_controls", "has_fields_excluded_from_search") + + await queryInterface.addColumn("visualization_controls", "has_preview_row_limit", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: true, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET has_preview_row_limit = 1 -- instead of has_search_row_limits for security reasons + `) + await queryInterface.removeColumn("visualization_controls", "has_search_row_limits") + + await queryInterface.addColumn("visualization_controls", "preview_row_limit", { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 10, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET preview_row_limit = ISNULL(search_row_limit_maximum, 10) + `) + await queryInterface.removeColumn("visualization_controls", "search_row_limit_maximum") +} + +export const down: Migration = async ({ context: queryInterface }) => { + await queryInterface.addColumn("visualization_controls", "search_row_limit_maximum", { + type: DataTypes.INTEGER, + allowNull: true, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET search_row_limit_maximum = preview_row_limit + `) + await queryInterface.removeColumn("visualization_controls", "preview_row_limit") + + await queryInterface.addColumn("visualization_controls", "has_search_row_limits", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET has_search_row_limits = has_preview_row_limit + `) + await queryInterface.removeColumn("visualization_controls", "has_preview_row_limit") + + await queryInterface.addColumn("visualization_controls", "has_search_customizations", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET has_search_customizations = has_preview + `) + await queryInterface.removeColumn("visualization_controls", "has_preview") + + await queryInterface.addColumn("visualization_controls", "has_fields_excluded_from_search", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE visualization_controls + SET has_fields_excluded_from_search = has_fields_excluded_from_preview + `) + await queryInterface.removeColumn("visualization_controls", "has_fields_excluded_from_preview") +} diff --git a/api/src/db/migrations/2024.05.17T00.15.32.update-dataset-field-exclusion-flag-to-refer-to-preview-not-search.ts b/api/src/db/migrations/2024.05.17T00.15.32.update-dataset-field-exclusion-flag-to-refer-to-preview-not-search.ts new file mode 100644 index 00000000..d494deeb --- /dev/null +++ b/api/src/db/migrations/2024.05.17T00.15.32.update-dataset-field-exclusion-flag-to-refer-to-preview-not-search.ts @@ -0,0 +1,29 @@ +import { DataTypes } from "sequelize" + +import type { Migration } from "@/db/umzug" + +export const up: Migration = async ({ context: queryInterface }) => { + await queryInterface.addColumn("dataset_fields", "is_excluded_from_preview", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: true, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE dataset_fields + SET is_excluded_from_preview = 1 -- instead of is_excluded_from_search for security reasons + `) + await queryInterface.removeColumn("dataset_fields", "is_excluded_from_search") +} + +export const down: Migration = async ({ context: queryInterface }) => { + await queryInterface.addColumn("dataset_fields", "is_excluded_from_search", { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }) + await queryInterface.sequelize.query(/* sql */ ` + UPDATE dataset_fields + SET is_excluded_from_search = 0 -- instead of is_excluded_from_preview as conceptually different + `) + await queryInterface.removeColumn("dataset_fields", "is_excluded_from_preview") +} diff --git a/api/src/db/migrations/2024.05.17T20.38.17.create-dataset-entry-previews-table.ts b/api/src/db/migrations/2024.05.17T20.38.17.create-dataset-entry-previews-table.ts new file mode 100644 index 00000000..27c3b8c6 --- /dev/null +++ b/api/src/db/migrations/2024.05.17T20.38.17.create-dataset-entry-previews-table.ts @@ -0,0 +1,63 @@ +import { DataTypes, Op } from "sequelize" + +import type { Migration } from "@/db/umzug" +import { MssqlSimpleTypes } from "@/db/utils/mssql-simple-types" + +export const up: Migration = async ({ context: queryInterface }) => { + await queryInterface.createTable("dataset_entry_previews", { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + allowNull: false, + autoIncrement: true, + }, + dataset_id: { + type: DataTypes.INTEGER, + allowNull: false, + references: { + model: "datasets", + key: "id", + }, + }, + dataset_entry_id: { + type: DataTypes.INTEGER, + allowNull: false, + references: { + model: "dataset_entries", + key: "id", + }, + }, + json_data: { + type: DataTypes.TEXT, + allowNull: false, + }, + created_at: { + type: MssqlSimpleTypes.DATETIME2(0), + allowNull: false, + defaultValue: MssqlSimpleTypes.NOW, + }, + updated_at: { + type: MssqlSimpleTypes.DATETIME2(0), + allowNull: false, + defaultValue: MssqlSimpleTypes.NOW, + }, + deleted_at: { + type: MssqlSimpleTypes.DATETIME2(0), + allowNull: true, + }, + }) + + await queryInterface.addIndex("dataset_entry_previews", ["dataset_id"]) + await queryInterface.addIndex("dataset_entry_previews", ["dataset_entry_id"], { + unique: true, + where: { + deleted_at: { + [Op.is]: null, + }, + }, + }) +} + +export const down: Migration = async ({ context: queryInterface }) => { + await queryInterface.dropTable("dataset_entry_previews") +} diff --git a/api/src/db/utils/mssql-json-object-types.ts b/api/src/db/utils/mssql-json-object-types.ts new file mode 100644 index 00000000..3a047221 --- /dev/null +++ b/api/src/db/utils/mssql-json-object-types.ts @@ -0,0 +1,11 @@ +/** + * See https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16#return-value + */ +export enum JsonDataType { + NULL = 0, + STRING = 1, + NUMBER = 2, + BOOLEAN = 3, + ARRAY = 4, + OBJECT = 5, +} diff --git a/api/src/db/utils/nestable-transaction.ts b/api/src/db/utils/nestable-transaction.ts new file mode 100644 index 00000000..a0d3d61a --- /dev/null +++ b/api/src/db/utils/nestable-transaction.ts @@ -0,0 +1,60 @@ +import { Sequelize, Transaction, TransactionOptions } from "sequelize" + +import db, { transactionManager } from "@/db/db-client" + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const sequelizeVersion = (Sequelize as any).version +const major = sequelizeVersion.split(".").map(Number)[0] + +if (major >= 7) { + console.warn("nestableTransaction is no longer needed in Sequelize v7!") +} + +/* +Operates just like Sequelize transaction, but nests by default. + +e.g. Using this function is the equivalent of using the following code: +db.transaction((t1) => { + // lots of layers of other code + db.transaction({ transaction: t1 }, () => { + // Your nested transaction logic here + }); +}); + +But without the hassle of having to pass the transaction object around. +*/ +export function nestableTransaction( + options: TransactionOptions, + autoCallback: (t: Transaction) => PromiseLike +): Promise +export function nestableTransaction(autoCallback: (t: Transaction) => PromiseLike): Promise +export function nestableTransaction(options?: TransactionOptions): Promise +export function nestableTransaction( + optionsOrAutoCallback?: TransactionOptions | ((t: Transaction) => PromiseLike), + autoCallback?: (t: Transaction) => PromiseLike +) { + const parentTransaction = transactionManager.get("transaction") + if (typeof optionsOrAutoCallback === "object" && typeof autoCallback === "function") { + return db.transaction( + { + ...optionsOrAutoCallback, + transaction: parentTransaction, + }, + autoCallback + ) + } else if (typeof optionsOrAutoCallback === "function" && autoCallback === undefined) { + return db.transaction( + { + transaction: parentTransaction, + }, + optionsOrAutoCallback + ) + } else { + return db.transaction({ + ...optionsOrAutoCallback, + transaction: parentTransaction, + }) + } +} + +export default nestableTransaction diff --git a/api/src/models/dataset-entries/dataset-entries-search.ts b/api/src/models/dataset-entries/dataset-entries-search.ts index e6a8cf30..51a826a6 100644 --- a/api/src/models/dataset-entries/dataset-entries-search.ts +++ b/api/src/models/dataset-entries/dataset-entries-search.ts @@ -2,6 +2,7 @@ import { literal } from "sequelize" import { Literal } from "sequelize/types/utils" import { compactSql } from "@/utils/compact-sql" +import { JsonDataType } from "@/db/utils/mssql-json-object-types" /** * Requires replacements to be passed in to query. @@ -9,104 +10,34 @@ import { compactSql } from "@/utils/compact-sql" */ export function datasetEntriesSearch(): Literal { /** - * Only applies search field exclusions when enabled. - * * TODO: add ability to inject early filtering, * or at least early filtering on dataset_id, as this will vastly speed up the query. + * + * See https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16#return-value */ - const searchResultsQuery = compactSql(/*sql*/ ` - SELECT - DISTINCT dataset_entries.id as dataset_entry_id, - dataset_entries.dataset_id - FROM - dataset_entries - INNER JOIN dataset_fields ON dataset_fields.dataset_id = dataset_entries.dataset_id - AND dataset_fields.deleted_at IS NULL - INNER JOIN visualization_controls ON visualization_controls.dataset_id = dataset_entries.dataset_id - AND visualization_controls.deleted_at IS NULL - WHERE - dataset_entries.deleted_at IS NULL - AND ( - ( - visualization_controls.has_search_customizations = 1 - AND visualization_controls.has_fields_excluded_from_search = 1 - AND dataset_fields.is_excluded_from_search = 0 - ) - OR ( - visualization_controls.has_search_customizations = 1 - AND visualization_controls.has_fields_excluded_from_search = 0 - ) - OR visualization_controls.has_search_customizations = 0 - ) - AND ( - ( - dataset_fields.data_type = 'text' - AND LOWER( - JSON_VALUE( - dataset_entries.json_data, - CONCAT('$."', dataset_fields.name, '"') - ) - ) LIKE LOWER(:searchTokenWildcard) - ) - OR ( - dataset_fields.data_type = 'integer' - AND TRY_CAST( - JSON_VALUE( - dataset_entries.json_data, - CONCAT('$."', dataset_fields.name, '"') - ) AS NVARCHAR - ) = :searchToken - ) - ) - `) - - /** - * Must deduplicate search results before adding row numbers for result limiting - * or results will be lost. - */ - const numberedSearchResultsQuery = compactSql(/*sql*/ ` - SELECT - dataset_entry_id, - dataset_id, - ROW_NUMBER() OVER( - PARTITION BY dataset_id - ORDER BY dataset_entry_id - ) AS search_result_number - FROM - (${searchResultsQuery}) as search_results - `) - - /** - * Only applies search row limits when enabled. - */ - const query = compactSql(/*sql*/ ` + const matchingEntries = compactSql(/*sql*/ ` ( SELECT - dataset_entry_id + dataset_entries.id FROM - (${numberedSearchResultsQuery}) as numbered_search_results - INNER JOIN visualization_controls ON visualization_controls.dataset_id = numbered_search_results.dataset_id - AND visualization_controls.deleted_at IS NULL + dataset_entries + CROSS APPLY OPENJSON(dataset_entries.json_data) AS json_values WHERE - ( - visualization_controls.has_search_customizations = 1 - AND visualization_controls.has_search_row_limits = 1 - AND numbered_search_results.search_result_number <= visualization_controls.search_row_limit_maximum - ) - OR ( - visualization_controls.has_search_customizations = 1 - AND visualization_controls.has_search_row_limits = 1 - AND visualization_controls.search_row_limit_maximum IS NULL - ) - OR ( - visualization_controls.has_search_customizations = 1 - AND visualization_controls.has_search_row_limits = 0 + dataset_entries.deleted_at IS NULL + AND ( + ( + json_values.[type] = ${JsonDataType.STRING} + AND LOWER(json_values.value) LIKE LOWER(:searchTokenWildcard) + ) + OR ( + json_values.[type] = ${JsonDataType.NUMBER} + AND TRY_CAST(json_values.value AS NVARCHAR) = :searchToken + ) ) - OR visualization_controls.has_search_customizations = 0 ) `) - return literal(query) + return literal(matchingEntries) } export default datasetEntriesSearch diff --git a/api/src/models/dataset-entry-preview.ts b/api/src/models/dataset-entry-preview.ts new file mode 100644 index 00000000..6e216fc9 --- /dev/null +++ b/api/src/models/dataset-entry-preview.ts @@ -0,0 +1,170 @@ +import { + Association, + BelongsToCreateAssociationMixin, + BelongsToGetAssociationMixin, + BelongsToSetAssociationMixin, + CreationOptional, + DataTypes, + ForeignKey, + InferAttributes, + InferCreationAttributes, + NonAttribute, + Op, +} from "sequelize" +import { isEmpty } from "lodash" + +import sequelize from "@/db/db-client" + +import { datasetEntryPreviewsSearch } from "@/models/dataset-entry-previews" +import BaseModel from "@/models/base-model" +import Dataset from "@/models/dataset" +import DatasetEntry, { DatasetEntryJsonDataType } from "@/models/dataset-entry" +import VisualizationControl from "@/models/visualization-control" + +export class DatasetEntryPreview extends BaseModel< + InferAttributes, + InferCreationAttributes +> { + declare id: CreationOptional + declare datasetId: ForeignKey + declare datasetEntryId: ForeignKey + declare jsonData: DatasetEntryJsonDataType + declare createdAt: CreationOptional + declare updatedAt: CreationOptional + declare deletedAt: CreationOptional + + // https://sequelize.org/docs/v6/other-topics/typescript/#usage + // https://sequelize.org/docs/v6/core-concepts/assocs/#special-methodsmixins-added-to-instances + // https://sequelize.org/api/v7/types/_sequelize_core.index.belongstocreateassociationmixin + declare getDataset: BelongsToGetAssociationMixin + declare setDataset: BelongsToSetAssociationMixin + declare createDataset: BelongsToCreateAssociationMixin + + declare getDatasetEntry: BelongsToGetAssociationMixin + declare setDatasetEntry: BelongsToSetAssociationMixin + declare createDatasetEntry: BelongsToCreateAssociationMixin + + declare dataset?: NonAttribute + declare datasetEntry?: NonAttribute + declare visualizationControl?: NonAttribute + + declare static associations: { + dataset: Association + datasetEntry: Association + visualizationControl: Association + } + + static establishAssociations() { + this.belongsTo(Dataset, { as: "dataset" }) + this.belongsTo(DatasetEntry, { as: "datasetEntry" }) + this.belongsTo(VisualizationControl, { + as: "visualizationControl", + foreignKey: "datasetId", + targetKey: "datasetId", + }) + } +} + +DatasetEntryPreview.init( + { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + allowNull: false, + autoIncrement: true, + }, + datasetId: { + type: DataTypes.INTEGER, + allowNull: false, + references: { + model: Dataset, + key: "id", + }, + }, + datasetEntryId: { + type: DataTypes.INTEGER, + allowNull: false, + references: { + model: DatasetEntry, + key: "id", + }, + }, + jsonData: { + type: DataTypes.TEXT, + allowNull: false, + get() { + const value = this.getDataValue("jsonData") as unknown as string + return JSON.parse(value) + }, + set(value: string) { + // TODO: assert value matches schema + this.setDataValue("jsonData", JSON.stringify(value) as unknown as DatasetEntryJsonDataType) + }, + }, + createdAt: { + type: DataTypes.DATE, + allowNull: false, + defaultValue: DataTypes.NOW, + }, + updatedAt: { + type: DataTypes.DATE, + allowNull: false, + defaultValue: DataTypes.NOW, + }, + deletedAt: { + type: DataTypes.DATE, + allowNull: true, + }, + }, + { + sequelize, + indexes: [ + { + fields: ["datasetId"], + }, + { + unique: true, + fields: ["dataset_entry_id"], + where: { + deleted_at: { + [Op.is]: null, + }, + }, + }, + ], + scopes: { + search(searchToken: string) { + if (isEmpty(searchToken)) { + return {} + } + + return { + where: { + id: { + [Op.in]: datasetEntryPreviewsSearch(), + }, + }, + replacements: { + searchTokenWildcard: `%${searchToken}%`, + searchToken, + }, + } + }, + withPreviewEnabled() { + return { + include: [ + { + association: "visualizationControl", + attributes: [], + where: { + hasPreview: true, + }, + }, + ], + } + }, + }, + } +) + +export default DatasetEntryPreview diff --git a/api/src/models/dataset-entry-previews/dataset-entry-previews-search.ts b/api/src/models/dataset-entry-previews/dataset-entry-previews-search.ts new file mode 100644 index 00000000..9134d742 --- /dev/null +++ b/api/src/models/dataset-entry-previews/dataset-entry-previews-search.ts @@ -0,0 +1,43 @@ +import { literal } from "sequelize" +import { Literal } from "sequelize/types/utils" + +import { compactSql } from "@/utils/compact-sql" +import { JsonDataType } from "@/db/utils/mssql-json-object-types" + +/** + * Requires replacements to be passed in to query. + * e.g. { replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken } + */ +export function datasetEntryPreviewsSearch(): Literal { + /** + * TODO: add ability to inject early filtering, + * or at least early filtering on dataset_id, as this will vastly speed up the query. + * + * See https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16#return-value + */ + const matchingEntries = compactSql(/*sql*/ ` + ( + SELECT + dataset_entry_previews.id + FROM + dataset_entry_previews + CROSS APPLY OPENJSON(dataset_entry_previews.json_data) AS json_values + WHERE + dataset_entry_previews.deleted_at IS NULL + AND ( + ( + json_values.[type] = ${JsonDataType.STRING} + AND LOWER(json_values.value) LIKE LOWER(:searchTokenWildcard) + ) + OR ( + json_values.[type] = ${JsonDataType.NUMBER} + AND TRY_CAST(json_values.value AS NVARCHAR) = :searchToken + ) + ) + ) + `) + + return literal(matchingEntries) +} + +export default datasetEntryPreviewsSearch diff --git a/api/src/models/dataset-entry-previews/index.ts b/api/src/models/dataset-entry-previews/index.ts new file mode 100644 index 00000000..ed072f5c --- /dev/null +++ b/api/src/models/dataset-entry-previews/index.ts @@ -0,0 +1 @@ +export { datasetEntryPreviewsSearch } from "./dataset-entry-previews-search" diff --git a/api/src/models/dataset-field.ts b/api/src/models/dataset-field.ts index 7e663719..aba9b480 100644 --- a/api/src/models/dataset-field.ts +++ b/api/src/models/dataset-field.ts @@ -35,7 +35,7 @@ export class DatasetField extends Model< declare dataType: DatasetFieldDataTypes declare description: CreationOptional declare note: CreationOptional - declare isExcludedFromSearch: CreationOptional + declare isExcludedFromPreview: CreationOptional declare createdAt: CreationOptional declare updatedAt: CreationOptional declare deletedAt: CreationOptional @@ -97,7 +97,7 @@ DatasetField.init( type: DataTypes.TEXT, allowNull: true, }, - isExcludedFromSearch: { + isExcludedFromPreview: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, diff --git a/api/src/models/datasets/datasets-with-approved-access-requests-for.ts b/api/src/models/datasets/datasets-with-approved-access-requests-for.ts index 9e91389a..f96d87d4 100644 --- a/api/src/models/datasets/datasets-with-approved-access-requests-for.ts +++ b/api/src/models/datasets/datasets-with-approved-access-requests-for.ts @@ -17,6 +17,7 @@ export function datasetsWithApprovedAccessRequestsFor(user: User): Literal { WHERE access_requests.requestor_id = ${user.id} AND access_requests.approved_at IS NOT NULL + AND access_requests.revoked_at IS NULL ) `) diff --git a/api/src/models/datasets/datasets-with-field-exclusions-disabled.ts b/api/src/models/datasets/datasets-with-field-exclusions-disabled.ts new file mode 100644 index 00000000..7d5bf8a1 --- /dev/null +++ b/api/src/models/datasets/datasets-with-field-exclusions-disabled.ts @@ -0,0 +1,25 @@ +import { literal } from "sequelize" +import { Literal } from "sequelize/types/utils" + +import { compactSql } from "@/utils/compact-sql" + +export function datasetsWithFieldExclusionsDisabled(): Literal { + const query = compactSql(/* sql */ ` + ( + SELECT + datasets.id + FROM + datasets + INNER JOIN visualization_controls ON + visualization_controls.dataset_id = datasets.id + AND datasets.deleted_at IS NULL + AND visualization_controls.deleted_at IS NULL + WHERE + visualization_controls.has_fields_excluded_from_preview = 0 + ) + `) + + return literal(query) +} + +export default datasetsWithFieldExclusionsDisabled diff --git a/api/src/models/datasets/datasets-with-preview-disabled.ts b/api/src/models/datasets/datasets-with-preview-disabled.ts new file mode 100644 index 00000000..69b37b8d --- /dev/null +++ b/api/src/models/datasets/datasets-with-preview-disabled.ts @@ -0,0 +1,25 @@ +import { literal } from "sequelize" +import { Literal } from "sequelize/types/utils" + +import { compactSql } from "@/utils/compact-sql" + +export function datasetsWithPreviewDisabled(): Literal { + const query = compactSql(/* sql */ ` + ( + SELECT + datasets.id + FROM + datasets + INNER JOIN visualization_controls ON + visualization_controls.dataset_id = datasets.id + AND datasets.deleted_at IS NULL + AND visualization_controls.deleted_at IS NULL + WHERE + visualization_controls.has_preview = 0 + ) + `) + + return literal(query) +} + +export default datasetsWithPreviewDisabled diff --git a/api/src/models/datasets/index.ts b/api/src/models/datasets/index.ts index 3bda13c2..0b65cb23 100644 --- a/api/src/models/datasets/index.ts +++ b/api/src/models/datasets/index.ts @@ -3,4 +3,6 @@ export { datasetIsAccessibleViaOpenAccessGrantBy } from "./dataset-is-accessible export { datasetsAccessibleViaAccessGrantsBy } from "./datasets-accessible-via-access-grants-by" export { datasetsAccessibleViaOwner } from "./datasets-accessible-via-owner" export { datasetsWithApprovedAccessRequestsFor } from "./datasets-with-approved-access-requests-for" +export { datasetsWithFieldExclusionsDisabled } from "./datasets-with-field-exclusions-disabled" +export { datasetsWithPreviewDisabled } from "./datasets-with-preview-disabled" export { mostPermissiveAccessGrantFor } from "./most-permissive-access-grant-for" diff --git a/api/src/models/index.ts b/api/src/models/index.ts index 331343f3..4844ba8d 100644 --- a/api/src/models/index.ts +++ b/api/src/models/index.ts @@ -1,43 +1,45 @@ import db from "@/db/db-client" -import { Role } from "@/models/role" -import { User } from "@/models/user" +import { AccessGrant } from "@/models/access-grant" +import { AccessRequest } from "@/models/access-request" import { Dataset } from "@/models/dataset" +import { DatasetEntry } from "@/models/dataset-entry" +import { DatasetEntryPreview } from "@/models/dataset-entry-preview" +import { DatasetField } from "@/models/dataset-field" +import { DatasetFile } from "@/models/dataset-file" +import { DatasetIntegration } from "@/models/dataset-integration" import { DatasetStewardship } from "@/models/dataset-stewardship" -import { UserGroup } from "@/models/user-groups" -import { UserGroupMembership } from "@/models/user-group-membership" +import { Role } from "@/models/role" import { Tag } from "@/models/tag" import { Tagging } from "@/models/tagging" -import { AccessGrant } from "@/models/access-grant" -import { AccessRequest } from "@/models/access-request" -import { DatasetField } from "@/models/dataset-field" +import { User } from "@/models/user" +import { UserGroup } from "@/models/user-groups" +import { UserGroupMembership } from "@/models/user-group-membership" import { VisualizationControl } from "@/models/visualization-control" -import { DatasetEntry } from "@/models/dataset-entry" -import { DatasetIntegration } from "@/models/dataset-integration" -import { DatasetFile } from "@/models/dataset-file" -// Estabilish associations between models, order likely matters +AccessGrant.establishAssociations() +AccessRequest.establishAssociations() +Dataset.establishAssociations() +DatasetEntry.establishAssociations() +DatasetEntryPreview.establishAssociations() +DatasetField.establishAssociations() +DatasetFile.establishAssociations() +DatasetIntegration.establishAssociations() +DatasetStewardship.establishAssociations() Role.establishAssociations() +Tag.establishAssociations() +Tagging.establishAssociations() User.establishAssociations() -Dataset.establishAssociations() UserGroup.establishAssociations() UserGroupMembership.establishAssociations() -Tag.establishAssociations() -Tagging.establishAssociations() -AccessGrant.establishAssociations() -AccessRequest.establishAssociations() -DatasetStewardship.establishAssociations() -DatasetField.establishAssociations() VisualizationControl.establishAssociations() -DatasetEntry.establishAssociations() -DatasetIntegration.establishAssociations() -DatasetFile.establishAssociations() export { AccessGrant, AccessRequest, Dataset, DatasetEntry, + DatasetEntryPreview, DatasetField, DatasetFile, DatasetIntegration, diff --git a/api/src/models/visualization-control.ts b/api/src/models/visualization-control.ts index 18f612cc..9e9f1433 100644 --- a/api/src/models/visualization-control.ts +++ b/api/src/models/visualization-control.ts @@ -34,10 +34,10 @@ export class VisualizationControl extends Model< declare id: CreationOptional declare datasetId: ForeignKey declare isDownloadableAsCsv: CreationOptional - declare hasSearchCustomizations: CreationOptional - declare hasFieldsExcludedFromSearch: CreationOptional - declare hasSearchRowLimits: CreationOptional - declare searchRowLimitMaximum: CreationOptional + declare hasPreview: CreationOptional + declare hasFieldsExcludedFromPreview: CreationOptional + declare hasPreviewRowLimit: CreationOptional + declare previewRowLimit: CreationOptional declare createdAt: CreationOptional declare updatedAt: CreationOptional declare deletedAt: CreationOptional @@ -49,54 +49,54 @@ export class VisualizationControl extends Model< declare setDataset: BelongsToSetAssociationMixin declare createDataset: BelongsToCreateAssociationMixin - declare getSearchExcludedDatasetFields: HasManyGetAssociationsMixin - declare setSearchExcludedDatasetFields: HasManySetAssociationsMixin< + declare getPreviewExcludedDatasetFields: HasManyGetAssociationsMixin + declare setPreviewExcludedDatasetFields: HasManySetAssociationsMixin< DatasetField, DatasetField["datasetId"] > - declare hasSearchExcludedDatasetField: HasManyHasAssociationMixin< + declare hasPreviewExcludedDatasetField: HasManyHasAssociationMixin< DatasetField, DatasetField["datasetId"] > - declare hasSearchExcludedDatasetFields: HasManyHasAssociationsMixin< + declare hasPreviewExcludedDatasetFields: HasManyHasAssociationsMixin< DatasetField, DatasetField["datasetId"] > - declare addSearchExcludedDatasetField: HasManyAddAssociationMixin< + declare addPreviewExcludedDatasetField: HasManyAddAssociationMixin< DatasetField, DatasetField["datasetId"] > - declare addSearchExcludedDatasetFields: HasManyAddAssociationsMixin< + declare addPreviewExcludedDatasetFields: HasManyAddAssociationsMixin< DatasetField, DatasetField["datasetId"] > - declare removeSearchExcludedDatasetField: HasManyRemoveAssociationMixin< + declare removePreviewExcludedDatasetField: HasManyRemoveAssociationMixin< DatasetField, DatasetField["datasetId"] > - declare removeSearchExcludedDatasetFields: HasManyRemoveAssociationsMixin< + declare removePreviewExcludedDatasetFields: HasManyRemoveAssociationsMixin< DatasetField, DatasetField["datasetId"] > - declare countSearchExcludedDatasetFields: HasManyCountAssociationsMixin - declare createSearchExcludedDatasetField: HasManyCreateAssociationMixin + declare countPreviewExcludedDatasetFields: HasManyCountAssociationsMixin + declare createPreviewExcludedDatasetField: HasManyCreateAssociationMixin declare dataset?: NonAttribute - declare searchExcludedDatasetFields?: NonAttribute + declare previewExcludedDatasetFields?: NonAttribute declare static associations: { dataset: Association - searchExcludedDatasetFields: Association + previewExcludedDatasetFields: Association } static establishAssociations() { this.belongsTo(Dataset, { as: "dataset" }) this.hasMany(DatasetField, { - as: "searchExcludedDatasetFields", + as: "previewExcludedDatasetFields", sourceKey: "datasetId", foreignKey: "datasetId", scope: { - isExcludedFromSearch: true, + isExcludedFromPreview: true, }, }) } @@ -123,24 +123,28 @@ VisualizationControl.init( allowNull: false, defaultValue: false, }, - hasSearchCustomizations: { + hasPreview: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, }, - hasFieldsExcludedFromSearch: { + hasFieldsExcludedFromPreview: { type: DataTypes.BOOLEAN, allowNull: false, - defaultValue: false, + defaultValue: true, }, - hasSearchRowLimits: { + hasPreviewRowLimit: { type: DataTypes.BOOLEAN, allowNull: false, - defaultValue: false, + defaultValue: true, }, - searchRowLimitMaximum: { + previewRowLimit: { type: DataTypes.INTEGER, - allowNull: true, + allowNull: false, + defaultValue: 10, + validate: { + min: 10, + }, }, createdAt: { type: DataTypes.DATE, diff --git a/api/src/policies/dataset-entry-previews-policy.ts b/api/src/policies/dataset-entry-previews-policy.ts new file mode 100644 index 00000000..4b8f6c22 --- /dev/null +++ b/api/src/policies/dataset-entry-previews-policy.ts @@ -0,0 +1,48 @@ +import { Attributes, FindOptions, Op } from "sequelize" + +import { DatasetEntryPreview, User } from "@/models" +import { + datasetsAccessibleViaAccessGrantsBy, + datasetsAccessibleViaOwner, + datasetsWithPreviewDisabled, +} from "@/models/datasets" +import { PolicyFactory } from "@/policies/base-policy" + +export class DatasetEntryPreviewsPolicy extends PolicyFactory(DatasetEntryPreview) { + static policyScope(user: User): FindOptions> { + if (user.isSystemAdmin || user.isBusinessAnalyst) { + return {} + } + + const datasetsAccessibleViaAccessGrantsByUserQuery = datasetsAccessibleViaAccessGrantsBy(user) + const datasetsWithPreviewDisabledQuery = datasetsWithPreviewDisabled() + + if (user.isDataOwner) { + const datasetsAccessibleViaOwnerQuery = datasetsAccessibleViaOwner(user) + return { + where: { + datasetId: { + [Op.or]: [ + { [Op.in]: datasetsAccessibleViaOwnerQuery }, + { + [Op.in]: datasetsAccessibleViaAccessGrantsByUserQuery, + [Op.notIn]: datasetsWithPreviewDisabledQuery, + }, + ], + }, + }, + } + } + + return { + where: { + datasetId: { + [Op.in]: datasetsAccessibleViaAccessGrantsByUserQuery, + [Op.notIn]: datasetsWithPreviewDisabledQuery, + }, + }, + } + } +} + +export default DatasetEntryPreviewsPolicy diff --git a/api/src/policies/dataset-fields-policy.ts b/api/src/policies/dataset-fields-policy.ts index cac202de..9bc3ec48 100644 --- a/api/src/policies/dataset-fields-policy.ts +++ b/api/src/policies/dataset-fields-policy.ts @@ -8,10 +8,11 @@ import { datasetsAccessibleViaAccessGrantsBy, datasetsAccessibleViaOwner, datasetsWithApprovedAccessRequestsFor, + datasetsWithFieldExclusionsDisabled, + datasetsWithPreviewDisabled, } from "@/models/datasets" -import DatasetsPolicy from "@/policies/datasets-policy" - import { PolicyFactory } from "@/policies/base-policy" +import DatasetsPolicy from "@/policies/datasets-policy" export type DatasetFieldWithDataset = DatasetField & { dataset: NonAttribute } @@ -46,37 +47,71 @@ export class DatasetFieldsPolicy extends PolicyFactory & { - searchExcludedDatasetFields: DatasetField[] + previewExcludedDatasetFields: DatasetField[] } export class ShowSerializer extends BaseSerializer { @@ -16,8 +16,8 @@ export class ShowSerializer extends BaseSerializer { } perform(): VisualizationControlShowView { - if (isUndefined(this.record.searchExcludedDatasetFields)) { - throw new Error("Expected record to have a searchExcludedDatasetFields association") + if (isUndefined(this.record.previewExcludedDatasetFields)) { + throw new Error("Expected record to have a previewExcludedDatasetFields association") } return { @@ -25,14 +25,14 @@ export class ShowSerializer extends BaseSerializer { "id", "datasetId", "isDownloadableAsCsv", - "hasSearchCustomizations", - "hasFieldsExcludedFromSearch", - "hasSearchRowLimits", - "searchRowLimitMaximum", + "hasPreview", + "hasFieldsExcludedFromPreview", + "hasPreviewRowLimit", + "previewRowLimit", "createdAt", "updatedAt", ]), - searchExcludedDatasetFields: this.record.searchExcludedDatasetFields, + previewExcludedDatasetFields: this.record.previewExcludedDatasetFields, } } } diff --git a/api/src/services/visualization-controls/index.ts b/api/src/services/visualization-controls/index.ts index 3688a202..4ccb6587 100644 --- a/api/src/services/visualization-controls/index.ts +++ b/api/src/services/visualization-controls/index.ts @@ -1 +1,2 @@ +export { RefreshDatasetEntryPreviewService } from "./refresh-dataset-entry-preview-service" export { UpdateService } from "./update-service" diff --git a/api/src/services/visualization-controls/refresh-dataset-entry-preview-service.ts b/api/src/services/visualization-controls/refresh-dataset-entry-preview-service.ts new file mode 100644 index 00000000..2c616431 --- /dev/null +++ b/api/src/services/visualization-controls/refresh-dataset-entry-preview-service.ts @@ -0,0 +1,68 @@ +import { isEmpty, pick } from "lodash" + +import nestableTransaction from "@/db/utils/nestable-transaction" +import { + DatasetEntry, + DatasetEntryPreview, + DatasetField, + User, + VisualizationControl, +} from "@/models" +import BaseService from "@/services/base-service" + +export class RefreshDatasetEntryPreviewService extends BaseService { + constructor( + protected visualizationControl: VisualizationControl, + protected currentUser: User + ) { + super() + } + + async perform(): Promise { + return nestableTransaction(async () => { + await DatasetEntryPreview.destroy({ + where: { + datasetId: this.visualizationControl.datasetId, + }, + }) + + const fieldExclusionCondition = + this.visualizationControl.hasFieldsExcludedFromPreview == true + ? { isExcludedFromPreview: false } + : {} + const previewableDatasetFields = await DatasetField.findAll({ + where: { + datasetId: this.visualizationControl.datasetId, + ...fieldExclusionCondition, + }, + }) + const previewableDatasetFieldNames = previewableDatasetFields.map((field) => field.name) + if (isEmpty(previewableDatasetFieldNames)) { + return [] + } + + const limitCondition = + this.visualizationControl.hasPreviewRowLimit == true + ? { limit: this.visualizationControl.previewRowLimit } + : {} + const datasetEntriesForPreview = await DatasetEntry.findAll({ + where: { + datasetId: this.visualizationControl.datasetId, + }, + offset: 0, + ...limitCondition, + }) + const datasetEntryPreviewsAttributes = datasetEntriesForPreview.map((datasetEntry) => { + const jsonData = pick(datasetEntry.jsonData, previewableDatasetFieldNames) + return { + datasetId: this.visualizationControl.datasetId, + datasetEntryId: datasetEntry.id, + jsonData, + } + }) + return DatasetEntryPreview.bulkCreate(datasetEntryPreviewsAttributes) + }) + } +} + +export default RefreshDatasetEntryPreviewService diff --git a/api/src/services/visualization-controls/update-service.ts b/api/src/services/visualization-controls/update-service.ts index d1ada41c..5db1d390 100644 --- a/api/src/services/visualization-controls/update-service.ts +++ b/api/src/services/visualization-controls/update-service.ts @@ -1,10 +1,11 @@ import db, { DatasetField, User, VisualizationControl } from "@/models" import BaseService from "@/services/base-service" +import RefreshDatasetEntryPreviewService from "@/services/visualization-controls/refresh-dataset-entry-preview-service" -type DatasetFieldsAttributes = Pick[] +type DatasetFieldsAttributes = Pick[] type Attributes = Partial & { - searchExcludedDatasetFieldsAttributes?: DatasetFieldsAttributes + previewExcludedDatasetFieldsAttributes?: DatasetFieldsAttributes } export class UpdateService extends BaseService { @@ -20,15 +21,19 @@ export class UpdateService extends BaseService { return db.transaction(async () => { await this.visualizationControl.update(this.attributes) - const { searchExcludedDatasetFieldsAttributes } = this.attributes - if (searchExcludedDatasetFieldsAttributes) { - await this.bulkReplaceSearchExcludeOnDatasetFields(searchExcludedDatasetFieldsAttributes) + const { previewExcludedDatasetFieldsAttributes } = this.attributes + if (previewExcludedDatasetFieldsAttributes) { + await this.bulkReplaceSearchExcludeOnDatasetFields(previewExcludedDatasetFieldsAttributes) + } + + if (this.visualizationControl.hasPreview) { + await RefreshDatasetEntryPreviewService.perform(this.visualizationControl, this.currentUser) } // TODO: log user action return this.visualizationControl.reload({ - include: ["searchExcludedDatasetFields"], + include: ["previewExcludedDatasetFields"], }) }) } @@ -36,21 +41,21 @@ export class UpdateService extends BaseService { private async bulkReplaceSearchExcludeOnDatasetFields(attributes: DatasetFieldsAttributes) { await DatasetField.update( { - isExcludedFromSearch: false, + isExcludedFromPreview: false, }, { where: { datasetId: this.visualizationControl.datasetId, - isExcludedFromSearch: true, + isExcludedFromPreview: true, }, } ) const datasetFieldIds = attributes - .filter((attributes) => attributes.isExcludedFromSearch) + .filter((attributes) => attributes.isExcludedFromPreview) .map((attributes) => attributes.id) await DatasetField.update( { - isExcludedFromSearch: true, + isExcludedFromPreview: true, }, { where: { diff --git a/api/src/utils/compact-sql.ts b/api/src/utils/compact-sql.ts index 97c39996..dc00a3cc 100644 --- a/api/src/utils/compact-sql.ts +++ b/api/src/utils/compact-sql.ts @@ -1,3 +1,16 @@ +/** + * Don't overuse this, it's not a full SQL parser. + * It's only purpose is to make SQL formatted by Sequelize 6 a bit more readable during development. + * It is no longer needed in Sequelize 7. + */ export function compactSql(sql: string) { - return sql.replace(/\s+/g, " ").trim() + const multiLineCommentPattern = /\/\*[\s\S]*?\*\//g + const singleLineCommentPattern = /--.*$/gm + const multiWhitespacePattern = /\s+/g + + return sql + .replace(multiLineCommentPattern, "") + .replace(singleLineCommentPattern, "") + .replace(multiWhitespacePattern, " ") + .trim() } diff --git a/api/tests/factories/dataset-entry-preview-factory.ts b/api/tests/factories/dataset-entry-preview-factory.ts new file mode 100644 index 00000000..944566f7 --- /dev/null +++ b/api/tests/factories/dataset-entry-preview-factory.ts @@ -0,0 +1,39 @@ +import { type DeepPartial } from "fishery" + +import { DatasetEntryPreview } from "@/models" +import { DatasetEntryJsonDataType } from "@/models/dataset-entry" + +import BaseFactory from "@/factories/base-factory" + +export const datasetEntryPreviewFactory = BaseFactory.define( + ({ sequence, params, onCreate }) => { + onCreate((datasetEntryPreview) => datasetEntryPreview.save()) + + assertParamsHasDatasetId(params) + assertParamsHasJsonData(params) + + return DatasetEntryPreview.build({ + id: sequence, + datasetId: params.datasetId, + jsonData: params.jsonData, + }) + } +) + +function assertParamsHasDatasetId( + params: DeepPartial +): asserts params is DeepPartial & { datasetId: number } { + if (typeof params.datasetId !== "number") { + throw new Error("datasetId is must be a number") + } +} + +function assertParamsHasJsonData( + params: DeepPartial +): asserts params is DeepPartial & { jsonData: DatasetEntryJsonDataType } { + if (typeof params.jsonData !== "object") { + throw new Error("jsonData is must be an object") + } +} + +export default datasetEntryPreviewFactory diff --git a/api/tests/factories/index.ts b/api/tests/factories/index.ts index 730da999..47bde9b6 100644 --- a/api/tests/factories/index.ts +++ b/api/tests/factories/index.ts @@ -2,6 +2,7 @@ export { accessGrantFactory } from "./access-grant-factory" export { accessRequestFactory } from "./access-request-factory" export { datasetEntryFactory } from "./dataset-entry-factory" +export { datasetEntryPreviewFactory } from "./dataset-entry-preview-factory" export { datasetFactory } from "./dataset-factory" export { datasetFieldFactory } from "./dataset-field-factory" export { datasetIntegrationFactory } from "./dataset-integration-factory" diff --git a/api/tests/factories/visualization-control-factory.ts b/api/tests/factories/visualization-control-factory.ts index 2b6ceff9..c62e0485 100644 --- a/api/tests/factories/visualization-control-factory.ts +++ b/api/tests/factories/visualization-control-factory.ts @@ -15,10 +15,10 @@ export const visualizationControlFactory = BaseFactory.define { // Act const searchToken = "mar" const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ + const result = await DatasetEntry.findAll({ where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, }) @@ -68,63 +66,6 @@ describe("api/src/models/dataset-entries/dataset-entries-search.ts", () => { ]) }) - test("does not include results from excluded columns", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: true, - hasFieldsExcludedFromSearch: true, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - isExcludedFromSearch: true, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "firstName", - dataType: DatasetField.DataTypes.TEXT, - }) - await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "mbrunner@test.com", firstName: "Marlen" }, - }) - await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "mdoe@test.com", firstName: "Mark" }, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "m.doe@test.com", firstName: "Test" }, - }) - - // Act - const searchToken = "test" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - ]) - }) - test("when token is an exact match, it includes results from integer columns", async () => { // Arrange const user = await userFactory.create() @@ -161,10 +102,8 @@ describe("api/src/models/dataset-entries/dataset-entries-search.ts", () => { // Act const searchToken = "33" const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ + const result = await DatasetEntry.findAll({ where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, }) @@ -220,144 +159,8 @@ describe("api/src/models/dataset-entries/dataset-entries-search.ts", () => { // Act const searchToken = "Mar" const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry3.jsonData.email, - }), - }), - ]) - }) - - test("when search field exclusions are disabled, includes results from excluded columns", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: true, - hasFieldsExcludedFromSearch: false, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - isExcludedFromSearch: true, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "firstName", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "mbrunner@test.com", firstName: "Marlen" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "mdoe@test.com", firstName: "Mark" }, - }) - const datasetEntry3 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "m.doe@test.com", firstName: "Test" }, - }) - - // Act - const searchToken = "test" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry3.jsonData.email, - }), - }), - ]) - }) - - test("when search customization is disabled, even if fields excluded from search is true, includes results from excluded columns", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: false, - hasFieldsExcludedFromSearch: true, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - isExcludedFromSearch: true, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "firstName", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "mbrunner@test.com", firstName: "Marlen" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "mdoe@test.com", firstName: "Mark" }, - }) - const datasetEntry3 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "m.doe@test.com", firstName: "Test" }, - }) - - // Act - const searchToken = "test" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ + const result = await DatasetEntry.findAll({ where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, }) @@ -382,311 +185,6 @@ describe("api/src/models/dataset-entries/dataset-entries-search.ts", () => { ]) }) - test("when search row limit is enabled, limits the results", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: true, - hasSearchRowLimits: true, - searchRowLimitMaximum: 2, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marlen@test.com" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Mark@test.com" }, - }) - await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marten@test.com" }, - }) - - // Act - const searchToken = "mar" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - ]) - }) - - test("when search row limit is enabled, and search customization is disabled, does not limit the results", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: false, - hasSearchRowLimits: true, - searchRowLimitMaximum: 2, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marlen@test.com" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Mark@test.com" }, - }) - const datsetEntry3 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marten@test.com" }, - }) - - // Act - const searchToken = "mar" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datsetEntry3.jsonData.email, - }), - }), - ]) - }) - - test("search customization is enabled, and search row limits is disabled, does not limit the results", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: true, - hasSearchRowLimits: false, - searchRowLimitMaximum: 2, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marlen@test.com" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Mark@test.com" }, - }) - const datsetEntry3 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marten@test.com" }, - }) - - // Act - const searchToken = "mar" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datsetEntry3.jsonData.email, - }), - }), - ]) - }) - - test("search row limiting is enabled, but row limit is not set, does not limit the results", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: true, - hasSearchRowLimits: true, - searchRowLimitMaximum: null, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marlen@test.com" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Mark@test.com" }, - }) - const datsetEntry3 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marten@test.com" }, - }) - - // Act - const searchToken = "mar" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datsetEntry3.jsonData.email, - }), - }), - ]) - }) - - test("when search row limit is enabled, and multiple fields exist, does not limit the results to much", async () => { - // Arrange - const user = await userFactory.create() - const dataset = await datasetFactory.create({ - creatorId: user.id, - ownerId: user.id, - }) - await visualizationControlFactory.create({ - datasetId: dataset.id, - hasSearchCustomizations: true, - hasSearchRowLimits: true, - searchRowLimitMaximum: 2, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "email", - dataType: DatasetField.DataTypes.TEXT, - }) - await datasetFieldFactory.create({ - datasetId: dataset.id, - name: "firstName", - dataType: DatasetField.DataTypes.TEXT, - }) - const datasetEntry1 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marlen@test.com", firstName: "Marlen" }, - }) - const datasetEntry2 = await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Mark@test.com", firstName: "Mark" }, - }) - await datasetEntryFactory.create({ - datasetId: dataset.id, - jsonData: { email: "Marten@test.com", firstName: "Marten" }, - }) - - // Act - const searchToken = "mar" - const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ - where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ - replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, - }) - - // Assert - expect.assertions(1) - expect(result).toEqual([ - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry1.jsonData.email, - }), - }), - expect.objectContaining({ - jsonData: expect.objectContaining({ - email: datasetEntry2.jsonData.email, - }), - }), - ]) - }) - test("when dataset field name includes spaces, includes the expected results", async () => { // Arrange const user = await userFactory.create() @@ -718,10 +216,8 @@ describe("api/src/models/dataset-entries/dataset-entries-search.ts", () => { // Act const searchToken = "mar" const query = datasetEntriesSearch() - const scope = DatasetEntry.scope({ + const result = await DatasetEntry.findAll({ where: { id: { [Op.in]: query } }, - }) - const result = await scope.findAll({ replacements: { searchTokenWildcard: `%${searchToken}%`, searchToken }, }) diff --git a/api/tests/models/datasets/datasets-with-approved-access-requests-for.test.ts b/api/tests/models/datasets/datasets-with-approved-access-requests-for.test.ts index 575478af..13f5b110 100644 --- a/api/tests/models/datasets/datasets-with-approved-access-requests-for.test.ts +++ b/api/tests/models/datasets/datasets-with-approved-access-requests-for.test.ts @@ -50,7 +50,7 @@ describe("api/src/models/datasets/datasets-with-approved-access-requests-for.ts" const result = await scope.findAll() // Assert - expect(Dataset.count()).resolves.toBe(2) + expect(await Dataset.count()).toBe(2) expect(result).toEqual([ expect.objectContaining({ id: screenedDataset.id, @@ -90,8 +90,54 @@ describe("api/src/models/datasets/datasets-with-approved-access-requests-for.ts" const result = await scope.findAll() // Assert - expect(Dataset.count()).resolves.toBe(1) + expect(await Dataset.count()).toBe(1) expect(result).toHaveLength(0) }) + + test("when access request is approved, then later revoked, it does not return datasets", async () => { + // Arrange + const requestingUser = await userFactory.create() + + const datasetOwner = await userFactory.create() + + const screenedAccessGrant = accessGrantFactory.build({ + creatorId: datasetOwner.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const screenedDataset = await datasetFactory + .associations({ + accessGrants: [screenedAccessGrant], + }) + .create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + await accessRequestFactory.create({ + datasetId: screenedDataset.id, + accessGrantId: screenedAccessGrant.id, + requestorId: requestingUser.id, + approvedAt: new Date(), + revokedAt: new Date(), + }) + // inaccessible Dataset - for control case + await datasetFactory + .associations({ + accessGrants: [], + }) + .create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + + // Act + const query = datasetsWithApprovedAccessRequestsFor(requestingUser) + const scope = Dataset.scope({ where: { id: { [Op.in]: query } } }) + const result = await scope.findAll() + + // Assert + expect(await Dataset.count()).toBe(2) + expect(result).toEqual([]) + }) }) }) diff --git a/api/tests/models/datasets/datasets-with-preview-disabled.test.ts b/api/tests/models/datasets/datasets-with-preview-disabled.test.ts new file mode 100644 index 00000000..6c5cf6d2 --- /dev/null +++ b/api/tests/models/datasets/datasets-with-preview-disabled.test.ts @@ -0,0 +1,59 @@ +import { Op } from "sequelize" + +import { Dataset } from "@/models" +import { datasetsWithPreviewDisabled } from "@/models/datasets" +import { datasetFactory, userFactory, visualizationControlFactory } from "@/factories" + +describe("api/src/models/datasets/datasets-with-preview-disabled.ts", () => { + describe(".datasetsWithPreviewDisabled", () => { + test("when dataset visualization control has preview disabled, restricts the result", async () => { + // Arrange + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + await visualizationControlFactory.create({ + datasetId: dataset.id, + hasPreview: false, + }) + + // Act + const query = datasetsWithPreviewDisabled() + const result = await Dataset.findAll({ + where: { id: { [Op.notIn]: query } }, + }) + + expect(await Dataset.count()).toBe(1) + expect(result).toHaveLength(0) + expect(result).toEqual([]) + }) + + test("when dataset visualization control has preview enabled, shows the result", async () => { + // Arrange + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + await visualizationControlFactory.create({ + datasetId: dataset.id, + hasPreview: true, + }) + + // Act + const query = datasetsWithPreviewDisabled() + const result = await Dataset.findAll({ + where: { id: { [Op.notIn]: query } }, + }) + + expect(await Dataset.count()).toBe(1) + expect(result).toHaveLength(1) + expect(result).toEqual([ + expect.objectContaining({ + id: dataset.id, + }), + ]) + }) + }) +}) diff --git a/api/tests/policies/dataset-entries-policy.test.ts b/api/tests/policies/dataset-entries-policy.test.ts new file mode 100644 index 00000000..1db80934 --- /dev/null +++ b/api/tests/policies/dataset-entries-policy.test.ts @@ -0,0 +1,168 @@ +import { RoleTypes } from "@/models/role" +import { DatasetEntry } from "@/models" +import { DatasetEntriesPolicy } from "@/policies" +import { + accessGrantFactory, + accessRequestFactory, + datasetEntryFactory, + datasetFactory, + roleFactory, + userFactory, + userGroupFactory, + userGroupMembershipFactory, +} from "@/factories" +import { UserGroupTypes } from "@/models/user-groups" +import { AccessTypes, GrantLevels } from "@/models/access-grant" + +describe("api/src/policies/dataset-entries-policy.ts", () => { + describe("DatasetEntryPreviewsPolicy", () => { + describe(".applyScope", () => { + test.each([{ roleType: RoleTypes.SYSTEM_ADMIN }, { roleType: RoleTypes.BUSINESS_ANALYST }])( + "when user role is `$roleType`, it returns all records", + async ({ roleType }) => { + // Arrange + const role = roleFactory.build({ role: roleType }) + const requestingUser = await userFactory + .associations({ + roles: [role], + }) + .create() + const datasetOwner = await userFactory.create() + + const dataset = await datasetFactory.create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + const accessibleDatasetEntry = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { field1: "value1" }, + }) + const scopedQuery = DatasetEntriesPolicy.applyScope(DatasetEntry, requestingUser) + + // Act + const result = await scopedQuery.findAll() + + // Assert + expect(result).toEqual([ + expect.objectContaining({ + id: accessibleDatasetEntry.id, + }), + ]) + } + ) + + test("when viewer role is User, and entry belongs to a dataset with accessible, screened access grants, without an approved request, restricts dataset entries", async () => { + // Arrange + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const role = roleFactory.build({ role: RoleTypes.USER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [role], + groupMembership: viewerGroupMembership, + }) + .create() + const datasetOwner = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessGrant = accessGrantFactory.build({ + creatorId: datasetOwner.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const dataset = await datasetFactory + .associations({ + accessGrants: [accessGrant], + }) + .create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { field1: "value1" }, + }) + + // Act + const scopedDatasetEntries = DatasetEntriesPolicy.applyScope(DatasetEntry, viewer) + const results = await scopedDatasetEntries.findAll() + + // Assert + expect(results).toHaveLength(0) + }) + + test("when viewer role is User, and entry belongs to a dataset with accessible, screened access grants, with an approved request, shows dataset entries", async () => { + // Arrange + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const role = roleFactory.build({ role: RoleTypes.USER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [role], + groupMembership: viewerGroupMembership, + }) + .create() + const datasetOwner = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessGrant = accessGrantFactory.build({ + creatorId: datasetOwner.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const dataset = await datasetFactory + .associations({ + accessGrants: [accessGrant], + }) + .create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + await accessRequestFactory.create({ + datasetId: dataset.id, + accessGrantId: accessGrant.id, + requestorId: viewer.id, + approvedAt: new Date(), + }) + const datasetEntry = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { field1: "value1" }, + }) + + // Act + const scopedDatasetEntries = DatasetEntriesPolicy.applyScope(DatasetEntry, viewer) + const results = await scopedDatasetEntries.findAll() + + // Assert + expect(results).toHaveLength(1) + expect(results.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntry.id, + datasetId: dataset.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + ]) + }) + }) + }) +}) diff --git a/api/tests/policies/dataset-entry-previews-policy.test.ts b/api/tests/policies/dataset-entry-previews-policy.test.ts new file mode 100644 index 00000000..434664d9 --- /dev/null +++ b/api/tests/policies/dataset-entry-previews-policy.test.ts @@ -0,0 +1,557 @@ +import { AccessTypes, GrantLevels } from "@/models/access-grant" +import { RoleTypes } from "@/models/role" +import { UserGroupTypes } from "@/models/user-groups" +import { DatasetEntryPreview } from "@/models" +import { DatasetEntryPreviewsPolicy } from "@/policies" +import { + accessGrantFactory, + datasetEntryFactory, + datasetEntryPreviewFactory, + datasetFactory, + roleFactory, + userFactory, + userGroupFactory, + userGroupMembershipFactory, + visualizationControlFactory, +} from "@/factories" + +describe("api/src/policies/dataset-entry-previews-policy.ts", () => { + describe("DatasetEntryPreviewsPolicy", () => { + describe(".applyScope", () => { + test.each([{ roleType: RoleTypes.SYSTEM_ADMIN }, { roleType: RoleTypes.BUSINESS_ANALYST }])( + `when viewer role is "$roleType", it returns all records`, + async ({ roleType }) => { + // Arrange + const role = roleFactory.build({ role: roleType }) + const viewer = await userFactory + .associations({ + roles: [role], + }) + .create() + const datasetOwner = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { field1: "value1" }, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { field1: "value2" }, + }) + const datasetEntry3 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { field1: "value3" }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: dataset.id, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview2 = await datasetEntryPreviewFactory.create({ + datasetId: dataset.id, + datasetEntryId: datasetEntry2.id, + jsonData: { field1: "value2" }, + }) + const datasetEntryPreview3 = await datasetEntryPreviewFactory.create({ + datasetId: dataset.id, + datasetEntryId: datasetEntry3.id, + jsonData: { field1: "value3" }, + }) + + // Act + const scopedQuery = DatasetEntryPreviewsPolicy.applyScope([], viewer) + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetEntryPreview.count()).toBe(3) + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: dataset.id, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + expect.objectContaining({ + id: datasetEntryPreview2.id, + datasetId: dataset.id, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ field1: "value2" }), + }), + expect.objectContaining({ + id: datasetEntryPreview3.id, + datasetId: dataset.id, + datasetEntryId: datasetEntry3.id, + jsonData: JSON.stringify({ field1: "value3" }), + }), + ]) + } + ) + + test("when viewer is a data owner, and owns dataset being previewed, it shows the preview records", async () => { + // Arrange + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerUserGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const viewerRole = roleFactory.build({ role: RoleTypes.DATA_OWNER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [viewerRole], + groupMembership: viewerUserGroupMembership, + }) + .create() + const otherUser = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessibleDataset = await datasetFactory.create({ + creatorId: viewer.id, + ownerId: viewer.id, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: accessibleDataset.id, + jsonData: { field1: "value1" }, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: accessibleDataset.id, + jsonData: { field1: "value2" }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset.id, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview2 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset.id, + datasetEntryId: datasetEntry2.id, + jsonData: { field1: "value2" }, + }) + const inaccessibleDataset = await datasetFactory.create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const inaccessibleDatasetEntry = await datasetEntryFactory.create({ + datasetId: inaccessibleDataset.id, + jsonData: { field1: "value3" }, + }) + // inaccessible dataset entry preview for control case + await datasetEntryPreviewFactory.create({ + datasetId: inaccessibleDataset.id, + datasetEntryId: inaccessibleDatasetEntry.id, + jsonData: { field1: "value3" }, + }) + + // Act + const scopedQuery = DatasetEntryPreviewsPolicy.applyScope([], viewer) + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetEntryPreview.count()).toBe(3) + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: accessibleDataset.id, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + expect.objectContaining({ + id: datasetEntryPreview2.id, + datasetId: accessibleDataset.id, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ field1: "value2" }), + }), + ]) + }) + + test.each([ + { + accessType: AccessTypes.OPEN_ACCESS, + }, + { + accessType: AccessTypes.SELF_SERVE_ACCESS, + }, + { + accessType: AccessTypes.SCREENED_ACCESS, + }, + ])( + `when viewer is a data owner, and dataset is accessible via "$accessType" grant, it shows the preview records`, + async ({ accessType }) => { + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerUserGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const viewerRole = roleFactory.build({ role: RoleTypes.DATA_OWNER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [viewerRole], + groupMembership: viewerUserGroupMembership, + }) + .create() + const otherUser = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessibleDataset1 = await datasetFactory.create({ + creatorId: viewer.id, + ownerId: viewer.id, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: accessibleDataset1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const accessGrant = accessGrantFactory.build({ + creatorId: otherUser.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType, + }) + const accessibleDataset2 = await datasetFactory + .associations({ + accessGrants: [accessGrant], + }) + .create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: accessibleDataset2.id, + jsonData: { field1: "value2" }, + }) + const datasetEntryPreview2 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset2.id, + datasetEntryId: datasetEntry2.id, + jsonData: { field1: "value2" }, + }) + const inaccessibleDataset = await datasetFactory.create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const inaccessibleDatasetEntry = await datasetEntryFactory.create({ + datasetId: inaccessibleDataset.id, + jsonData: { field1: "value3" }, + }) + // inaccessible dataset entry preview for control case + await datasetEntryPreviewFactory.create({ + datasetId: inaccessibleDataset.id, + datasetEntryId: inaccessibleDatasetEntry.id, + jsonData: { field1: "value3" }, + }) + + // Act + const scopedQuery = DatasetEntryPreviewsPolicy.applyScope([], viewer) + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetEntryPreview.count()).toBe(3) + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + expect.objectContaining({ + id: datasetEntryPreview2.id, + datasetId: accessibleDataset2.id, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ field1: "value2" }), + }), + ]) + } + ) + + test("when viewer is a data owner, and dataset is accessible via screened access grant, but dataset preivew is disabled, it restricts the preview records", async () => { + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerUserGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const viewerRole = roleFactory.build({ role: RoleTypes.DATA_OWNER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [viewerRole], + groupMembership: viewerUserGroupMembership, + }) + .create() + const otherUser = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessibleDataset1 = await datasetFactory.create({ + creatorId: viewer.id, + ownerId: viewer.id, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: accessibleDataset1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const accessGrant = accessGrantFactory.build({ + creatorId: otherUser.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const inaccessibleDataset1 = await datasetFactory + .associations({ + accessGrants: [accessGrant], + }) + .create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + await visualizationControlFactory.create({ + datasetId: inaccessibleDataset1.id, + hasPreview: false, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: inaccessibleDataset1.id, + jsonData: { field1: "value2" }, + }) + // inaccessible via preview mode disabled + await datasetEntryPreviewFactory.create({ + datasetId: inaccessibleDataset1.id, + datasetEntryId: datasetEntry2.id, + jsonData: { field1: "value2" }, + }) + const inaccessibleDataset2 = await datasetFactory.create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const inaccessibleDatasetEntry = await datasetEntryFactory.create({ + datasetId: inaccessibleDataset2.id, + jsonData: { field1: "value3" }, + }) + // inaccessible dataset entry preview for control case + await datasetEntryPreviewFactory.create({ + datasetId: inaccessibleDataset2.id, + datasetEntryId: inaccessibleDatasetEntry.id, + jsonData: { field1: "value3" }, + }) + + // Act + const scopedQuery = DatasetEntryPreviewsPolicy.applyScope([], viewer) + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetEntryPreview.count()).toBe(3) + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + ]) + }) + + test.each([ + { + accessType: AccessTypes.OPEN_ACCESS, + }, + { + accessType: AccessTypes.SELF_SERVE_ACCESS, + }, + { + accessType: AccessTypes.SCREENED_ACCESS, + }, + ])( + `when viewer is a user, and dataset is accessible via "$accessType" grant, it shows the preview records`, + async ({ accessType }) => { + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerUserGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const viewerRole = roleFactory.build({ role: RoleTypes.USER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [viewerRole], + groupMembership: viewerUserGroupMembership, + }) + .create() + const otherUser = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessGrant = accessGrantFactory.build({ + creatorId: otherUser.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType, + }) + const accessibleDataset1 = await datasetFactory + .associations({ + accessGrants: [accessGrant], + }) + .create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: accessibleDataset1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const inaccessibleDataset = await datasetFactory.create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const inaccessibleDatasetEntry = await datasetEntryFactory.create({ + datasetId: inaccessibleDataset.id, + jsonData: { field1: "value2" }, + }) + // inaccessible dataset entry preview for control case + await datasetEntryPreviewFactory.create({ + datasetId: inaccessibleDataset.id, + datasetEntryId: inaccessibleDatasetEntry.id, + jsonData: { field1: "value2" }, + }) + + // Act + const scopedQuery = DatasetEntryPreviewsPolicy.applyScope([], viewer) + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetEntryPreview.count()).toBe(2) + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + ]) + } + ) + + test("when viewer is a user, and dataset is accessible via screened access grant, but dataset preivew is disabled, it restricts the preview records", async () => { + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const viewerUserGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const viewerRole = roleFactory.build({ role: RoleTypes.USER }) + const viewer = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [viewerRole], + groupMembership: viewerUserGroupMembership, + }) + .create() + const otherUser = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + const accessGrant1 = accessGrantFactory.build({ + creatorId: otherUser.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const accessibleDataset1 = await datasetFactory + .associations({ + accessGrants: [accessGrant1], + }) + .create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: accessibleDataset1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const accessGrant2 = accessGrantFactory.build({ + creatorId: otherUser.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const inaccessibleDataset1 = await datasetFactory + .associations({ + accessGrants: [accessGrant2], + }) + .create({ + creatorId: otherUser.id, + ownerId: otherUser.id, + }) + await visualizationControlFactory.create({ + datasetId: inaccessibleDataset1.id, + hasPreview: false, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: inaccessibleDataset1.id, + jsonData: { field1: "value2" }, + }) + // inaccessible via preview mode disabled + await datasetEntryPreviewFactory.create({ + datasetId: inaccessibleDataset1.id, + datasetEntryId: datasetEntry2.id, + jsonData: { field1: "value2" }, + }) + + // Act + const scopedQuery = DatasetEntryPreviewsPolicy.applyScope([], viewer) + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetEntryPreview.count()).toBe(2) + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: accessibleDataset1.id, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + ]) + }) + }) + }) +}) diff --git a/api/tests/policies/dataset-fields-policy.test.ts b/api/tests/policies/dataset-fields-policy.test.ts index cd90adf7..1779ed08 100644 --- a/api/tests/policies/dataset-fields-policy.test.ts +++ b/api/tests/policies/dataset-fields-policy.test.ts @@ -12,6 +12,7 @@ import { userFactory, userGroupFactory, userGroupMembershipFactory, + visualizationControlFactory, } from "@/factories" import { AccessTypes, GrantLevels } from "@/models/access-grant" @@ -211,9 +212,14 @@ describe("api/src/policies/dataset-fields-policy.ts", () => { ownerId: datasetOwner.id, }) + const accessibleDatasetField = await datasetFieldFactory.create({ + datasetId: screenedDataset.id, + isExcludedFromPreview: false, + }) // inaccessible dataset field - via screened access await datasetFieldFactory.create({ datasetId: screenedDataset.id, + isExcludedFromPreview: true, }) // inaccessible dataset field - for control case await datasetFieldFactory.create({ @@ -225,8 +231,14 @@ describe("api/src/policies/dataset-fields-policy.ts", () => { const result = await scopedQuery.findAll() // Assert - expect(DatasetField.count()).resolves.toBe(2) - expect(result).toHaveLength(0) + expect(DatasetField.count()).resolves.toBe(3) + expect(result).toHaveLength(1) + expect(result).toEqual([ + expect.objectContaining({ + id: accessibleDatasetField.id, + isExcludedFromPreview: false, + }), + ]) }) test("when user has role type user, and field belongs to dataset with accessible screened access grants, with an approved request, returns the datasets fields", async () => { @@ -438,6 +450,71 @@ describe("api/src/policies/dataset-fields-policy.ts", () => { }), ]) }) + + test("when user has role type user, and field belongs to dataset with limited access and preview mode disabled, restricts all datasets fields", async () => { + // Arrange + const department = await userGroupFactory.create({ type: UserGroupTypes.DEPARTMENT }) + const requestingUserGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const datasetOwnerGroupMembership = userGroupMembershipFactory.build({ + departmentId: department.id, + }) + const role = roleFactory.build({ role: RoleTypes.USER }) + const requestingUser = await userFactory + .transient({ + include: ["groupMembership"], + }) + .associations({ + roles: [role], + groupMembership: requestingUserGroupMembership, + }) + .create() + + const datasetOwner = await userFactory + .associations({ + groupMembership: datasetOwnerGroupMembership, + }) + .create() + + const accessGrant = accessGrantFactory.build({ + creatorId: datasetOwner.id, + grantLevel: GrantLevels.GOVERNMENT_WIDE, + accessType: AccessTypes.SCREENED_ACCESS, + }) + const dataset = await datasetFactory + .associations({ + accessGrants: [accessGrant], + }) + .create({ + creatorId: datasetOwner.id, + ownerId: datasetOwner.id, + }) + await visualizationControlFactory.create({ + datasetId: dataset.id, + hasPreview: false, + }) + + // inaccessible via preview mode disabled + await datasetFieldFactory.create({ + datasetId: dataset.id, + isExcludedFromPreview: false, + }) + // inaccessible via screened access + await datasetFieldFactory.create({ + datasetId: dataset.id, + isExcludedFromPreview: true, + }) + const scopedQuery = DatasetFieldsPolicy.applyScope([], requestingUser) + + // Act + const result = await scopedQuery.findAll() + + // Assert + expect(await DatasetField.count()).toBe(2) + expect(result).toHaveLength(0) + expect(result).toEqual([]) + }) }) }) }) diff --git a/api/tests/services/visualization-controls/refresh-dataset-entry-preview-service.test.ts b/api/tests/services/visualization-controls/refresh-dataset-entry-preview-service.test.ts new file mode 100644 index 00000000..1c6e2482 --- /dev/null +++ b/api/tests/services/visualization-controls/refresh-dataset-entry-preview-service.test.ts @@ -0,0 +1,346 @@ +import { faker } from "@faker-js/faker" +import { Op } from "sequelize" + +import { DatasetEntryPreview } from "@/models" +import { RefreshDatasetEntryPreviewService } from "@/services/visualization-controls" +import { + datasetEntryFactory, + datasetEntryPreviewFactory, + datasetFactory, + datasetFieldFactory, + userFactory, + visualizationControlFactory, +} from "@/factories" + +describe("api/src/services/visualization-controls/refresh-dataset-entry-preview-service.ts", () => { + describe("RefreshDatasetEntryPreviewService", () => { + describe("#perform", () => { + test("when dataset entry preview is refreshed, it includes all fields that are not excluded from preview", async () => { + // Arrange + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + const visualizationControl = await visualizationControlFactory.create({ + datasetId: dataset.id, + hasFieldsExcludedFromPreview: true, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field1", + isExcludedFromPreview: false, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field2", + isExcludedFromPreview: false, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field3", + isExcludedFromPreview: true, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value1", + field2: "value2", + field3: "value3", + }, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value4", + field2: "value5", + field3: "value6", + }, + }) + const datasetEntry3 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value7", + field2: "value8", + field3: "value9", + }, + }) + + // Act + const result = await RefreshDatasetEntryPreviewService.perform(visualizationControl, user) + + // Assert + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ + field1: "value1", + field2: "value2", + }), + }), + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ + field1: "value4", + field2: "value5", + }), + }), + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry3.id, + jsonData: JSON.stringify({ + field1: "value7", + field2: "value8", + }), + }), + ]) + }) + + test("when all fields that are excluded from preview, it returns an empty array", async () => { + // Arrange + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + const visualizationControl = await visualizationControlFactory.create({ + datasetId: dataset.id, + hasFieldsExcludedFromPreview: true, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field1", + isExcludedFromPreview: true, + }) + await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value1", + }, + }) + + // Act + const result = await RefreshDatasetEntryPreviewService.perform(visualizationControl, user) + + // Assert + expect(result).toEqual([]) + }) + + test("when dataset entry previews exist, they are deleted before creating new ones", async () => { + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + const visualizationControl = await visualizationControlFactory.create({ + datasetId: dataset.id, + hasFieldsExcludedFromPreview: true, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field1", + isExcludedFromPreview: false, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field2", + isExcludedFromPreview: false, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value1", + field2: "value2", + field3: "value3", + }, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value4", + field2: "value5", + field3: "value6", + }, + }) + const datasetEntryPreview1 = await datasetEntryPreviewFactory.create({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry1.id, + jsonData: { field1: "value1" }, + }) + const datasetEntryPreview2 = await datasetEntryPreviewFactory.create({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry2.id, + jsonData: { field1: "value4" }, + }) + + // Act + await RefreshDatasetEntryPreviewService.perform(visualizationControl, user) + + // Assert + const deletedDatasetEntryPreviews = await DatasetEntryPreview.findAll({ + where: { deletedAt: { [Op.not]: null } }, + paranoid: false, + }) + expect(deletedDatasetEntryPreviews.length).toEqual(2) + expect(deletedDatasetEntryPreviews.map((m) => m.dataValues)).toEqual([ + expect.objectContaining({ + id: datasetEntryPreview1.id, + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ field1: "value1" }), + }), + expect.objectContaining({ + id: datasetEntryPreview2.id, + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ field1: "value4" }), + }), + ]) + const datasetEntryPreviews = await DatasetEntryPreview.findAll() + expect(datasetEntryPreviews.length).toBe(2) + expect(datasetEntryPreviews.map((m) => m.dataValues)).toEqual([ + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ + field1: "value1", + field2: "value2", + }), + }), + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ + field1: "value4", + field2: "value5", + }), + }), + ]) + }) + + test("when visualization control preview row limit is enabled, it limits preview results", async () => { + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + const visualizationControl = await visualizationControlFactory.create({ + datasetId: dataset.id, + hasPreviewRowLimit: true, + previewRowLimit: 10, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field1", + isExcludedFromPreview: false, + }) + await datasetEntryFactory.createList(12, { + datasetId: dataset.id, + jsonData: { + field1: faker.lorem.word(), + }, + }) + + // Act + const result = await RefreshDatasetEntryPreviewService.perform(visualizationControl, user) + + // Assert + expect(result.length).toBe(10) + }) + + test("when visualization control has field exclusions disabled, it includes all fields in preview", async () => { + // Arrange + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + const visualizationControl = await visualizationControlFactory.create({ + datasetId: dataset.id, + hasFieldsExcludedFromPreview: false, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field1", + isExcludedFromPreview: true, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field2", + isExcludedFromPreview: false, + }) + const datasetEntry1 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value1", + field2: "value2", + field3: "value3", + }, + }) + const datasetEntry2 = await datasetEntryFactory.create({ + datasetId: dataset.id, + jsonData: { + field1: "value4", + field2: "value5", + field3: "value6", + }, + }) + + // Act + const result = await RefreshDatasetEntryPreviewService.perform(visualizationControl, user) + + // Assert + expect(result.map((r) => r.dataValues)).toEqual([ + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry1.id, + jsonData: JSON.stringify({ + field1: "value1", + field2: "value2", + }), + }), + expect.objectContaining({ + datasetId: visualizationControl.datasetId, + datasetEntryId: datasetEntry2.id, + jsonData: JSON.stringify({ + field1: "value4", + field2: "value5", + }), + }), + ]) + }) + + test("when visualization control preview row limit is disabled, it includes all entries in preview", async () => { + const user = await userFactory.create() + const dataset = await datasetFactory.create({ + creatorId: user.id, + ownerId: user.id, + }) + const visualizationControl = await visualizationControlFactory.create({ + datasetId: dataset.id, + hasPreviewRowLimit: false, + previewRowLimit: 10, + }) + await datasetFieldFactory.create({ + datasetId: dataset.id, + name: "field1", + isExcludedFromPreview: false, + }) + await datasetEntryFactory.createList(12, { + datasetId: dataset.id, + jsonData: { + field1: faker.lorem.word(), + }, + }) + + // Act + const result = await RefreshDatasetEntryPreviewService.perform(visualizationControl, user) + + // Assert + expect(result.length).toBe(12) + }) + }) + }) +}) diff --git a/web/src/api/dataset-entry-previews-api.ts b/web/src/api/dataset-entry-previews-api.ts new file mode 100644 index 00000000..49a67256 --- /dev/null +++ b/web/src/api/dataset-entry-previews-api.ts @@ -0,0 +1,38 @@ +import http from "@/api/http-client" + +import { Dataset } from "@/api/datasets-api" +import { DatasetEntry, DatasetEntryJsonDataType } from "@/api/dataset-entries-api" + +export type DatasetEntryPreview = { + id: number + datasetId: Dataset["id"] + datasetEntryId: DatasetEntry["id"] + jsonData: DatasetEntryJsonDataType + createdAt: string + updatedAt: string +} + +export type DatasetEntryPreviewFilters = { + search?: string +} + +export const datasetEntryPreviewsApi = { + async list( + params: { + where?: Record // TODO: consider adding Sequelize types to front-end? + filters?: DatasetEntryPreviewFilters + page?: number + perPage?: number + } = {} + ): Promise<{ + datasetEntryPreviews: DatasetEntryPreview[] + totalCount: number + }> { + const { data } = await http.get("/api/dataset-entry-previews", { + params, + }) + return data + }, +} + +export default datasetEntryPreviewsApi diff --git a/web/src/api/dataset-fields-api.ts b/web/src/api/dataset-fields-api.ts index 88601ebb..0bcd3c9e 100644 --- a/web/src/api/dataset-fields-api.ts +++ b/web/src/api/dataset-fields-api.ts @@ -16,7 +16,7 @@ export type DatasetField = { dataType: DatasetFieldDataTypes description: string note: string - isExcludedFromSearch: boolean + isExcludedFromPreview: boolean createdAt: string updatedAt: string } diff --git a/web/src/api/datasets-api.ts b/web/src/api/datasets-api.ts index 948cb761..f99ed33a 100644 --- a/web/src/api/datasets-api.ts +++ b/web/src/api/datasets-api.ts @@ -10,8 +10,6 @@ import { type User } from "@/api/users-api" import { type UserGroup } from "@/api/user-groups-api" import { type VisualizationControl } from "@/api/visualization-controls-api" -export { type Policy } - export enum DatasetErrorTypes { OK = "ok", ERRORED = "errored", @@ -66,6 +64,10 @@ export type DatasetsFilters = { withTagNames?: string[] } +export type DatasetPolicy = Policy & { + showUnlimited: boolean +} + export const datasetsApi = { DatasetErrorTypes, async list(params: { @@ -82,7 +84,7 @@ export const datasetsApi = { }, async get(idOrSlug: number | string): Promise<{ dataset: DatasetDetailedResult - policy: Policy + policy: DatasetPolicy }> { const { data } = await http.get(`/api/datasets/${idOrSlug}`) return data diff --git a/web/src/api/visualization-controls-api.ts b/web/src/api/visualization-controls-api.ts index 325a1790..2ebdfd1f 100644 --- a/web/src/api/visualization-controls-api.ts +++ b/web/src/api/visualization-controls-api.ts @@ -7,21 +7,21 @@ export type VisualizationControl = { id: number datasetId: Dataset["id"] isDownloadableAsCsv: boolean - hasSearchCustomizations: boolean - hasFieldsExcludedFromSearch: boolean - hasSearchRowLimits: boolean - searchRowLimitMaximum: number | null + hasPreview: boolean + hasFieldsExcludedFromPreview: boolean + hasPreviewRowLimit: boolean + previewRowLimit: number createdAt: string updatedAt: string // Associations - searchExcludedDatasetFields: DatasetField[] + previewExcludedDatasetFields: DatasetField[] } -export type searchExcludedDatasetFieldsAttributes = Pick +export type previewExcludedDatasetFieldsAttributes = Pick export type VisualizationControlUpdate = Partial & { - searchExcludedDatasetFieldsAttributes?: searchExcludedDatasetFieldsAttributes[] + previewExcludedDatasetFieldsAttributes?: previewExcludedDatasetFieldsAttributes[] } export const visualizationControlsApi = { diff --git a/web/src/components/dataset-entry-previews/DatasetEntryPreviewsTable.vue b/web/src/components/dataset-entry-previews/DatasetEntryPreviewsTable.vue new file mode 100644 index 00000000..49d20c97 --- /dev/null +++ b/web/src/components/dataset-entry-previews/DatasetEntryPreviewsTable.vue @@ -0,0 +1,166 @@ + + + diff --git a/web/src/components/dataset-entry-previews/SwitchableDatasetEntryPreviewsTable.vue b/web/src/components/dataset-entry-previews/SwitchableDatasetEntryPreviewsTable.vue new file mode 100644 index 00000000..c6ce6675 --- /dev/null +++ b/web/src/components/dataset-entry-previews/SwitchableDatasetEntryPreviewsTable.vue @@ -0,0 +1,83 @@ + + + diff --git a/web/src/components/dataset-layout/AccessTab.vue b/web/src/components/dataset-layout/AccessTab.vue index e7eaa782..506159f9 100644 --- a/web/src/components/dataset-layout/AccessTab.vue +++ b/web/src/components/dataset-layout/AccessTab.vue @@ -1,20 +1,17 @@ diff --git a/web/src/components/dataset-layout/DescriptionTab.vue b/web/src/components/dataset-layout/DescriptionTab.vue index bea4decf..1d756d03 100644 --- a/web/src/components/dataset-layout/DescriptionTab.vue +++ b/web/src/components/dataset-layout/DescriptionTab.vue @@ -4,27 +4,21 @@ diff --git a/web/src/components/dataset-layout/FieldsTab.vue b/web/src/components/dataset-layout/FieldsTab.vue index 707f1d42..6b12379c 100644 --- a/web/src/components/dataset-layout/FieldsTab.vue +++ b/web/src/components/dataset-layout/FieldsTab.vue @@ -1,41 +1,24 @@ diff --git a/web/src/components/dataset-layout/VisualizeTab.vue b/web/src/components/dataset-layout/VisualizeTab.vue index 62d8bad9..eaa2f1ee 100644 --- a/web/src/components/dataset-layout/VisualizeTab.vue +++ b/web/src/components/dataset-layout/VisualizeTab.vue @@ -1,41 +1,24 @@ diff --git a/web/src/components/visualization-controls/VisualizePropertiesFormCard.vue b/web/src/components/visualization-controls/VisualizePropertiesFormCard.vue index 1a324507..e7797b46 100644 --- a/web/src/components/visualization-controls/VisualizePropertiesFormCard.vue +++ b/web/src/components/visualization-controls/VisualizePropertiesFormCard.vue @@ -15,6 +15,7 @@ /> @@ -35,8 +36,8 @@ class="py-0" > @@ -53,8 +54,8 @@ class="py-0" > @@ -64,12 +65,12 @@ class="py-0" > @@ -85,8 +86,8 @@ class="py-0" > @@ -96,11 +97,12 @@ class="py-0" > @@ -112,9 +114,10 @@ diff --git a/web/src/pages/DatasetVisualizeManagePage.vue b/web/src/pages/DatasetVisualizeManagePage.vue index 16b6215c..6581d014 100644 --- a/web/src/pages/DatasetVisualizeManagePage.vue +++ b/web/src/pages/DatasetVisualizeManagePage.vue @@ -14,9 +14,9 @@ v-if="isNil(dataset)" type="table" /> - @@ -29,7 +29,7 @@ import { isNil } from "lodash" import { useBreadcrumbs } from "@/use/use-breadcrumbs" import { useDataset } from "@/use/use-dataset" -import DatasetEntriesTable from "@/components/dataset-entries/DatasetEntriesTable.vue" +import SwitchableDatasetEntryPreviewsTable from "@/components/dataset-entry-previews/SwitchableDatasetEntryPreviewsTable.vue" import VisualizePropertiesFormCard from "@/components/visualization-controls/VisualizePropertiesFormCard.vue" const props = defineProps({ @@ -42,14 +42,16 @@ const props = defineProps({ const { slug } = toRefs(props) const { dataset } = useDataset(slug) -const datasetEntriesTable = ref | null>(null) +const switchableDatasetEntryPreviewsTable = ref | null>(null) function refreshTable() { - if (isNil(datasetEntriesTable.value)) { + if (isNil(switchableDatasetEntryPreviewsTable.value)) { return } - datasetEntriesTable.value.refresh() + switchableDatasetEntryPreviewsTable.value.refresh() } const { setBreadcrumbs } = useBreadcrumbs() diff --git a/web/src/pages/DatasetVisualizeReadPage.vue b/web/src/pages/DatasetVisualizeReadPage.vue index 240e3ccc..e0a62bb6 100644 --- a/web/src/pages/DatasetVisualizeReadPage.vue +++ b/web/src/pages/DatasetVisualizeReadPage.vue @@ -21,26 +21,40 @@ - + - +