feat(doc-storage): init

.eslintrc.js

@@ -54,7 +54,7 @@ const allPackages = [
   'packages/common/debug',
   'packages/common/env',
   'packages/common/infra',
-  'packages/common/theme',
+  'packages/common/doc-storage',
   'tools/cli',
 ];
 

packages/backend/server/migrations/20241023065619_blobs/migration.sql

@@ -0,0 +1,14 @@
+-- CreateTable
+CREATE TABLE "blobs" (
+    "workspace_id" VARCHAR NOT NULL,
+    "key" VARCHAR NOT NULL,
+    "size" INTEGER NOT NULL,
+    "mime" VARCHAR NOT NULL,
+    "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "deleted_at" TIMESTAMPTZ(3),
+
+    CONSTRAINT "blobs_pkey" PRIMARY KEY ("workspace_id","key")
+);
+
+-- AddForeignKey
+ALTER TABLE "blobs" ADD CONSTRAINT "blobs_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/backend/server/schema.prisma

@@ -106,6 +106,7 @@ model Workspace {
   permissions     WorkspaceUserPermission[]
   pagePermissions WorkspacePageUserPermission[]
   features        WorkspaceFeature[]
+  blobs           Blob[]
 
   @@map("workspaces")
 }
@@ -335,7 +336,7 @@ model UserSubscription {
   // yearly/monthly/lifetime
   recurring            String    @db.VarChar(20)
   // onetime subscription or anything else
-  variant              String?    @db.VarChar(20)
+  variant              String?   @db.VarChar(20)
   // subscription.id, null for linefetime payment or one time payment subscription
   stripeSubscriptionId String?   @unique @map("stripe_subscription_id")
   // subscription.status, active/past_due/canceled/unpaid...
@@ -499,3 +500,20 @@ model RuntimeConfig {
   @@unique([module, key])
   @@map("app_runtime_settings")
 }
+
+// Blob table only exists for fast non-data queries.
+// like, total size of blobs in a workspace, or blob list for sync service.
+// it should only be a map of metadata of blobs stored anywhere else
+model Blob {
+  workspaceId String   @map("workspace_id") @db.VarChar
+  key         String   @db.VarChar
+  size        Int      @db.Integer
+  mime        String   @db.VarChar
+  createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
+  deletedAt   DateTime? @map("deleted_at") @db.Timestamptz(3)
+
+  workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
+
+  @@id([workspaceId, key])
+  @@map("blobs")
+}

packages/backend/server/src/core/doc/storage/doc.ts

@@ -1,8 +1,10 @@
 import {
   applyUpdate,
+  diffUpdate,
   Doc,
   encodeStateAsUpdate,
   encodeStateVector,
+  encodeStateVectorFromUpdate,
   mergeUpdates,
   UndoManager,
 } from 'yjs';
@@ -19,6 +21,12 @@ export interface DocRecord {
   editor?: string;
 }
 
+export interface DocDiff {
+  missing: Uint8Array;
+  state: Uint8Array;
+  timestamp: number;
+}
+
 export interface DocUpdate {
   bin: Uint8Array;
   timestamp: number;
@@ -96,6 +104,27 @@ export abstract class DocStorageAdapter extends Connection {
     return snapshot;
   }
 
+  async getDocDiff(
+    spaceId: string,
+    docId: string,
+    stateVector?: Uint8Array
+  ): Promise<DocDiff | null> {
+    const doc = await this.getDoc(spaceId, docId);
+
+    if (!doc) {
+      return null;
+    }
+
+    const missing = stateVector ? diffUpdate(doc.bin, stateVector) : doc.bin;
+    const state = encodeStateVectorFromUpdate(doc.bin);
+
+    return {
+      missing,
+      state,
+      timestamp: doc.timestamp,
+    };
+  }
+
   abstract pushDocUpdates(
     spaceId: string,
     docId: string,

packages/backend/server/src/core/doc/storage/index.ts

@@ -1,4 +1,6 @@
-// TODO(@forehalo): share with frontend
+// This is a totally copy of definitions in [@affine/doc-storage]
+// because currently importing cross workspace package from [@affine/server] is not yet supported
+// should be kept updated with the original definitions in [@affine/doc-storage]
 import type { BlobStorageAdapter } from './blob';
 import { Connection } from './connection';
 import type { DocStorageAdapter } from './doc';

packages/backend/server/src/core/quota/resolver.ts

@@ -11,6 +11,7 @@ import { CurrentUser } from '../auth/session';
 import { EarlyAccessType } from '../features';
 import { UserType } from '../user';
 import { QuotaService } from './service';
+import { QuotaManagementService } from './storage';
 
 registerEnumType(EarlyAccessType, {
   name: 'EarlyAccessType',
@@ -55,14 +56,34 @@ class UserQuotaType {
   humanReadable!: UserQuotaHumanReadableType;
 }
 
+@ObjectType('UserQuotaUsage')
+class UserQuotaUsageType {
+  @Field(() => SafeIntResolver, { name: 'storageQuota' })
+  storageQuota!: number;
+}
+
 @Resolver(() => UserType)
 export class QuotaManagementResolver {
-  constructor(private readonly quota: QuotaService) {}
+  constructor(
+    private readonly quota: QuotaService,
+    private readonly management: QuotaManagementService
+  ) {}
 
   @ResolveField(() => UserQuotaType, { name: 'quota', nullable: true })
   async getQuota(@CurrentUser() me: UserType) {
     const quota = await this.quota.getUserQuota(me.id);
 
     return quota.feature;
   }
+
+  @ResolveField(() => UserQuotaUsageType, { name: 'quotaUsage' })
+  async getQuotaUsage(
+    @CurrentUser() me: UserType
+  ): Promise<UserQuotaUsageType> {
+    const usage = await this.management.getUserStorageUsage(me.id);
+
+    return {
+      storageQuota: usage,
+    };
+  }
 }

packages/backend/server/src/core/quota/storage.ts

@@ -40,7 +40,7 @@ export class QuotaManagementService {
     };
   }
 
-  async getUserUsage(userId: string) {
+  async getUserStorageUsage(userId: string) {
     const workspaces = await this.permissions.getOwnedWorkspaces(userId);
 
     const sizes = await Promise.allSettled(
@@ -88,7 +88,7 @@ export class QuotaManagementService {
   async getQuotaCalculator(userId: string) {
     const quota = await this.getUserQuota(userId);
     const { storageQuota, businessBlobLimit } = quota;
-    const usedSize = await this.getUserUsage(userId);
+    const usedSize = await this.getUserStorageUsage(userId);
 
     return this.generateQuotaCalculator(
       storageQuota,
@@ -128,7 +128,7 @@ export class QuotaManagementService {
       },
     } = await this.quota.getUserQuota(owner.id);
     // get all workspaces size of owner used
-    const usedSize = await this.getUserUsage(owner.id);
+    const usedSize = await this.getUserStorageUsage(owner.id);
     // relax restrictions if workspace has unlimited feature
     // todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota
     const unlimited = await this.feature.hasWorkspaceFeature(