feat(sync): add blob engine

packages/blocks/src/__tests__/adapters/html.unit.spec.ts

@@ -1,5 +1,5 @@
 import type { BlockSnapshot } from '@blocksuite/store';
-import { MemoryBlobManager } from '@blocksuite/store';
+import { MemoryBlobCRUD } from '@blocksuite/store';
 import { AssetsManager } from '@blocksuite/store';
 import { describe, expect, test } from 'vitest';
 
@@ -946,10 +946,10 @@ describe('snapshot to html', () => {
  );
 
  const htmlAdapter = new HtmlAdapter();
- const blobManager = new MemoryBlobManager();
+ const blobManager = new MemoryBlobCRUD();
  await blobManager.set(
- new Blob(),
- 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs='
+ 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=',
+ new Blob()
  );
  const assets = new AssetsManager({ blob: blobManager });
 

packages/blocks/src/__tests__/adapters/markdown.unit.spec.ts

@@ -1,5 +1,5 @@
 import type { BlockSnapshot, SliceSnapshot } from '@blocksuite/store';
-import { AssetsManager, MemoryBlobManager } from '@blocksuite/store';
+import { AssetsManager, MemoryBlobCRUD } from '@blocksuite/store';
 import { describe, expect, test } from 'vitest';
 
 import { MarkdownAdapter } from '../../_common/adapters/markdown.js';
@@ -1140,12 +1140,12 @@ hhh
  '![](assets/YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=.blob "aaa")\n\n';
 
  const mdAdapter = new MarkdownAdapter();
- const blobManager = new MemoryBlobManager();
- await blobManager.set(
- new Blob(),
- 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs='
+ const blobCRUD = new MemoryBlobCRUD();
+ await blobCRUD.set(
+ 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=',
+ new Blob()
  );
- const assets = new AssetsManager({ blob: blobManager });
+ const assets = new AssetsManager({ blob: blobCRUD });
 
  const target = await mdAdapter.fromBlockSnapshot({
  snapshot: blockSnapshot,

packages/blocks/src/__tests__/adapters/notion-html.unit.spec.ts

@@ -1,7 +1,7 @@
 import {
  AssetsManager,
  type BlockSnapshot,
- MemoryBlobManager,
+ MemoryBlobCRUD,
 } from '@blocksuite/store';
 import { describe, expect, test } from 'vitest';
 
@@ -1084,7 +1084,7 @@ describe('notion html to snapshot', () => {
  const adapter = new NotionHtmlAdapter();
  const rawBlockSnapshot = await adapter.toBlockSnapshot({
  file: html,
- assets: new AssetsManager({ blob: new MemoryBlobManager() }),
+ assets: new AssetsManager({ blob: new MemoryBlobCRUD() }),
  });
  expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot);
  });
@@ -1176,9 +1176,9 @@ describe('notion html to snapshot', () => {
  };
 
  const adapter = new NotionHtmlAdapter();
- const blobManager = new MemoryBlobManager();
- const key = await blobManager.set(new File([], 'README.pdf'));
- const assestsManager = new AssetsManager({ blob: blobManager });
+ const blobCRUD = new MemoryBlobCRUD();
+ const key = await blobCRUD.set(new File([], 'README.pdf'));
+ const assestsManager = new AssetsManager({ blob: blobCRUD });
  await assestsManager.readFromBlob(key);
  const rawBlockSnapshot = await adapter.toBlockSnapshot({
  file: html,

packages/blocks/src/_common/adapters/attachment.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { AssetsManager } from '@blocksuite/store';
 import {
  BaseAdapter,
@@ -10,7 +11,6 @@ import {
  type FromSliceSnapshotPayload,
  type FromSliceSnapshotResult,
  nanoid,
- sha,
  type SliceSnapshot,
  type ToBlockSnapshotPayload,
  type ToDocSnapshotPayload,

packages/blocks/src/_common/adapters/html.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { DeltaInsert } from '@blocksuite/inline';
 import type {
  FromBlockSnapshotPayload,
@@ -14,7 +15,6 @@ import {
  BlockSnapshotSchema,
  getAssetName,
  nanoid,
- sha,
 } from '@blocksuite/store';
 import { ASTWalker, BaseAdapter } from '@blocksuite/store';
 import {

packages/blocks/src/_common/adapters/image.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { AssetsManager } from '@blocksuite/store';
 import {
  BaseAdapter,
@@ -10,7 +11,6 @@ import {
  type FromSliceSnapshotPayload,
  type FromSliceSnapshotResult,
  nanoid,
- sha,
  type SliceSnapshot,
  type ToBlockSnapshotPayload,
  type ToDocSnapshotPayload,

packages/blocks/src/_common/adapters/markdown.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { DeltaInsert } from '@blocksuite/inline/types';
 import type {
  FromBlockSnapshotPayload,
@@ -18,7 +19,6 @@ import {
  type DocSnapshot,
  getAssetName,
  nanoid,
- sha,
  type SliceSnapshot,
 } from '@blocksuite/store';
 import { format } from 'date-fns/format';

packages/blocks/src/_common/adapters/notion-html.ts

@@ -1,4 +1,4 @@
-import { isEqual } from '@blocksuite/global/utils';
+import { isEqual, sha } from '@blocksuite/global/utils';
 import type { DeltaInsert } from '@blocksuite/inline';
 import type {
  FromBlockSnapshotPayload,
@@ -16,7 +16,6 @@ import {
  type DocSnapshot,
  getAssetName,
  nanoid,
- sha,
  type SliceSnapshot,
 } from '@blocksuite/store';
 import rehypeParse from 'rehype-parse';

packages/blocks/src/_common/export-manager/export-manager.ts

@@ -242,7 +242,7 @@ export class ExportManager {
  if (matchFlavours(block, ['affine:image'])) {
  if (!block.sourceId) return;
 
- const blob = await block.doc.blob.get(block.sourceId);
+ const blob = await block.doc.blobSync.get(block.sourceId);
  if (!blob) return;
 
  const blobToImage = (blob: Blob) =>

packages/blocks/src/_common/transformers/zip.ts

@@ -1,12 +1,12 @@
-import { assertExists } from '@blocksuite/global/utils';
+import { assertExists, sha } from '@blocksuite/global/utils';
 import type {
  CollectionInfoSnapshot,
  Doc,
  DocCollection,
  DocSnapshot,
  JobMiddleware,
 } from '@blocksuite/store';
-import { extMimeMap, getAssetName, Job, sha } from '@blocksuite/store';
+import { extMimeMap, getAssetName, Job } from '@blocksuite/store';
 import JSZip from 'jszip';
 
 import { replaceIdMiddleware, titleMiddleware } from './middlewares.js';

packages/blocks/src/_common/utils/filesys.ts

@@ -1,5 +1,3 @@
-import type { BlobManager } from '@blocksuite/store';
-
 // Polyfill for `showOpenFilePicker` API
 // See https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/wicg-file-system-access/index.d.ts
 // See also https://caniuse.com/?search=showOpenFilePicker
@@ -223,29 +221,6 @@ export async function getImageFilesFromLocal() {
  return imageFiles;
 }
 
-export async function uploadImageFromLocal(storage: BlobManager) {
- const imageFiles = await openFileOrFiles({
- acceptType: 'Images',
- multiple: true,
- });
- if (!imageFiles) return [];
- return loadImages(imageFiles, storage);
-}
-
-export async function loadImages(images: File[], storage: BlobManager) {
- const res: { file: File; sourceId: string }[] = [];
- for (let i = 0; i < images.length; i++) {
- const file = images[i];
- const sourceId = await storage.set(file);
- res.push({ file, sourceId });
- }
- const { saveAttachmentData } = withTempBlobData();
- res.forEach(({ file, sourceId }) => {
- saveAttachmentData(sourceId, { name: file.name });
- });
- return res;
-}
-
 export function downloadBlob(blob: Blob, name: string) {
  const dataURL = URL.createObjectURL(blob);
  const tmpLink = document.createElement('a');

packages/blocks/src/_common/utils/render-linked-doc.ts

@@ -323,7 +323,7 @@ async function renderImageAbstract(
  const sourceId = (image as ImageBlockModel).sourceId;
  if (!sourceId) return;
 
- const storage = card.linkedDoc?.blob;
+ const storage = card.linkedDoc?.blobSync;
  if (!storage) return;
 
  const blob = await storage.get(sourceId);

packages/blocks/src/attachment-block/utils.ts

@@ -52,7 +52,7 @@ async function uploadAttachmentBlob(
 
  try {
  setAttachmentUploading(blockId);
- sourceId = await doc.blob.set(blob);
+ sourceId = await doc.blobSync.set(blob);
  } catch (error) {
  console.error(error);
  if (error instanceof Error) {
@@ -84,7 +84,7 @@ async function getAttachmentBlob(model: AttachmentBlockModel) {
  }
 
  const doc = model.doc;
- let blob = await doc.blob.get(sourceId);
+ let blob = await doc.blobSync.get(sourceId);
 
  if (blob) {
  blob = new Blob([blob], { type: model.type });

packages/blocks/src/image-block/utils.ts

@@ -39,7 +39,7 @@ export async function uploadBlobForImage(
 
  try {
  setImageUploaded(blockId);
- sourceId = await doc.blob.set(blob);
+ sourceId = await doc.blobSync.set(blob);
  } catch (error) {
  console.error(error);
  if (error instanceof Error) {
@@ -69,7 +69,7 @@ async function getImageBlob(model: ImageBlockModel) {
  }
 
  const doc = model.doc;
- const blob = await doc.blob.get(sourceId);
+ const blob = await doc.blobSync.get(sourceId);
 
  if (!blob) {
  return null;
@@ -123,7 +123,7 @@ export async function fetchImageBlob(block: ImageBlockComponent) {
  throw new Error('Image sourceId is missing!');
  }
 
- const blob = await doc.blob.get(sourceId);
+ const blob = await doc.blobSync.get(sourceId);
  if (!blob) {
  throw new Error('Image blob is missing!');
  }

packages/blocks/src/root-block/edgeless/edgeless-root-block.ts

@@ -417,7 +417,7 @@ export class EdgelessRootBlockComponent extends BlockElement<
  const uploadPromises = imageFiles.map(async (file, index) => {
  const { point, blockId } = dropInfos[index];
 
- const sourceId = await this.doc.blob.set(file);
+ const sourceId = await this.doc.blobSync.set(file);
  const imageSize = await readImageSize(file);
 
  const center = Vec.toVec(point);
@@ -502,7 +502,7 @@ export class EdgelessRootBlockComponent extends BlockElement<
  let sourceId: string | undefined;
  try {
  setAttachmentUploading(blockId);
- sourceId = await this.doc.blob.set(file);
+ sourceId = await this.doc.blobSync.set(file);
  } catch (error) {
  console.error(error);
  if (error instanceof Error) {

packages/blocks/src/root-block/widgets/linked-doc/import-doc/import-doc.ts

@@ -1,7 +1,8 @@
 import '../../../../_common/components/loader.js';
 
 import { WithDisposable } from '@blocksuite/block-std';
-import { type DocCollection, extMimeMap, sha } from '@blocksuite/store';
+import { sha } from '@blocksuite/global/utils';
+import { type DocCollection, extMimeMap } from '@blocksuite/store';
 import { Job } from '@blocksuite/store';
 import JSZip from 'jszip';
 import { html, LitElement, type PropertyValues } from 'lit';

packages/blocks/src/surface-block/mini-mindmap/mindmap-preview.ts

@@ -127,7 +127,6 @@ export class MiniMindmapPreview extends WithDisposable(LitElement) {
  id: 'MINI_MINDMAP_TEMPORARY',
  schema,
  idGenerator: Generator.NanoID,
- blobStorages: [],
  awarenessSources: [],
  };
 

packages/framework/global/package.json

@@ -49,6 +49,7 @@
  "!dist/__tests__"
  ],
  "dependencies": {
+ "lib0": "^0.2.93",
  "zod": "^3.23.8"
  }
 }

packages/framework/global/src/utils/index.ts

@@ -1,4 +1,5 @@
 export * from './assert.js';
+export * from './crypto.js';
 export * from './disposable.js';
 export * from './function.js';
 export * from './logger.js';

packages/framework/store/src/__tests__/transformer.unit.spec.ts

@@ -1,7 +1,7 @@
 import { expect, test } from 'vitest';
 import * as Y from 'yjs';
 
-import { MemoryBlobManager } from '../adapter/index.js';
+import { MemoryBlobCRUD } from '../adapter/index.js';
 import { Text } from '../reactive/index.js';
 import {
  type BlockModel,
@@ -51,8 +51,8 @@ function createTestOptions() {
 }
 
 const transformer = new BaseBlockTransformer();
-const blobManager = new MemoryBlobManager();
-const assets = new AssetsManager({ blob: blobManager });
+const blobCRUD = new MemoryBlobCRUD();
+const assets = new AssetsManager({ blob: blobCRUD });
 
 test('model to snapshot', () => {
  const options = createTestOptions();

packages/framework/store/src/adapter/assets.ts

@@ -1,19 +1,30 @@
-import { assertExists } from '@blocksuite/global/utils';
+import { assertExists, sha } from '@blocksuite/global/utils';
 
-import { sha } from '../persistence/blob/utils.js';
-
-export class MemoryBlobManager {
+/**
+ * @internal just for test
+ */
+export class MemoryBlobCRUD {
  private readonly _map = new Map<string, Blob>();
- private readonly _blobsRef = new Map<string, number>();
 
  get(key: string) {
  return this._map.get(key) ?? null;
  }
 
- async set(value: Blob, key?: string) {
- const _key = key || (await sha(await value.arrayBuffer()));
- this._map.set(_key, value);
- return _key;
+ async set(value: Blob): Promise<string>;
+ async set(key: string, value: Blob): Promise<string>;
+ async set(valueOrKey: string | Blob, _value?: Blob) {
+ const key =
+ typeof valueOrKey === 'string'
+ ? valueOrKey
+ : await sha(await valueOrKey.arrayBuffer());
+ const value = typeof valueOrKey === 'string' ? _value : valueOrKey;
+
+ if (!value) {
+ throw new Error('value is required');
+ }
+
+ this._map.set(key, value);
+ return key;
  }
 
  delete(key: string) {
@@ -23,27 +34,6 @@ export class MemoryBlobManager {
  list() {
  return Array.from(this._map.keys());
  }
-
- gc() {
- const blobs = this.list();
- blobs.forEach(blobId => {
- const ref = this._blobsRef.get(blobId);
- if (!ref || ref <= 0) {
- this.delete(blobId);
- this._blobsRef.delete(blobId);
- }
- });
- }
-
- increaseRef(blobId: string) {
- const ref = this._blobsRef.get(blobId) ?? 0;
- this._blobsRef.set(blobId, ref + 1);
- }
-
- decreaseRef(blobId: string) {
- const ref = this._blobsRef.get(blobId) ?? 0;
- this._blobsRef.set(blobId, Math.max(ref - 1, 0));
- }
 }
 
 export const mimeExtMap = new Map([