From 685c9fb9328fa0ede957d6c313361f28fc0035ce Mon Sep 17 00:00:00 2001 From: milanro <30476743+milanro@users.noreply.github.com> Date: Tue, 9 Aug 2022 17:55:06 +0200 Subject: [PATCH] Milan multiple compression (#11083) --- .../packages/property-dds/package.json | 1 + .../property-dds/src/propertyTreeExt.ts | 12 +- .../src/propertyTreeExtFactories.ts | 201 +++++++++++++----- .../src/test/propertyTree.spec.ts | 7 +- 4 files changed, 161 insertions(+), 60 deletions(-) diff --git a/experimental/PropertyDDS/packages/property-dds/package.json b/experimental/PropertyDDS/packages/property-dds/package.json index e46e5c139246..4f4dcd3012fd 100644 --- a/experimental/PropertyDDS/packages/property-dds/package.json +++ b/experimental/PropertyDDS/packages/property-dds/package.json @@ -48,6 +48,7 @@ "axios": "^0.26.0", "fastest-json-copy": "^1.0.1", "lodash": "^4.17.21", + "lz4js": "^0.2.0", "msgpackr": "^1.4.7", "pako": "^2.0.4", "uuid": "^8.3.1" diff --git a/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExt.ts b/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExt.ts index 15d55353820f..fa3c702051e4 100644 --- a/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExt.ts +++ b/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExt.ts @@ -5,7 +5,7 @@ import { IChannelFactory, IFluidDataStoreRuntime } from "@fluidframework/datastore-definitions"; import { SharedPropertyTree } from "./propertyTree"; -import { DeflatedPropertyTreeFactory } from "./propertyTreeExtFactories"; +import { DeflatedPropertyTreeFactory, LZ4PropertyTreeFactory } from "./propertyTreeExtFactories"; /** * This class is the extension of SharedPropertyTree which compresses @@ -20,3 +20,13 @@ export class DeflatedPropertyTree extends SharedPropertyTree { return new DeflatedPropertyTreeFactory(); } } + +export class LZ4PropertyTree extends SharedPropertyTree { + public static create(runtime: IFluidDataStoreRuntime, id?: string, queryString?: string) { + return runtime.createChannel(id, LZ4PropertyTreeFactory.Type) as LZ4PropertyTree; + } + + public static getFactory(): IChannelFactory { + return new LZ4PropertyTreeFactory(); + } +} diff --git a/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExtFactories.ts b/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExtFactories.ts index 43031acaeecf..c4a776b668c8 100644 --- a/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExtFactories.ts +++ b/experimental/PropertyDDS/packages/property-dds/src/propertyTreeExtFactories.ts @@ -2,68 +2,107 @@ * Copyright (c) Microsoft Corporation and contributors. All rights reserved. * Licensed under the MIT License. */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ import { deflate, inflate } from "pako"; +import { compress, decompress } from "lz4js"; import { bufferToString, stringToBuffer } from "@fluidframework/common-utils"; import { IChannelAttributes, IFluidDataStoreRuntime, IChannelServices, IChannelFactory, + IChannel, } from "@fluidframework/datastore-definitions"; -import { IPropertyTreeMessage, ISharedPropertyTreeEncDec, ISnapshotSummary, SharedPropertyTreeOptions } -from "./propertyTree"; -import { DeflatedPropertyTree } from "./propertyTreeExt"; - -function encodeSummary(snapshotSummary: ISnapshotSummary) { - const summaryStr = JSON.stringify(snapshotSummary); - const unzipped = new TextEncoder().encode(summaryStr); - const serializedSummary: Buffer = deflate(unzipped); - return serializedSummary; +import { + IPropertyTreeConfig, IPropertyTreeMessage, ISharedPropertyTreeEncDec, + ISnapshotSummary, SharedPropertyTree, SharedPropertyTreeOptions, } + from "./propertyTree"; +import { DeflatedPropertyTree, LZ4PropertyTree } from "./propertyTreeExt"; -function decodeSummary(serializedSummary): ISnapshotSummary { - const unzipped = inflate(serializedSummary); - const summaryStr = new TextDecoder().decode(unzipped); - const snapshotSummary: ISnapshotSummary = JSON.parse(summaryStr); - return snapshotSummary; -} +export abstract class CompressedPropertyTreeFactory implements IChannelFactory { + public abstract get attributes(); + public abstract get type(); + public abstract getEncodeFce(); + public abstract getDecodeFce(); + private createCompressionMethods(encodeFn, decodeFn): ISharedPropertyTreeEncDec { + return { + messageEncoder: { + encode: (change: IPropertyTreeMessage) => { + const changeSetStr = JSON.stringify(change.changeSet); + const unzipped = new TextEncoder().encode(changeSetStr); + const zipped: Buffer = encodeFn(unzipped); + const zippedStr = bufferToString(zipped, "base64"); + if (zippedStr.length < changeSetStr.length) { + // eslint-disable-next-line @typescript-eslint/dot-notation + change["isZipped"] = "1"; + change.changeSet = zippedStr; + } + return change; + }, + decode: (transferChange: IPropertyTreeMessage) => { + // eslint-disable-next-line @typescript-eslint/dot-notation + if (transferChange["isZipped"]) { + const zipped = new Uint8Array(stringToBuffer(transferChange.changeSet, "base64")); + const unzipped = decodeFn(zipped); + const changeSetStr = new TextDecoder().decode(unzipped); + transferChange.changeSet = JSON.parse(changeSetStr); + } + return transferChange; + }, + }, + summaryEncoder: { + encode: (snapshotSummary: ISnapshotSummary): Buffer => { + const summaryStr = JSON.stringify(snapshotSummary); + const unzipped = new TextEncoder().encode(summaryStr); + const serializedSummary: Buffer = encodeFn(unzipped); + return serializedSummary; + }, + decode: (serializedSummary): ISnapshotSummary => { + const unzipped = decodeFn(serializedSummary); + const summaryStr = new TextDecoder().decode(unzipped); + const snapshotSummary: ISnapshotSummary = JSON.parse(summaryStr); + return snapshotSummary; + }, + }, + }; + } + public getEncDec(): ISharedPropertyTreeEncDec { + return this.createCompressionMethods(this.getEncodeFce(), this.getDecodeFce()); + } + public abstract newPropertyTree( + id: string, + runtime: IFluidDataStoreRuntime, + attributes: IChannelAttributes, + options: SharedPropertyTreeOptions, + propertyTreeConfig: IPropertyTreeConfig): SharedPropertyTree; -function encodeMessage(change: IPropertyTreeMessage) { - const changeSetStr = JSON.stringify(change.changeSet); - const unzipped = new TextEncoder().encode(changeSetStr); - const zipped: Buffer = deflate(unzipped); - const zippedStr = bufferToString(zipped, "base64"); - if (zippedStr.length < changeSetStr.length) { - // eslint-disable-next-line @typescript-eslint/dot-notation - change["isZipped"] = "1"; - change.changeSet = zippedStr; + public async load( + runtime: IFluidDataStoreRuntime, + id: string, + services: IChannelServices, + attributes: IChannelAttributes, + url?: string, + ): Promise { + const options = {}; + const instance = this.newPropertyTree(id, runtime, attributes, + options as SharedPropertyTreeOptions + , { encDec: this.getEncDec() }); + await instance.load(services); + return instance; } - return change; -} -function decodeMessage(transferChange: IPropertyTreeMessage) { - // eslint-disable-next-line @typescript-eslint/dot-notation - if (transferChange["isZipped"]) { - const zipped = stringToBuffer(transferChange.changeSet, "base64"); - const unzipped = inflate(zipped); - const changeSetStr = new TextDecoder().decode(unzipped); - transferChange.changeSet = JSON.parse(changeSetStr); + public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): SharedPropertyTree { + const options = {}; + const cell = this.newPropertyTree(id, document, + this.attributes, options as SharedPropertyTreeOptions, + { encDec: this.getEncDec() }); + cell.initializeLocal(); + return cell; } - return transferChange; } -const encDec: ISharedPropertyTreeEncDec = { - messageEncoder: { - encode: encodeMessage, - decode: decodeMessage, - }, - summaryEncoder: { - encode: encodeSummary, - decode: decodeSummary, - }, -}; - -export class DeflatedPropertyTreeFactory implements IChannelFactory { +export class DeflatedPropertyTreeFactory extends CompressedPropertyTreeFactory { public static readonly Type = "DeflatedPropertyTree:84534a0fe613522101f6"; public static readonly Attributes: IChannelAttributes = { @@ -72,6 +111,20 @@ export class DeflatedPropertyTreeFactory implements IChannelFactory { packageVersion: "0.0.1", }; + public async load( + runtime: IFluidDataStoreRuntime, + id: string, + services: IChannelServices, + attributes: IChannelAttributes, + url?: string, + ): Promise { + return await super.load(runtime, id, services, attributes, url) as DeflatedPropertyTree; + } + + public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): DeflatedPropertyTree { + return super.create(document, id, requestUrl); + } + public get type() { return DeflatedPropertyTreeFactory.Type; } @@ -80,25 +133,57 @@ export class DeflatedPropertyTreeFactory implements IChannelFactory { return DeflatedPropertyTreeFactory.Attributes; } + public getEncodeFce() { return deflate; } + public getDecodeFce() { return inflate; } + public newPropertyTree( + id: string, + runtime: IFluidDataStoreRuntime, + attributes: IChannelAttributes, + options: SharedPropertyTreeOptions, + propertyTreeConfig: IPropertyTreeConfig): SharedPropertyTree { + return new DeflatedPropertyTree(id, runtime, attributes, options, propertyTreeConfig); + } +} + +export class LZ4PropertyTreeFactory extends CompressedPropertyTreeFactory { + public static readonly Type = "LZ4PropertyTree:84534a0fe613522101f6"; + + public static readonly Attributes: IChannelAttributes = { + type: LZ4PropertyTreeFactory.Type, + snapshotFormatVersion: "0.1", + packageVersion: "0.0.1", + }; + public async load( runtime: IFluidDataStoreRuntime, id: string, services: IChannelServices, attributes: IChannelAttributes, url?: string, - ): Promise { - const options = {}; - const instance = new DeflatedPropertyTree(id, runtime, attributes, options as SharedPropertyTreeOptions - , { encDec }); - await instance.load(services); - return instance; + ): Promise { + return await super.load(runtime, id, services, attributes, url) as LZ4PropertyTree; } - public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): DeflatedPropertyTree { - const options = {}; - const cell = new DeflatedPropertyTree(id, document, - this.attributes, options as SharedPropertyTreeOptions, { encDec }); - cell.initializeLocal(); - return cell; + public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): LZ4PropertyTree { + return super.create(document, id, requestUrl); + } + + public get type() { + return LZ4PropertyTreeFactory.Type; + } + + public get attributes() { + return LZ4PropertyTreeFactory.Attributes; + } + + public getEncodeFce() { return compress; } + public getDecodeFce() { return decompress; } + public newPropertyTree( + id: string, + runtime: IFluidDataStoreRuntime, + attributes: IChannelAttributes, + options: SharedPropertyTreeOptions, + propertyTreeConfig: IPropertyTreeConfig): SharedPropertyTree { + return new LZ4PropertyTree(id, runtime, attributes, options, propertyTreeConfig); } } diff --git a/experimental/PropertyDDS/packages/property-dds/src/test/propertyTree.spec.ts b/experimental/PropertyDDS/packages/property-dds/src/test/propertyTree.spec.ts index ff2c0b464c6d..c937abb4d99a 100644 --- a/experimental/PropertyDDS/packages/property-dds/src/test/propertyTree.spec.ts +++ b/experimental/PropertyDDS/packages/property-dds/src/test/propertyTree.spec.ts @@ -19,7 +19,7 @@ import { TestFluidObjectFactory, } from "@fluidframework/test-utils"; import { PropertyFactory, StringProperty, BaseProperty } from "@fluid-experimental/property-properties"; -import { DeflatedPropertyTree } from "../propertyTreeExt"; +import { DeflatedPropertyTree, LZ4PropertyTree } from "../propertyTreeExt"; import { SharedPropertyTree } from "../propertyTree"; describe("PropertyTree", () => { @@ -39,6 +39,11 @@ describe("PropertyTree", () => { describe("SharedPropertyTree", () => { executePerPropertyTreeType(codeDetails, factory2, documentId, documentLoadUrl, propertyDdsId); }); + + const factory3 = new TestFluidObjectFactory([[propertyDdsId, LZ4PropertyTree.getFactory()]]); + describe("LZ4PropertyTree", () => { + executePerPropertyTreeType(codeDetails, factory1, documentId, documentLoadUrl, propertyDdsId); + }); }); function executePerPropertyTreeType(codeDetails: IFluidCodeDetails, factory: TestFluidObjectFactory, documentId: string, documentLoadUrl: