Skip to content

Commit

Permalink
Milan multiple compression (#11083)
Browse files Browse the repository at this point in the history
  • Loading branch information
@milanro
milanro authored Aug 9, 2022
1 parent 4c59b71 commit 685c9fb
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 60 deletions.
1 change: 1 addition & 0 deletions experimental/PropertyDDS/packages/property-dds/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
"axios": "^0.26.0",
"fastest-json-copy": "^1.0.1",
"lodash": "^4.17.21",
"lz4js": "^0.2.0",
"msgpackr": "^1.4.7",
"pako": "^2.0.4",
"uuid": "^8.3.1"
Expand Down
12 changes: 11 additions & 1 deletion experimental/PropertyDDS/packages/property-dds/src/propertyTreeExt.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import { IChannelFactory, IFluidDataStoreRuntime } from "@fluidframework/datastore-definitions";
import { SharedPropertyTree } from "./propertyTree";
import { DeflatedPropertyTreeFactory } from "./propertyTreeExtFactories";
import { DeflatedPropertyTreeFactory, LZ4PropertyTreeFactory } from "./propertyTreeExtFactories";

/**
* This class is the extension of SharedPropertyTree which compresses
Expand All @@ -20,3 +20,13 @@ export class DeflatedPropertyTree extends SharedPropertyTree {
return new DeflatedPropertyTreeFactory();
}
}

export class LZ4PropertyTree extends SharedPropertyTree {
public static create(runtime: IFluidDataStoreRuntime, id?: string, queryString?: string) {
return runtime.createChannel(id, LZ4PropertyTreeFactory.Type) as LZ4PropertyTree;
}

public static getFactory(): IChannelFactory {
return new LZ4PropertyTreeFactory();
}
}
201 changes: 143 additions & 58 deletions experimental/PropertyDDS/packages/property-dds/src/propertyTreeExtFactories.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,68 +2,107 @@
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
* Licensed under the MIT License.
*/
/* eslint-disable @typescript-eslint/no-unsafe-return */
import { deflate, inflate } from "pako";
import { compress, decompress } from "lz4js";
import { bufferToString, stringToBuffer } from "@fluidframework/common-utils";
import {
IChannelAttributes,
IFluidDataStoreRuntime,
IChannelServices,
IChannelFactory,
IChannel,
} from "@fluidframework/datastore-definitions";
import { IPropertyTreeMessage, ISharedPropertyTreeEncDec, ISnapshotSummary, SharedPropertyTreeOptions }
from "./propertyTree";
import { DeflatedPropertyTree } from "./propertyTreeExt";

function encodeSummary(snapshotSummary: ISnapshotSummary) {
const summaryStr = JSON.stringify(snapshotSummary);
const unzipped = new TextEncoder().encode(summaryStr);
const serializedSummary: Buffer = deflate(unzipped);
return serializedSummary;
import {
IPropertyTreeConfig, IPropertyTreeMessage, ISharedPropertyTreeEncDec,
ISnapshotSummary, SharedPropertyTree, SharedPropertyTreeOptions,
}
from "./propertyTree";
import { DeflatedPropertyTree, LZ4PropertyTree } from "./propertyTreeExt";

function decodeSummary(serializedSummary): ISnapshotSummary {
const unzipped = inflate(serializedSummary);
const summaryStr = new TextDecoder().decode(unzipped);
const snapshotSummary: ISnapshotSummary = JSON.parse(summaryStr);
return snapshotSummary;
}
export abstract class CompressedPropertyTreeFactory implements IChannelFactory {
public abstract get attributes();
public abstract get type();
public abstract getEncodeFce();
public abstract getDecodeFce();
private createCompressionMethods(encodeFn, decodeFn): ISharedPropertyTreeEncDec {
return {
messageEncoder: {
encode: (change: IPropertyTreeMessage) => {
const changeSetStr = JSON.stringify(change.changeSet);
const unzipped = new TextEncoder().encode(changeSetStr);
const zipped: Buffer = encodeFn(unzipped);
const zippedStr = bufferToString(zipped, "base64");
if (zippedStr.length < changeSetStr.length) {
// eslint-disable-next-line @typescript-eslint/dot-notation
change["isZipped"] = "1";
change.changeSet = zippedStr;
}
return change;
},
decode: (transferChange: IPropertyTreeMessage) => {
// eslint-disable-next-line @typescript-eslint/dot-notation
if (transferChange["isZipped"]) {
const zipped = new Uint8Array(stringToBuffer(transferChange.changeSet, "base64"));
const unzipped = decodeFn(zipped);
const changeSetStr = new TextDecoder().decode(unzipped);
transferChange.changeSet = JSON.parse(changeSetStr);
}
return transferChange;
},
},
summaryEncoder: {
encode: (snapshotSummary: ISnapshotSummary): Buffer => {
const summaryStr = JSON.stringify(snapshotSummary);
const unzipped = new TextEncoder().encode(summaryStr);
const serializedSummary: Buffer = encodeFn(unzipped);
return serializedSummary;
},
decode: (serializedSummary): ISnapshotSummary => {
const unzipped = decodeFn(serializedSummary);
const summaryStr = new TextDecoder().decode(unzipped);
const snapshotSummary: ISnapshotSummary = JSON.parse(summaryStr);
return snapshotSummary;
},
},
};
}
public getEncDec(): ISharedPropertyTreeEncDec {
return this.createCompressionMethods(this.getEncodeFce(), this.getDecodeFce());
}
public abstract newPropertyTree(
id: string,
runtime: IFluidDataStoreRuntime,
attributes: IChannelAttributes,
options: SharedPropertyTreeOptions,
propertyTreeConfig: IPropertyTreeConfig): SharedPropertyTree;

function encodeMessage(change: IPropertyTreeMessage) {
const changeSetStr = JSON.stringify(change.changeSet);
const unzipped = new TextEncoder().encode(changeSetStr);
const zipped: Buffer = deflate(unzipped);
const zippedStr = bufferToString(zipped, "base64");
if (zippedStr.length < changeSetStr.length) {
// eslint-disable-next-line @typescript-eslint/dot-notation
change["isZipped"] = "1";
change.changeSet = zippedStr;
public async load(
runtime: IFluidDataStoreRuntime,
id: string,
services: IChannelServices,
attributes: IChannelAttributes,
url?: string,
): Promise<SharedPropertyTree> {
const options = {};
const instance = this.newPropertyTree(id, runtime, attributes,
options as SharedPropertyTreeOptions
, { encDec: this.getEncDec() });
await instance.load(services);
return instance;
}
return change;
}

function decodeMessage(transferChange: IPropertyTreeMessage) {
// eslint-disable-next-line @typescript-eslint/dot-notation
if (transferChange["isZipped"]) {
const zipped = stringToBuffer(transferChange.changeSet, "base64");
const unzipped = inflate(zipped);
const changeSetStr = new TextDecoder().decode(unzipped);
transferChange.changeSet = JSON.parse(changeSetStr);
public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): SharedPropertyTree {
const options = {};
const cell = this.newPropertyTree(id, document,
this.attributes, options as SharedPropertyTreeOptions,
{ encDec: this.getEncDec() });
cell.initializeLocal();
return cell;
}
return transferChange;
}

const encDec: ISharedPropertyTreeEncDec = {
messageEncoder: {
encode: encodeMessage,
decode: decodeMessage,
},
summaryEncoder: {
encode: encodeSummary,
decode: decodeSummary,
},
};

export class DeflatedPropertyTreeFactory implements IChannelFactory {
export class DeflatedPropertyTreeFactory extends CompressedPropertyTreeFactory {
public static readonly Type = "DeflatedPropertyTree:84534a0fe613522101f6";

public static readonly Attributes: IChannelAttributes = {
Expand All @@ -72,6 +111,20 @@ export class DeflatedPropertyTreeFactory implements IChannelFactory {
packageVersion: "0.0.1",
};

public async load(
runtime: IFluidDataStoreRuntime,
id: string,
services: IChannelServices,
attributes: IChannelAttributes,
url?: string,
): Promise<DeflatedPropertyTree> {
return await super.load(runtime, id, services, attributes, url) as DeflatedPropertyTree;
}

public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): DeflatedPropertyTree {
return super.create(document, id, requestUrl);
}

public get type() {
return DeflatedPropertyTreeFactory.Type;
}
Expand All @@ -80,25 +133,57 @@ export class DeflatedPropertyTreeFactory implements IChannelFactory {
return DeflatedPropertyTreeFactory.Attributes;
}

public getEncodeFce() { return deflate; }
public getDecodeFce() { return inflate; }
public newPropertyTree(
id: string,
runtime: IFluidDataStoreRuntime,
attributes: IChannelAttributes,
options: SharedPropertyTreeOptions,
propertyTreeConfig: IPropertyTreeConfig): SharedPropertyTree {
return new DeflatedPropertyTree(id, runtime, attributes, options, propertyTreeConfig);
}
}

export class LZ4PropertyTreeFactory extends CompressedPropertyTreeFactory {
public static readonly Type = "LZ4PropertyTree:84534a0fe613522101f6";

public static readonly Attributes: IChannelAttributes = {
type: LZ4PropertyTreeFactory.Type,
snapshotFormatVersion: "0.1",
packageVersion: "0.0.1",
};

public async load(
runtime: IFluidDataStoreRuntime,
id: string,
services: IChannelServices,
attributes: IChannelAttributes,
url?: string,
): Promise<DeflatedPropertyTree> {
const options = {};
const instance = new DeflatedPropertyTree(id, runtime, attributes, options as SharedPropertyTreeOptions
, { encDec });
await instance.load(services);
return instance;
): Promise<LZ4PropertyTree> {
return await super.load(runtime, id, services, attributes, url) as LZ4PropertyTree;
}

public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): DeflatedPropertyTree {
const options = {};
const cell = new DeflatedPropertyTree(id, document,
this.attributes, options as SharedPropertyTreeOptions, { encDec });
cell.initializeLocal();
return cell;
public create(document: IFluidDataStoreRuntime, id: string, requestUrl?: string): LZ4PropertyTree {
return super.create(document, id, requestUrl);
}

public get type() {
return LZ4PropertyTreeFactory.Type;
}

public get attributes() {
return LZ4PropertyTreeFactory.Attributes;
}

public getEncodeFce() { return compress; }
public getDecodeFce() { return decompress; }
public newPropertyTree(
id: string,
runtime: IFluidDataStoreRuntime,
attributes: IChannelAttributes,
options: SharedPropertyTreeOptions,
propertyTreeConfig: IPropertyTreeConfig): SharedPropertyTree {
return new LZ4PropertyTree(id, runtime, attributes, options, propertyTreeConfig);
}
}
7 changes: 6 additions & 1 deletion experimental/PropertyDDS/packages/property-dds/src/test/propertyTree.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
TestFluidObjectFactory,
} from "@fluidframework/test-utils";
import { PropertyFactory, StringProperty, BaseProperty } from "@fluid-experimental/property-properties";
import { DeflatedPropertyTree } from "../propertyTreeExt";
import { DeflatedPropertyTree, LZ4PropertyTree } from "../propertyTreeExt";
import { SharedPropertyTree } from "../propertyTree";

describe("PropertyTree", () => {
Expand All @@ -39,6 +39,11 @@ describe("PropertyTree", () => {
describe("SharedPropertyTree", () => {
executePerPropertyTreeType(codeDetails, factory2, documentId, documentLoadUrl, propertyDdsId);
});

const factory3 = new TestFluidObjectFactory([[propertyDdsId, LZ4PropertyTree.getFactory()]]);
describe("LZ4PropertyTree", () => {
executePerPropertyTreeType(codeDetails, factory1, documentId, documentLoadUrl, propertyDdsId);
});
});
function executePerPropertyTreeType(codeDetails: IFluidCodeDetails,
factory: TestFluidObjectFactory, documentId: string, documentLoadUrl:
Expand Down

0 comments on commit 685c9fb

Please sign in to comment.