From a6fa07cfc5f65df9a1c9efb08e9b66d19f97761c Mon Sep 17 00:00:00 2001 From: Yidong Li Date: Wed, 9 Aug 2023 21:48:43 +0800 Subject: [PATCH] feat(splitting): support minChunkSize option --- cmd/esbuild/main.go | 1 + .../bundler_tests/bundler_splitting_test.go | 31 ++++++++++++ .../snapshots/snapshots_splitting.txt | 47 ++++++++++++++++++ internal/config/config.go | 1 + internal/graph/graph.go | 3 ++ internal/linker/linker.go | 48 ++++++++++++++++++- lib/shared/common.ts | 2 + lib/shared/types.ts | 2 + pkg/api/api.go | 1 + pkg/api/api_impl.go | 1 + pkg/cli/cli_impl.go | 13 +++++ scripts/end-to-end-tests.js | 29 +++++++++++ 12 files changed, 178 insertions(+), 1 deletion(-) diff --git a/cmd/esbuild/main.go b/cmd/esbuild/main.go index 94671d6d5d3..3385d33a273 100644 --- a/cmd/esbuild/main.go +++ b/cmd/esbuild/main.go @@ -50,6 +50,7 @@ var helpText = func(colors logger.Colors) string { --serve=... Start a local HTTP server on this host:port for outputs --sourcemap Emit a source map --splitting Enable code splitting (currently only for esm) + --min-chunk-size Control min chunk source size for code splitting (currently only for js) --target=... Environment target (e.g. es2017, chrome58, firefox57, safari11, edge16, node10, ie9, opera45, default esnext) --watch Watch mode: rebuild on file system changes (stops when diff --git a/internal/bundler_tests/bundler_splitting_test.go b/internal/bundler_tests/bundler_splitting_test.go index 5b199a2363c..761d2e24869 100644 --- a/internal/bundler_tests/bundler_splitting_test.go +++ b/internal/bundler_tests/bundler_splitting_test.go @@ -138,6 +138,37 @@ func TestSplittingDynamicAndNotDynamicCommonJSIntoES6(t *testing.T) { }) } +func TestSplittingWithMinChunkSize(t *testing.T) { + splitting_suite.expectBundled(t, bundled{ + files: map[string]string{ + "/a.js": ` + import * as ns from './common_mini' + import * as nsl from './common_large' + export let a = 'a' + ns.foo + export let aa = 'a' + nsl.bar + `, + "/b.js": ` + import * as ns from './common_mini' + export let b = 'b' + ns.foo + `, + "/c.js": ` + import * as ns from './common_large' + export let b = 'b' + ns.bar + `, + "/common_mini.js": `export let foo = 123`, + "/common_large.js": `export let bar = 1234`, + }, + entryPaths: []string{"/a.js", "/b.js", "/c.js"}, + options: config.Options{ + Mode: config.ModeBundle, + CodeSplitting: true, + MinChunkSize: 21, + OutputFormat: config.FormatESModule, + AbsOutputDir: "/out", + }, + }) +} + func TestSplittingAssignToLocal(t *testing.T) { splitting_suite.expectBundled(t, bundled{ files: map[string]string{ diff --git a/internal/bundler_tests/snapshots/snapshots_splitting.txt b/internal/bundler_tests/snapshots/snapshots_splitting.txt index 3976512cc97..af614681852 100644 --- a/internal/bundler_tests/snapshots/snapshots_splitting.txt +++ b/internal/bundler_tests/snapshots/snapshots_splitting.txt @@ -622,3 +622,50 @@ export { a, b }; + +================================================================================ +TestSplittingWithMinChunkSize +---------- /out/a.js ---------- +import { + bar +} from "./chunk-RIRTUAAF.js"; + +// common_mini.js +var foo = 123; + +// a.js +var a = "a" + foo; +var aa = "a" + bar; +export { + a, + aa +}; + +---------- /out/b.js ---------- +// common_mini.js +var foo = 123; + +// b.js +var b = "b" + foo; +export { + b +}; + +---------- /out/c.js ---------- +import { + bar +} from "./chunk-RIRTUAAF.js"; + +// c.js +var b = "b" + bar; +export { + b +}; + +---------- /out/chunk-RIRTUAAF.js ---------- +// common_large.js +var bar = 1234; + +export { + bar +}; diff --git a/internal/config/config.go b/internal/config/config.go index c309f5cc095..43c8de1799e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -454,6 +454,7 @@ type Options struct { MinifySyntax bool ProfilerNames bool CodeSplitting bool + MinChunkSize int WatchMode bool AllowOverwrite bool LegalComments LegalComments diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 72e6d9b07a2..784ad80a4a8 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -60,6 +60,9 @@ type LinkerFile struct { // This is true if this file has been marked as live by the tree shaking // algorithm. IsLive bool + + // If true, this part can't be extract to cross chunk dependencies. + SplitOff bool } func (f *LinkerFile) IsEntryPoint() bool { diff --git a/internal/linker/linker.go b/internal/linker/linker.go index ef4b59dfca0..6cbaa02c4b0 100644 --- a/internal/linker/linker.go +++ b/internal/linker/linker.go @@ -90,6 +90,9 @@ type chunkInfo struct { // For code splitting crossChunkImports []chunkImport + // This is the total size of input files + chunkSize int + // This is the representation-specific information chunkRepr chunkRepr @@ -962,6 +965,13 @@ func (c *linkerContext) computeCrossChunkDependencies() { // chunk. In that case this will overwrite the same value below which // is fine. for _, declared := range part.DeclaredSymbols { + // Ignore all symbols if bound file is SplitOff + // If SourceIndex is larger than graph.Files, it means the symbol + // is from a generated chunk and SplitOff check can be skipped + if declared.Ref.SourceIndex < uint32(len(c.graph.Files)) && c.graph.Files[declared.Ref.SourceIndex].SplitOff { + continue + } + if declared.IsTopLevel { c.graph.Symbols.Get(declared.Ref).ChunkIndex = ast.MakeIndex32(uint32(chunkIndex)) } @@ -973,6 +983,13 @@ func (c *linkerContext) computeCrossChunkDependencies() { for ref := range part.SymbolUses { symbol := c.graph.Symbols.Get(ref) + // Ignore all symbols if bound file is SplitOff + // If SourceIndex is larger than graph.Files, it means the symbol + // is from a generated chunk and SplitOff check can be skipped + if symbol.Link.SourceIndex < uint32(len(c.graph.Files)) && c.graph.Files[symbol.Link.SourceIndex].SplitOff { + continue + } + // Ignore unbound symbols, which don't have declarations if symbol.Kind == ast.SymbolUnbound { continue @@ -3676,6 +3693,7 @@ func (c *linkerContext) computeChunks() { jsChunks := make(map[string]chunkInfo) cssChunks := make(map[string]chunkInfo) + entryKeys := make([]string, 0, len(c.graph.EntryPoints())) // Create chunks for entry points for i, entryPoint := range c.graph.EntryPoints() { @@ -3691,8 +3709,10 @@ func (c *linkerContext) computeChunks() { isEntryPoint: true, sourceIndex: entryPoint.SourceIndex, entryPointBit: uint(i), + chunkSize: 0, filesWithPartsInChunk: make(map[uint32]bool), } + entryKeys = append(entryKeys, key) switch file.InputFile.Repr.(type) { case *graph.JSRepr: @@ -3723,6 +3743,7 @@ func (c *linkerContext) computeChunks() { externalImportsInOrder: externalOrder, filesInChunkInOrder: internalOrder, }, + chunkSize: 0, } chunkRepr.hasCSSChunk = true } @@ -3750,6 +3771,7 @@ func (c *linkerContext) computeChunks() { chunk.entryBits = file.EntryBits chunk.filesWithPartsInChunk = make(map[uint32]bool) chunk.chunkRepr = &chunkReprJS{} + chunk.chunkSize = 0 jsChunks[key] = chunk } chunk.filesWithPartsInChunk[uint32(sourceIndex)] = true @@ -3757,6 +3779,30 @@ func (c *linkerContext) computeChunks() { } } + // remove chunks by user configuration. This matters because auto code splitting + // may generate lots of mini chunks + for key := range jsChunks { + jsChunk := jsChunks[key] + // calculate each jsChunk's size + for sourceIdx := range jsChunk.filesWithPartsInChunk { + jsChunk.chunkSize += len(c.graph.Files[sourceIdx].InputFile.Source.Contents) + } + // If current js chunk is smaller than the minimal chunkSize config, mark this file as SplitOff + // and move it to the entryChunks it belongs to + if !jsChunk.isEntryPoint && jsChunk.chunkSize < c.options.MinChunkSize { + for _, entryKey := range entryKeys { + entryChunk := jsChunks[entryKey] + if jsChunk.entryBits.HasBit(entryChunk.entryPointBit) { + for sourceIdx := range jsChunk.filesWithPartsInChunk { + c.graph.Files[sourceIdx].SplitOff = true + entryChunk.filesWithPartsInChunk[sourceIdx] = true + } + } + } + delete(jsChunks, key) + } + } + // Sort the chunks for determinism. This matters because we use chunk indices // as sorting keys in a few places. sortedChunks := make([]chunkInfo, 0, len(jsChunks)+len(cssChunks)) @@ -3970,7 +4016,7 @@ func (c *linkerContext) findImportedPartsInJSOrder(chunk *chunkInfo) (js []uint3 file := &c.graph.Files[sourceIndex] if repr, ok := file.InputFile.Repr.(*graph.JSRepr); ok { - isFileInThisChunk := chunk.entryBits.Equals(file.EntryBits) + isFileInThisChunk := file.SplitOff || chunk.entryBits.Equals(file.EntryBits) // Wrapped files can't be split because they are all inside the wrapper canFileBeSplit := repr.Meta.Wrap == graph.WrapNone diff --git a/lib/shared/common.ts b/lib/shared/common.ts index 608f79ff416..70c74b37755 100644 --- a/lib/shared/common.ts +++ b/lib/shared/common.ts @@ -249,6 +249,7 @@ function flagsForBuildOptions( let sourcemap = getFlag(options, keys, 'sourcemap', mustBeStringOrBoolean) let bundle = getFlag(options, keys, 'bundle', mustBeBoolean) let splitting = getFlag(options, keys, 'splitting', mustBeBoolean) + let minChunkSize = getFlag(options, keys, 'minChunkSize', mustBeInteger) let preserveSymlinks = getFlag(options, keys, 'preserveSymlinks', mustBeBoolean) let metafile = getFlag(options, keys, 'metafile', mustBeBoolean) let outfile = getFlag(options, keys, 'outfile', mustBeString) @@ -284,6 +285,7 @@ function flagsForBuildOptions( if (bundle) flags.push('--bundle') if (allowOverwrite) flags.push('--allow-overwrite') if (splitting) flags.push('--splitting') + if (minChunkSize) flags.push(`--min-chunk-size=${minChunkSize}`) if (preserveSymlinks) flags.push('--preserve-symlinks') if (metafile) flags.push(`--metafile`) if (outfile) flags.push(`--outfile=${outfile}`) diff --git a/lib/shared/types.ts b/lib/shared/types.ts index d938a6df626..6ec1ae4883a 100644 --- a/lib/shared/types.ts +++ b/lib/shared/types.ts @@ -112,6 +112,8 @@ export interface BuildOptions extends CommonOptions { bundle?: boolean /** Documentation: https://esbuild.github.io/api/#splitting */ splitting?: boolean + /** */ + minChunkSize?: number /** Documentation: https://esbuild.github.io/api/#preserve-symlinks */ preserveSymlinks?: boolean /** Documentation: https://esbuild.github.io/api/#outfile */ diff --git a/pkg/api/api.go b/pkg/api/api.go index 155e1843655..b797fdb265d 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -309,6 +309,7 @@ type BuildOptions struct { Bundle bool // Documentation: https://esbuild.github.io/api/#bundle PreserveSymlinks bool // Documentation: https://esbuild.github.io/api/#preserve-symlinks Splitting bool // Documentation: https://esbuild.github.io/api/#splitting + MinChunkSize int // Documentation: Outfile string // Documentation: https://esbuild.github.io/api/#outfile Metafile bool // Documentation: https://esbuild.github.io/api/#metafile Outdir string // Documentation: https://esbuild.github.io/api/#outdir diff --git a/pkg/api/api_impl.go b/pkg/api/api_impl.go index f695e6430bf..936e79a6455 100644 --- a/pkg/api/api_impl.go +++ b/pkg/api/api_impl.go @@ -1303,6 +1303,7 @@ func validateBuildOptions( TreeShaking: validateTreeShaking(buildOpts.TreeShaking, buildOpts.Bundle, buildOpts.Format), GlobalName: validateGlobalName(log, buildOpts.GlobalName), CodeSplitting: buildOpts.Splitting, + MinChunkSize: buildOpts.MinChunkSize, OutputFormat: validateFormat(buildOpts.Format), AbsOutputFile: validatePath(log, realFS, buildOpts.Outfile, "outfile path"), AbsOutputDir: validatePath(log, realFS, buildOpts.Outdir, "outdir path"), diff --git a/pkg/cli/cli_impl.go b/pkg/cli/cli_impl.go index 8e292cc315b..aef926b1e50 100644 --- a/pkg/cli/cli_impl.go +++ b/pkg/cli/cli_impl.go @@ -113,6 +113,19 @@ func parseOptionsImpl( buildOpts.Splitting = value } + case strings.HasPrefix(arg, "--min-chunk-size="): + value := arg[len("--min-chunk-size="):] + minChunkSize, err := strconv.Atoi(value) + if err != nil || minChunkSize < 0 { + return parseOptionsExtras{}, cli_helpers.MakeErrorWithNote( + fmt.Sprintf("Invalid value %q in %q", value, arg), + "The min chunk size must be a non-negative integer.", + ) + } + if buildOpts != nil { + buildOpts.MinChunkSize = minChunkSize + } + case isBoolFlag(arg, "--allow-overwrite") && buildOpts != nil: if value, err := parseBoolFlag(arg, true); err != nil { return parseOptionsExtras{}, err diff --git a/scripts/end-to-end-tests.js b/scripts/end-to-end-tests.js index 9c86ea3285d..04db70b3a61 100644 --- a/scripts/end-to-end-tests.js +++ b/scripts/end-to-end-tests.js @@ -7038,6 +7038,35 @@ tests.push( `, }), + // Code splitting via minChunkSize control + test([ + 'a.js', 'b.js', 'c.js', '--splitting', + '--outdir=out', '--format=esm', '--bundle', '--min-chunk-size=22' + ], { + 'a.js': ` + import * as ns from './common_mini' + import * as nsl from './common_large' + export let a = 'a' + ns.foo + export let aa = 'a' + nsl.bar + `, + 'b.js': ` + import * as ns from './common_mini' + export let b = 'b' + ns.foo + `, + 'c.js': ` + import * as ns from './common_large' + export let c = 'c' + ns.bar + `, + 'common_mini.js': `export let foo = 123`, + 'common_large.js': `export let bar = 1234`, + 'node.js': ` + import {a, aa} from './out/a.js' + import {b} from './out/b.js' + import {c} from './out/c.js' + if (a !== 'a123' || aa !== 'a1234' || b !== 'b123' || c !== 'c1234') throw 'fail' + `, + }), + // Code splitting via ES6 module double-imported with sync and async imports test(['a.js', '--outdir=out', '--splitting', '--format=esm', '--bundle'], { 'a.js': `