From 9fa9ded80eeab36467f668ab585f9db9c36e2177 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 14 Sep 2023 22:08:01 +0800 Subject: [PATCH] Perf: run cpu intensive task with worker --- Build/index.js | 30 ++++++++++++- Build/lib/is-domain-loose.js | 4 +- package.json | 1 + pnpm-lock.yaml | 82 ++++++++++++++++++++++++++++++++++-- 4 files changed, 110 insertions(+), 7 deletions(-) diff --git a/Build/index.js b/Build/index.js index 37597e75d..a4fc69ddc 100644 --- a/Build/index.js +++ b/Build/index.js @@ -9,14 +9,42 @@ const { buildTelegramCIDR } = require('./build-telegram-cidr'); const { buildChnCidr } = require('./build-chn-cidr'); const { buildSpeedtestDomainSet } = require('./build-speedtest-domainset'); const { buildInternalCDNDomains } = require('./build-internal-cdn-rules'); -const { buildInternalReverseChnCIDR } = require('./build-internal-reverse-chn-cidr'); const { buildInternalChnDomains } = require('./build-internal-chn-domains'); const { buildDomesticRuleset } = require('./build-domestic-ruleset'); const { validate } = require('./validate-domainset'); const { buildPublicHtml } = require('./build-public'); +const { Worker } = require('jest-worker'); + +/** + * @template T + * @typedef {import('jest-worker').Worker & T} WithWorker + */ + +/** + * @template T + * @param {string} path + * @returns {WithWorker} + */ +const requireWorker = (path) => { + const _worker = (new Worker( + require.resolve(path), + { + numWorkers: 1, + maxRetries: 0, + enableWorkerThreads: true + } + )); + _worker.getStderr().pipe(process.stderr); + _worker.getStdout().pipe(process.stdout); + + return _worker; +}; + (async () => { + const { buildInternalReverseChnCIDR } = /** @type {WithWorker} */ (requireWorker('./build-internal-reverse-chn-cidr')); + // download-previous-build const downloadPreviousBuildPromise = downloadPreviousBuild(); const downloadPublicSuffixListPromise = downloadPublicSuffixList(); diff --git a/Build/lib/is-domain-loose.js b/Build/lib/is-domain-loose.js index 0b72271d4..7b170469a 100644 --- a/Build/lib/is-domain-loose.js +++ b/Build/lib/is-domain-loose.js @@ -10,10 +10,10 @@ module.exports.isDomainLoose = (domain) => { }; /** - * @param {string} domain + * @param {string | null | undefined} domain */ module.exports.normalizeDomain = (domain) => { - if (domain == null) { + if (!domain) { return null; } diff --git a/package.json b/package.json index dbddac0b5..e51a2743f 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "cidr-tools-wasm": "^0.0.11", "fs-extra": "^11.1.1", "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js", + "jest-worker": "^29.7.0", "mnemonist": "^0.39.5", "path-scurry": "^1.10.1", "picocolors": "^1.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec8b55fd7..ec8c593fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,6 +37,9 @@ dependencies: gorhill-publicsuffixlist: specifier: github:gorhill/publicsuffixlist.js version: github.com/gorhill/publicsuffixlist.js/3a1bc623073079184ff76933b88b7bf4f5d48978 + jest-worker: + specifier: ^29.7.0 + version: 29.7.0 mnemonist: specifier: ^0.39.5 version: 0.39.5 @@ -181,6 +184,25 @@ packages: resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==} dev: true + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@sinclair/typebox': 0.27.8 + dev: false + + /@jest/types@29.6.3: + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.4 + '@types/istanbul-reports': 3.0.1 + '@types/node': 20.6.0 + '@types/yargs': 17.0.24 + chalk: 4.1.2 + dev: false + /@nodelib/fs.scandir@2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -242,6 +264,10 @@ packages: resolution: {integrity: sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q==} dev: false + /@sinclair/typebox@0.27.8: + resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + dev: false + /@sukka/listdir@0.2.0: resolution: {integrity: sha512-UyVirNhAOXKwjiDehjUaGtpfk0QwNHyiXrlLb/FmWMtI+BGhaEvB9MypSfEAtiiMI3g6QTfG38ayNAorEuz5ow==} dev: false @@ -271,13 +297,38 @@ packages: resolution: {integrity: sha512-o0J30wqycjF5miWDKYKKzzOU1ZTLuA42HZ4HE7/zqTOc/jTLdQ5NhYWvsRQo45Nfi1KHoRdNhteSI4BAxTF1Pg==} dev: false + /@types/istanbul-lib-coverage@2.0.4: + resolution: {integrity: sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==} + dev: false + + /@types/istanbul-lib-report@3.0.0: + resolution: {integrity: sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==} + dependencies: + '@types/istanbul-lib-coverage': 2.0.4 + dev: false + + /@types/istanbul-reports@3.0.1: + resolution: {integrity: sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==} + dependencies: + '@types/istanbul-lib-report': 3.0.0 + dev: false + /@types/mocha@10.0.1: resolution: {integrity: sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q==} dev: true /@types/node@20.6.0: resolution: {integrity: sha512-najjVq5KN2vsH2U/xyh2opaSEz6cZMR2SetLIlxlj08nOcmPOemJmUK2o4kUzfLqfrWE0PIrNeE16XhYDd3nqg==} - dev: true + + /@types/yargs-parser@21.0.0: + resolution: {integrity: sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==} + dev: false + + /@types/yargs@17.0.24: + resolution: {integrity: sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==} + dependencies: + '@types/yargs-parser': 21.0.0 + dev: false /@vercel/fetch-retry@5.1.3(patch_hash=jczsp2gadg65vd4ekschpibzda)(node-fetch@2.6.11): resolution: {integrity: sha512-UIbFc4VsEZHOr6dWuE+kxY4NxnOLXFMCWm0fSKRRHUEtrIzaJLzHpWk2QskCXTSzFgFvhkLAvSrBK2XZg7NSzg==} @@ -445,7 +496,6 @@ packages: dependencies: ansi-styles: 4.3.0 supports-color: 7.2.0 - dev: true /check-error@1.0.2: resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} @@ -1015,6 +1065,28 @@ packages: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} dev: true + /jest-util@29.7.0: + resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/node': 20.6.0 + chalk: 4.1.2 + ci-info: 3.8.0 + graceful-fs: 4.2.10 + picomatch: 2.3.1 + dev: false + + /jest-worker@29.7.0: + resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@types/node': 20.6.0 + jest-util: 29.7.0 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: false + /js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true @@ -1091,6 +1163,10 @@ packages: yallist: 4.0.0 dev: true + /merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + dev: false + /minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: @@ -1279,7 +1355,6 @@ packages: /picomatch@2.3.1: resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} engines: {node: '>=8.6'} - dev: true /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} @@ -1426,7 +1501,6 @@ packages: engines: {node: '>=8'} dependencies: has-flag: 4.0.0 - dev: true /supports-color@8.1.1: resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==}