Perf: reduce infra runtime costs

Build/build-anti-bogus-domain.js

@@ -4,9 +4,9 @@ const { isIPv4, isIPv6 } = require('net');
 const { createRuleset } = require('./lib/create-file');
 const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const { processLine } = require('./lib/process-line');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
-const buildAntiBogusDomain = async () => {
+const buildAntiBogusDomain = task(__filename, async () => {
   /** @type {string[]} */
   const res = [];
   for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) {
@@ -56,7 +56,7 @@ const buildAntiBogusDomain = async () => {
     path.resolve(__dirname, '../List/ip/reject.conf'),
     path.resolve(__dirname, '../Clash/ip/reject.txt')
   ));
-};
+});
 
 module.exports.buildAntiBogusDomain = buildAntiBogusDomain;
 

Build/build-apple-cdn.js

@@ -1,9 +1,9 @@
 const path = require('path');
 const { createRuleset } = require('./lib/create-file');
 const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
-const buildAppleCdn = async () => {
+const buildAppleCdn = task(__filename, async () => {
   const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
 
   const description = [
@@ -40,7 +40,7 @@ const buildAppleCdn = async () => {
       path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt')
     )
   ]);
-};
+});
 
 module.exports.buildAppleCdn = buildAppleCdn;
 

Build/build-cdn-conf.js

@@ -3,13 +3,13 @@ const path = require('path');
 const { createRuleset } = require('./lib/create-file');
 const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const Trie = require('./lib/trie');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 const fs = require('fs');
 const { processLine } = require('./lib/process-line');
 
-const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix-list_dat.txt');
+const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix_list_dat.txt');
 
-const buildCdnConf = async () => {
+const buildCdnConf = task(__filename, async () => {
   const trie = new Trie();
 
   if (fs.existsSync(publicSuffixPath)) {
@@ -30,23 +30,33 @@ const buildCdnConf = async () => {
   const S3OSSDomains = new Set();
 
   trie.find('.amazonaws.com').forEach(line => {
-    if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) {
+    if (
+      (line.startsWith('s3-') || line.startsWith('s3.'))
+      && !line.includes('cn-')
+    ) {
       S3OSSDomains.add(line);
     }
   });
   trie.find('.scw.cloud').forEach(line => {
-    if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) {
+    if (
+      (line.startsWith('s3-') || line.startsWith('s3.'))
+      && !line.includes('cn-')
+    ) {
       S3OSSDomains.add(line);
     }
   });
 
   /** @type {string[]} */
   const cdnDomainsList = [];
   for await (const l of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) {
+    if (l === '# --- [AWS S3 Replace Me] ---') {
+      console.log(S3OSSDomains);
+      S3OSSDomains.forEach(domain => {
+        cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`);
+      });
+    }
     const line = processLine(l);
-    if (line === '# --- [AWS S3 Replace Me] ---') {
-      S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`));
-    } else if (line) {
+    if (line) {
       cdnDomainsList.push(line);
     }
   }
@@ -68,7 +78,7 @@ const buildCdnConf = async () => {
     path.resolve(__dirname, '../List/non_ip/cdn.conf'),
     path.resolve(__dirname, '../Clash/non_ip/cdn.txt')
   ));
-};
+});
 
 module.exports.buildCdnConf = buildCdnConf;
 

Build/build-chn-cidr.js

@@ -4,15 +4,15 @@ const { resolve: pathResolve } = require('path');
 // This should not use `createRuleset` API since we are going to generate ipcidr for Clash
 const { compareAndWriteFile, withBannerArray } = require('./lib/create-file');
 const { processLine } = require('./lib/process-line');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
 // https://github.com/misakaio/chnroutes2/issues/25
 const EXCLUDE_CIDRS = [
   '223.118.0.0/15',
   '223.120.0.0/15'
 ];
 
-const buildChnCidr = async () => {
+const buildChnCidr = task(__filename, async () => {
   const { exclude: excludeCidrs } = await import('cidr-tools-wasm');
 
   /** @type {string[]} */
@@ -56,7 +56,7 @@ const buildChnCidr = async () => {
       pathResolve(__dirname, '../Clash/ip/china_ip.txt')
     )
   ]);
-};
+});
 
 module.exports.buildChnCidr = buildChnCidr;
 

Build/build-common.js

@@ -6,7 +6,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const { processLine } = require('./lib/process-line');
 const { createRuleset } = require('./lib/create-file');
 const { domainDeduper } = require('./lib/domain-deduper');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
 const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
 const MAGIC_COMMAND_TITLE = '# $ meta_title ';
@@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source');
 const outputSurgeDir = path.resolve(__dirname, '../List');
 const outputClashDir = path.resolve(__dirname, '../Clash');
 
-const buildCommon = async () => {
+const buildCommon = task(__filename, async () => {
   /** @type {Promise<void>[]} */
   const promises = [];
 
@@ -43,7 +43,7 @@ const buildCommon = async () => {
   }
 
   return Promise.all(promises);
-};
+});
 
 module.exports.buildCommon = buildCommon;
 

Build/build-domestic-ruleset.js

@@ -5,9 +5,9 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const { processLine } = require('./lib/process-line');
 const { compareAndWriteFile, createRuleset } = require('./lib/create-file');
 const domainSorter = require('./lib/stable-sort-domain');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
-const buildDomesticRuleset = async () => {
+const buildDomesticRuleset = task(__filename, async () => {
   const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
   const results = [];
   for await (const l of rl) {
@@ -67,7 +67,7 @@ const buildDomesticRuleset = async () => {
       path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule')
     )
   ]);
-};
+});
 
 module.exports.buildDomesticRuleset = buildDomesticRuleset;
 

Build/build-internal-cdn-rules.js

@@ -5,7 +5,7 @@ const tldts = require('tldts');
 const { processLine } = require('./lib/process-line');
 const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const domainSorter = require('./lib/stable-sort-domain');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 const { compareAndWriteFile } = require('./lib/create-file');
 
 /**
@@ -15,7 +15,7 @@ const escapeRegExp = (string) => {
   return string.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
 };
 
-const buildInternalCDNDomains = async () => {
+const buildInternalCDNDomains = task(__filename, async () => {
   const set = new Set();
   const keywords = new Set();
 
@@ -87,7 +87,7 @@ const buildInternalCDNDomains = async () => {
     ],
     path.resolve(__dirname, '../List/internal/cdn.txt')
   );
-};
+});
 
 module.exports.buildInternalCDNDomains = buildInternalCDNDomains;
 

Build/build-internal-chn-domains.js

@@ -2,10 +2,10 @@
 const path = require('path');
 const fse = require('fs-extra');
 const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 const { compareAndWriteFile } = require('./lib/create-file');
 
-const buildInternalChnDomains = async () => {
+const buildInternalChnDomains = task(__filename, async () => {
   const [result] = await Promise.all([
     parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
     fse.ensureDir(path.resolve(__dirname, '../List/internal'))
@@ -15,7 +15,7 @@ const buildInternalChnDomains = async () => {
     result.map(line => `SUFFIX,${line}`),
     path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt')
   );
-};
+});
 
 module.exports.buildInternalChnDomains = buildInternalChnDomains;
 

Build/build-internal-reverse-chn-cidr.js

@@ -4,7 +4,7 @@ const { processLine } = require('./lib/process-line');
 const path = require('path');
 const fse = require('fs-extra');
 const fs = require('fs');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
 const RESERVED_IPV4_CIDR = [
   '0.0.0.0/8',
@@ -24,7 +24,7 @@ const RESERVED_IPV4_CIDR = [
   '240.0.0.0/4'
 ];
 
-const buildInternalReverseChnCIDR = async () => {
+const buildInternalReverseChnCIDR = task(__filename, async () => {
   const { exclude } = await import('cidr-tools-wasm');
 
   /** @type {string[]} */
@@ -47,7 +47,7 @@ const buildInternalReverseChnCIDR = async () => {
     path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'),
     `${reversedCidr.join('\n')}\n`
   );
-};
+});
 
 module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR;
 

Build/build-phishing-domainset.js

@@ -4,7 +4,7 @@ const path = require('path');
 const { createRuleset } = require('./lib/create-file');
 const { processLine } = require('./lib/process-line.js');
 const domainSorter = require('./lib/stable-sort-domain');
-const { runner, traceSync } = require('./lib/trace-runner.js');
+const { runner, traceSync, task } = require('./lib/trace-runner.js');
 
 const WHITELIST_DOMAIN = new Set([
   'w3s.link',
@@ -60,7 +60,7 @@ const BLACK_TLD = new Set([
   'com.cn'
 ]);
 
-const buildPhishingDomainSet = async () => {
+const buildPhishingDomainSet = task(__filename, async () => {
   const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
   const domainCountMap = {};
 
@@ -151,7 +151,7 @@ const buildPhishingDomainSet = async () => {
     path.resolve(__dirname, '../List/domainset/reject_phishing.conf'),
     path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt')
   ));
-};
+});
 
 module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
 

Build/build-public.js

@@ -2,7 +2,7 @@ const listDir = require('@sukka/listdir');
 const path = require('path');
 const fs = require('fs');
 const fse = require('fs-extra');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
 const rootPath = path.resolve(__dirname, '../');
 const publicPath = path.resolve(__dirname, '../public');
@@ -17,7 +17,7 @@ const folderAndFilesToBeDeployed = [
   'README.md'
 ];
 
-const buildPublicHtml = async () => {
+const buildPublicHtml = task(__filename, async () => {
   await fse.ensureDir(publicPath);
   await Promise.all(folderAndFilesToBeDeployed.map(dir => fse.copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir))));
 
@@ -29,7 +29,7 @@ const buildPublicHtml = async () => {
   const html = template(list);
 
   await fs.promises.writeFile(path.join(publicPath, 'index.html'), html, 'utf-8');
-};
+});
 
 module.exports.buildPublicHtml = buildPublicHtml;
 

Build/build-reject-domainset.js

@@ -12,7 +12,7 @@ const { domainDeduper } = require('./lib/domain-deduper');
 const createKeywordFilter = require('./lib/aho-corasick');
 const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const { createDomainSorter } = require('./lib/stable-sort-domain');
-const { traceSync, runner } = require('./lib/trace-runner');
+const { traceSync, runner, task } = require('./lib/trace-runner');
 const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix');
 const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse');
 
@@ -23,7 +23,7 @@ const domainKeywordsSet = new Set();
 /** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
 const domainSuffixSet = new Set();
 
-const buildRejectDomainSet = async () => {
+const buildRejectDomainSet = task(__dirname, async () => {
   /** @type Set<string> */
   const domainSets = new Set();
 
@@ -228,7 +228,7 @@ const buildRejectDomainSet = async () => {
     // Copy reject_sukka.conf for backward compatibility
     fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
   ]);
-};
+});
 
 module.exports.buildRejectDomainSet = buildRejectDomainSet;
 

Build/build-speedtest-domainset.js

@@ -5,7 +5,7 @@ const { createRuleset } = require('./lib/create-file');
 const domainSorter = require('./lib/stable-sort-domain');
 
 const { Sema } = require('async-sema');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 const s = new Sema(2);
 
 /**
@@ -46,7 +46,7 @@ const querySpeedtestApi = async (keyword) => {
   }
 };
 
-const buildSpeedtestDomainSet = async () => {
+const buildSpeedtestDomainSet = task(__filename, async () => {
   /** @type {Set<string>} */
   const domains = new Set([
     '.speedtest.net',
@@ -123,7 +123,7 @@ const buildSpeedtestDomainSet = async () => {
     path.resolve(__dirname, '../List/domainset/speedtest.conf'),
     path.resolve(__dirname, '../Clash/domainset/speedtest.txt')
   ));
-};
+});
 
 module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet;
 

Build/build-telegram-cidr.js

@@ -4,9 +4,9 @@ const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 const { processLine } = require('./lib/process-line');
 const { createRuleset } = require('./lib/create-file');
-const { runner } = require('./lib/trace-runner');
+const { runner, task } = require('./lib/trace-runner');
 
-const buildTelegramCIDR = async () => {
+const buildTelegramCIDR = task(__filename, async () => {
   /** @type {Response} */
   const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
   const lastModified = resp.headers.get('last-modified');
@@ -49,7 +49,7 @@ const buildTelegramCIDR = async () => {
     path.resolve(__dirname, '../List/ip/telegram.conf'),
     path.resolve(__dirname, '../Clash/ip/telegram.txt')
   ));
-};
+});
 
 module.exports.buildTelegramCIDR = buildTelegramCIDR;