From 50ca0c5e9e511118fa67b2a068d40226d47478d4 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 6 Aug 2024 19:10:29 +0800 Subject: [PATCH] Feat: implement Clash Meta mrs format --- Build/build-apple-cdn.ts | 3 +- Build/build-cdn-download-conf.ts | 6 ++- Build/build-common.ts | 4 +- Build/build-reject-domainset.ts | 6 ++- Build/build-speedtest-domainset.ts | 3 +- Build/lib/convert-clash-meta-mrs.ts | 57 +++++++++++++++++++++++++++++ Build/lib/create-file.ts | 19 ++++++++-- package.json | 1 + pnpm-lock.yaml | 25 +++++++++++++ 9 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 Build/lib/convert-clash-meta-mrs.ts diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index 4302634e4..19c9d259f 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -53,7 +53,8 @@ export const buildAppleCdn = task(require.main === module, __filename)(async (sp domainset, 'domainset', path.resolve(__dirname, '../List/domainset/apple_cdn.conf'), - path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt') + path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt'), + path.resolve(__dirname, '../Clash/clash_mrs_domain/apple_cdn.mrs') ) ]); }); diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index 8d9709819..8c829acdf 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -78,7 +78,8 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as sortDomains(domainDeduper(cdnDomainsList)), 'domainset', path.resolve(__dirname, '../List/domainset/cdn.conf'), - path.resolve(__dirname, '../Clash/domainset/cdn.txt') + path.resolve(__dirname, '../Clash/domainset/cdn.txt'), + path.resolve(__dirname, '../Clash/clash_mrs_domain/cdn.mrs') ), createRuleset( span, @@ -92,7 +93,8 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as sortDomains(domainDeduper(downloadDomainSet)), 'domainset', path.resolve(__dirname, '../List/domainset/download.conf'), - path.resolve(__dirname, '../Clash/domainset/download.txt') + path.resolve(__dirname, '../Clash/domainset/download.txt'), + path.resolve(__dirname, '../Clash/clash_mrs_domain/download.mrs') ) ]); }); diff --git a/Build/build-common.ts b/Build/build-common.ts index b3ae39936..aa8d6b89b 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -127,6 +127,8 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: description = SHARED_DESCRIPTION; } + const clashFileBasename = relativePath.slice(0, -path.extname(relativePath).length); + return createRuleset( span, title, @@ -135,7 +137,7 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: deduped, 'domainset', path.resolve(outputSurgeDir, relativePath), - path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`) + path.resolve(outputClashDir, `${clashFileBasename}.txt`) ); } ); diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index a6dc4edba..e2e399b4f 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -191,7 +191,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as span.traceChildSync('sort reject domainset (base)', () => sortDomains(dudupedDominArray, domainArrayMainDomainMap, domainArraySubdomainMap)), 'domainset', path.resolve(__dirname, '../List/domainset/reject.conf'), - path.resolve(__dirname, '../Clash/domainset/reject.txt') + path.resolve(__dirname, '../Clash/domainset/reject.txt'), + path.resolve(__dirname, '../Clash/clash_mrs_domain/reject.mrs') ), createRuleset( span, @@ -211,7 +212,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as span.traceChildSync('sort reject domainset (extra)', () => sortDomains(dudupedDominArrayExtra, domainArrayMainDomainMap, domainArraySubdomainMap)), 'domainset', path.resolve(__dirname, '../List/domainset/reject_extra.conf'), - path.resolve(__dirname, '../Clash/domainset/reject_extra.txt') + path.resolve(__dirname, '../Clash/domainset/reject_extra.txt'), + path.resolve(__dirname, '../Clash/clash_mrs_domain/reject_extra.mrs') ), compareAndWriteFile( span, diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index ffe502aec..a1664316f 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -251,6 +251,7 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename) deduped, 'domainset', path.resolve(__dirname, '../List/domainset/speedtest.conf'), - path.resolve(__dirname, '../Clash/domainset/speedtest.txt') + path.resolve(__dirname, '../Clash/domainset/speedtest.txt'), + path.resolve(__dirname, '../Clash/clash_mrs_domain/speedtest.mrs') ); }); diff --git a/Build/lib/convert-clash-meta-mrs.ts b/Build/lib/convert-clash-meta-mrs.ts new file mode 100644 index 000000000..751fb12fd --- /dev/null +++ b/Build/lib/convert-clash-meta-mrs.ts @@ -0,0 +1,57 @@ +import path from 'path'; +import fs from 'fs'; +import fsp from 'fs/promises'; +import { Readable } from 'stream'; +import { pipeline } from 'stream/promises'; +import zlib from 'zlib'; +import { async as ezspawn } from '@jsdevtools/ez-spawn'; + +const mihomoBinaryDir = path.join(__dirname, '../../node_modules/.cache/mihomo'); +const mihomoBinaryPath = path.join(mihomoBinaryDir, 'mihomo'); + +const mihomoBinaryUrl: Partial>>> = { + linux: { + x64: 'https://github.com/MetaCubeX/mihomo/releases/download/v1.18.7/mihomo-linux-amd64-compatible-v1.18.7.gz' + }, + darwin: { + x64: 'https://github.com/MetaCubeX/mihomo/releases/download/v1.18.7/mihomo-darwin-amd64-v1.18.7.gz', + arm64: 'https://github.com/MetaCubeX/mihomo/releases/download/v1.18.7/mihomo-darwin-arm64-v1.18.7.gz' + } +}; + +const ensureMihomoBinary = async () => { + await fsp.mkdir(mihomoBinaryDir, { recursive: true }); + if (!fs.existsSync(mihomoBinaryPath)) { + const writeStream = fs.createWriteStream(mihomoBinaryPath); + + const downloadUrl = mihomoBinaryUrl[process.platform]?.[process.arch]; + if (!downloadUrl) { + throw new Error(`Unsupported platform: ${process.platform} ${process.arch}`); + } + + const res = await fetch(downloadUrl); + + if (!res.ok || !res.body) { + throw new Error(`Failed to download mihomo binary: ${res.statusText}`); + } + + const gunzip = zlib.createGunzip(); + + await pipeline( + Readable.fromWeb(res.body), + gunzip, + writeStream + ); + } + await fsp.chmod(mihomoBinaryPath, 0o755); +}; + +export const convertClashMetaMrs = async (type: 'domain', format: 'text', input: string, output: string) => { + await ensureMihomoBinary(); + + const { stderr } = await ezspawn(mihomoBinaryPath, ['convert-ruleset', type, format, input, output]); + + if (stderr) { + throw new Error(stderr); + } +}; diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index 937793d89..2201d9666 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -151,8 +151,10 @@ const MARK = 'this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'; export const createRuleset = ( parentSpan: Span, title: string, description: string[] | readonly string[], date: Date, content: string[], - type: ('ruleset' | 'domainset' | string & {}), surgePath: string, clashPath: string -) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn((childSpan) => { + type: ('ruleset' | 'domainset' | string & {}), + surgePath: string, clashPath: string, + clashMrsPath?: string +) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn(async (childSpan) => { const surgeContent = withBannerArray( title, description, date, sortRuleSet(type === 'domainset' @@ -174,8 +176,19 @@ export const createRuleset = ( return withBannerArray(title, description, date, _clashContent); }); - return Promise.all([ + await Promise.all([ compareAndWriteFile(childSpan, surgeContent, surgePath), compareAndWriteFile(childSpan, clashContent, clashPath) ]); + + // if (clashMrsPath) { + // if (type === 'domainset') { + // await childSpan.traceChildAsync('clash meta mrs domain ' + clashMrsPath, async () => { + // await fs.promises.mkdir(path.dirname(clashMrsPath), { recursive: true }); + // await convertClashMetaMrs( + // 'domain', 'text', clashPath, clashMrsPath + // ); + // }); + // } + // } }); diff --git a/package.json b/package.json index 5b5c3d1ea..5bc37cee6 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "license": "ISC", "dependencies": { "@cliqz/adblocker": "^1.31.1", + "@jsdevtools/ez-spawn": "^3.0.4", "async-retry": "^1.3.3", "async-sema": "^3.1.1", "better-sqlite3": "^11.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd00b1deb..0ae6d945a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@cliqz/adblocker': specifier: ^1.31.1 version: 1.31.1 + '@jsdevtools/ez-spawn': + specifier: ^3.0.4 + version: 3.0.4 async-retry: specifier: ^1.3.3 version: 1.3.3 @@ -188,6 +191,10 @@ packages: resolution: {integrity: sha512-d2CGZR2o7fS6sWB7DG/3a95bGKQyHMACZ5aW8qGkkqQpUoZV6C0X7Pc7l4ZNMZkfNBf4VWNe9E1jRsf0G146Ew==} engines: {node: '>=18.18'} + '@jsdevtools/ez-spawn@3.0.4': + resolution: {integrity: sha512-f5DRIOZf7wxogefH03RjMPMdBF7ADTWUMoOs9kaJo06EfwF+aFhMZMDZxHg/Xe12hptN9xoZjGso2fdjapBRIA==} + engines: {node: '>=10'} + '@napi-rs/wasm-runtime@0.2.4': resolution: {integrity: sha512-9zESzOO5aDByvhIAsOy9TbpZ0Ur2AJbUI7UT73kcUTS2mxAMHOBaa1st/jAymNoCtvrit99kkzT1FZuXVcgfIQ==} @@ -615,6 +622,9 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + call-me-maybe@1.0.2: + resolution: {integrity: sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -1363,6 +1373,10 @@ packages: streamx@2.18.0: resolution: {integrity: sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==} + string-argv@0.3.2: + resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} + engines: {node: '>=0.6.19'} + string-similarity@4.0.4: resolution: {integrity: sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. @@ -1634,6 +1648,13 @@ snapshots: '@humanwhocodes/retry@0.3.0': {} + '@jsdevtools/ez-spawn@3.0.4': + dependencies: + call-me-maybe: 1.0.2 + cross-spawn: 7.0.3 + string-argv: 0.3.2 + type-detect: 4.0.8 + '@napi-rs/wasm-runtime@0.2.4': dependencies: '@emnapi/core': 1.2.0 @@ -2076,6 +2097,8 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + call-me-maybe@1.0.2: {} + callsites@3.1.0: {} camelcase@6.3.0: {} @@ -2884,6 +2907,8 @@ snapshots: optionalDependencies: bare-events: 2.4.2 + string-argv@0.3.2: {} + string-similarity@4.0.4: {} string-width@4.2.3: