diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index beb595d73..172219884 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -59,9 +59,14 @@ const querySpeedtestApi = async (keyword: string): Promise> retry: { retries: 2 } - })).then(r => r.json() as any).then((data: Array<{ url: string }>) => data.reduce( + })).then(r => r.json() as any).then((data: Array<{ url: string, host: string }>) => data.reduce( (prev, cur) => { - const hn = getHostname(cur.url, { detectIp: false, validateHostname: true }); + const line = cur.host || cur.url; + const hn = getHostname(line, { detectIp: false, validateHostname: true }); + if (line.includes('8080')) { + console.log({ line, hn, flag: 1 }); + return prev; + } if (hn) { prev.push(hn); } @@ -146,6 +151,8 @@ export const buildSpeedtestDomainSet = task(import.meta.main, import.meta.path)( '.speedtest.com.sg', '.ookla.ddnsgeek.com', '.speedtest.pni.tw', + '.speedtest.cmcnetworks.net', + '.speedtestwnet.com.br', // Cloudflare '.speed.cloudflare.com', // Wi-Fi Man @@ -178,14 +185,18 @@ export const buildSpeedtestDomainSet = task(import.meta.main, import.meta.path)( 'fetch previous speedtest domainset', async () => { try { - const contents = await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../List/domainset/speedtest.conf')); - contents.reduce((acc, line) => { - const hn = getHostname(line, { detectIp: false, validateHostname: true }); - if (hn) { - acc.push(hn); - } - return acc; - }, []).forEach(domainTrie.add); + (await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../List/domainset/speedtest.conf'))) + .forEach(line => { + const hn = getHostname(line, { detectIp: false, validateHostname: true }); + if (line.includes(':')) { + console.log({ line, hn, flag: 2 }); + return; + } + + if (hn) { + domainTrie.add(hn); + } + }); } catch { } } ); @@ -225,6 +236,11 @@ export const buildSpeedtestDomainSet = task(import.meta.main, import.meta.path)( pMap[keyword] = span.traceChildAsync(`fetch speedtest endpoints: ${keyword}`, () => querySpeedtestApi(keyword)).then(hostnameGroup => { return hostnameGroup.forEach(hostname => { if (hostname) { + if (hostname.includes(':')) { + console.log({ hostname, flag: 3 }); + return; + } + domainTrie.add(hostname); } }); @@ -250,6 +266,11 @@ export const buildSpeedtestDomainSet = task(import.meta.main, import.meta.path)( const deduped = span.traceChildSync('sort result', () => sortDomains(domainDeduper(domainTrie))); + console.log({ + trie: domainTrie.has('ourtestisbest.brolve.net:8080'), + deduped: deduped.some(line => line.includes(':')) + }); + const description = [ ...SHARED_DESCRIPTION, '', @@ -261,6 +282,8 @@ export const buildSpeedtestDomainSet = task(import.meta.main, import.meta.path)( 'Sukka\'s Ruleset - Speedtest Domains', description, new Date(), + /* The `deduped` variable is created by sorting and deduplicating the list of domains obtained from + the `domainTrie` data structure. */ deduped, 'domainset', path.resolve(import.meta.dir, '../List/domainset/speedtest.conf'),