Feat: validate domains are alive (and w/ CI)

.github/workflows/check-source-domain.yml

@@ -0,0 +1,24 @@
+name: Check Domain Availability
+on:
+  # manual trigger only
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ".node-version"
+          cache: "pnpm"
+      - run: pnpm install
+      - run: pnpm run node Build/validate-domain-alive.ts

.github/workflows/main.yml

@@ -16,16 +16,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
           persist-credentials: false
       - uses: pnpm/action-setup@v4
-        name: Install pnpm
         with:
           run_install: false
-      - name: Use Node.js
-        uses: actions/setup-node@v4
+      - uses: actions/setup-node@v4
         with:
           node-version-file: ".node-version"
           cache: "pnpm"
@@ -54,8 +51,7 @@ jobs:
             ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-
             ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-
             ${{ runner.os }}-v3-
-      - name: Install dependencies
-        run: pnpm install
+      - run: pnpm install
       - run: pnpm run build
       - name: Pre-deploy check
         # If the public directory doesn't exist, the build should fail.

.gitignore

@@ -4,4 +4,4 @@ node_modules
 .wireit
 .cache
 public
-tmp*
+tmp.*

Build/mod.d.ts

@@ -0,0 +1,22 @@
+import 'dns2';
+
+declare module 'dns2' {
+  import DNS from 'dns2';
+
+  declare namespace DNS {
+    interface DoHClientOption {
+      /** @example dns.google.com */
+      dns: string,
+      /** @description whether to use HTTP or HTTPS */
+      http: boolean
+    }
+
+    export type DnsResolver<T = DnsResponse> = (name: string, type: PacketQuestion) => Promise<T>;
+
+    declare function DOHClient(opt: DoHClientOption): DnsResolver;
+
+    export type $DnsResponse = DnsResponse;
+  }
+
+  export = DNS;
+}

Build/validate-domain-alive.ts

@@ -0,0 +1,236 @@
+import DNS2 from 'dns2';
+import { readFileByLine } from './lib/fetch-text-by-line';
+import { processLine } from './lib/process-line';
+import tldts from 'tldts';
+import { looseTldtsOpt } from './constants/loose-tldts-opt';
+import { fdir as Fdir } from 'fdir';
+import { SOURCE_DIR } from './constants/dir';
+import path from 'node:path';
+import { newQueue } from '@henrygd/queue';
+import asyncRetry from 'async-retry';
+import * as whoiser from 'whoiser';
+import picocolors from 'picocolors';
+
+const dohServers: Array<[string, DNS2.DnsResolver]> = ([
+  '8.8.8.8',
+  '8.8.4.4',
+  '1.0.0.1',
+  '1.1.1.1',
+  '162.159.36.1',
+  '162.159.46.1',
+  '101.101.101.101', // TWNIC
+  '185.222.222.222', // DNS.SB
+  '45.11.45.11', // DNS.SB
+  '9.9.9.10', // Quad9 unfiltered
+  '149.112.112.10', // Quad9 unfiltered
+  '208.67.222.2', // OpenDNS sandbox (unfiltered)
+  '208.67.220.2', // OpenDNS sandbox (unfiltered)
+  '94.140.14.140', // AdGuard unfiltered
+  '94.140.14.141', // AdGuard unfiltered
+  // '76.76.2.0', // ControlD unfiltered, path not /dns-query
+  // '76.76.10.0', // ControlD unfiltered, path not /dns-query
+  '193.110.81.0', // dns0.eu
+  '185.253.5.0', // dns0.eu
+  'dns.nextdns.io',
+  'wikimedia-dns.org',
+  // 'ordns.he.net',
+  'dns.mullvad.net'
+  // 'ada.openbld.net',
+  // 'dns.rabbitdns.org'
+] as const).map(server => [
+  server,
+  DNS2.DOHClient({
+    dns: server,
+    http: false
+  })
+] as const);
+
+const queue = newQueue(8);
+
+class DnsError extends Error {
+  name = 'DnsError';
+  constructor(readonly message: string, public readonly server: string) {
+    super(message);
+  }
+}
+
+interface DnsResponse extends DNS2.$DnsResponse {
+  dns: string
+}
+
+const resolve: DNS2.DnsResolver<DnsResponse> = async (...args) => {
+  try {
+    return await asyncRetry(async () => {
+      const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)];
+
+      try {
+        const resp = await dohClient(...args);
+        return {
+          ...resp,
+          dns: dohServer
+        } satisfies DnsResponse;
+      } catch (e) {
+        throw new DnsError((e as Error).message, dohServer);
+      }
+    }, { retries: 5 });
+  } catch (e) {
+    console.log('[doh error]', ...args, e);
+    throw e;
+  }
+};
+
+(async () => {
+  const domainSets = await new Fdir()
+    .withFullPaths()
+    .crawl(SOURCE_DIR + path.sep + 'domainset')
+    .withPromise();
+  const domainRules = await new Fdir()
+    .withFullPaths()
+    .crawl(SOURCE_DIR + path.sep + 'non_ip')
+    .withPromise();
+
+  await Promise.all([
+    ...domainSets.map(runAgainstDomainset),
+    ...domainRules.map(runAgainstRuleset)
+  ]);
+
+  console.log('done');
+})();
+
+const domainAliveMap = new Map<string, boolean>();
+async function isApexDomainAlive(apexDomain: string): Promise<[string, boolean]> {
+  if (domainAliveMap.has(apexDomain)) {
+    return [apexDomain, domainAliveMap.get(apexDomain)!];
+  }
+
+  const resp = await resolve(apexDomain, 'NS');
+
+  if (resp.answers.length > 0) {
+    console.log(picocolors.green('[domain alive]'), 'NS record', apexDomain);
+    return [apexDomain, true];
+  }
+
+  let whois;
+
+  try {
+    whois = await whoiser.domain(apexDomain);
+  } catch (e) {
+    console.log('[whois fail]', 'whois error', { domain: apexDomain }, e);
+    return [apexDomain, true];
+  }
+
+  if (Object.keys(whois).length > 0) {
+    // TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117
+    if ('text' in whois && (whois.text as string[]).some(value => value.includes('No match for'))) {
+      console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain });
+      domainAliveMap.set(apexDomain, false);
+      return [apexDomain, false];
+    }
+
+    console.log(picocolors.green('[domain alive]'), 'recorded in whois', apexDomain);
+    return [apexDomain, true];
+  }
+
+  if (!('dns' in whois)) {
+    console.log({ whois });
+  }
+
+  console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain });
+  domainAliveMap.set(apexDomain, false);
+  return [apexDomain, false];
+}
+
+const domainMutex = new Map<string, Promise<[string, boolean]>>();
+
+export async function isDomainAlive(domain: string, isSuffix: boolean): Promise<[string, boolean]> {
+  // const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)];
+
+  if (domain[0] === '.') {
+    domain = domain.slice(1);
+  }
+
+  const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
+  if (!apexDomain) {
+    console.log('[domain invalid]', 'no apex domain', { domain });
+    return [domain, true] as const;
+  }
+
+  let apexDomainAlivePromise;
+  if (domainMutex.has(domain)) {
+    apexDomainAlivePromise = domainMutex.get(domain)!;
+  } else {
+    apexDomainAlivePromise = queue.add(() => isApexDomainAlive(apexDomain).then(res => {
+      domainMutex.delete(domain);
+      return res;
+    }));
+    domainMutex.set(domain, apexDomainAlivePromise);
+  }
+  const apexDomainAlive = await apexDomainAlivePromise;
+
+  if (!apexDomainAlive[1]) {
+    domainAliveMap.set(domain, false);
+    return [domain, false] as const;
+  }
+
+  if (!isSuffix) {
+    const aRecords = (await resolve(domain, 'A'));
+    if (aRecords.answers.length === 0) {
+      const aaaaRecords = (await resolve(domain, 'AAAA'));
+      if (aaaaRecords.answers.length === 0) {
+        console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aRecords.dns, aaaa: aaaaRecords.dns });
+        domainAliveMap.set(domain, false);
+        return [domain, false] as const;
+      }
+    }
+  }
+
+  return [domain, true] as const;
+}
+
+export async function runAgainstRuleset(filepath: string) {
+  const promises: Array<Promise<[string, boolean]>> = [];
+
+  for await (const l of readFileByLine(filepath)) {
+    const line = processLine(l);
+    if (!line) continue;
+    const [type, domain] = line.split(',');
+    switch (type) {
+      case 'DOMAIN-SUFFIX':
+      case 'DOMAIN': {
+        if (!domainMutex.has(domain)) {
+          const promise = queue.add(() => isDomainAlive(domain, type === 'DOMAIN-SUFFIX')).then(res => {
+            domainMutex.delete(domain);
+            return res;
+          });
+          domainMutex.set(domain, promise);
+          promises.push(promise);
+        }
+        break;
+      }
+      // no default
+      // case 'DOMAIN-KEYWORD': {
+      //   break;
+      // }
+      // no default
+    }
+  }
+
+  return Promise.all(promises);
+}
+
+export async function runAgainstDomainset(filepath: string) {
+  const promises: Array<Promise<[string, boolean]>> = [];
+
+  for await (const l of readFileByLine(filepath)) {
+    const line = processLine(l);
+    if (!line) continue;
+    const apexDomain = tldts.getDomain(line, looseTldtsOpt);
+    if (apexDomain) {
+      promises.push(isDomainAlive(apexDomain, line[0] === '.'));
+    } else {
+      console.log('[domain invalid]', 'no apex domain', { line });
+    }
+  }
+
+  return Promise.all(promises);
+}

Source/domainset/cdn.conf

@@ -683,7 +683,7 @@ image.ibb.co
 .ax1x.com
 # PostImage
 .postimg.cc
-.postimg.org
+# .postimg.org - domain locked by registry since no later than Apr. 2018 (https://web.archive.org/web/20190208000038/https://www.phpbb.com/customise/db/extension/postimage/support/topic/191346)
 # Image Proxy
 images.weserv.nl
 # Imageshack

package.json

@@ -21,7 +21,9 @@
   "license": "ISC",
   "dependencies": {
     "@ghostery/adblocker": "^2.0.3",
+    "@henrygd/queue": "^1.0.7",
     "@jsdevtools/ez-spawn": "^3.0.4",
+    "async-retry": "^1.3.3",
     "async-sema": "^3.1.1",
     "better-sqlite3": "^11.5.0",
     "cacache": "^19.0.1",
@@ -30,6 +32,7 @@
     "cli-table3": "^0.6.5",
     "csv-parse": "^5.5.6",
     "devalue": "^5.1.1",
+    "dns2": "^2.1.0",
     "fast-cidr-tools": "^0.3.1",
     "fdir": "^6.4.2",
     "foxact": "^0.2.41",
@@ -43,15 +46,18 @@
     "tldts": "^6.1.58",
     "tldts-experimental": "^6.1.58",
     "undici": "6.20.1",
+    "whoiser": "^1.18.0",
     "why-is-node-running": "^3.2.1",
     "yaml": "^2.6.0"
   },
   "devDependencies": {
     "@eslint-sukka/node": "^6.9.0",
     "@swc-node/register": "^1.10.9",
     "@swc/core": "^1.7.42",
+    "@types/async-retry": "^1.4.9",
     "@types/better-sqlite3": "^7.6.11",
     "@types/cacache": "^17.0.2",
+    "@types/dns2": "^2.0.9",
     "@types/make-fetch-happen": "^10.0.4",
     "@types/mocha": "^10.0.9",
     "@types/node": "^22.8.7",