Build ruleset and geodata #990
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build ruleset and geodata | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 19 * * *" | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "**/README.md" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set variables | |
| run: | | |
| echo "update_version=$(date -d '+8 hours' +%Y-%m-%d)" >> ${GITHUB_ENV} | |
| echo "domains_download_url=https://raw.githubusercontent.com/DustinWin/domain-list-custom/domains" >> ${GITHUB_ENV} | |
| echo "ips_download_url=https://github.com/DustinWin/geoip/releases/download/ips" >> ${GITHUB_ENV} | |
| shell: bash | |
| - name: Clone Repository | |
| uses: actions/checkout@master | |
| - name: Checkout DustinWin/domain-list-custom | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: DustinWin/domain-list-custom | |
| path: custom | |
| - name: Checkout v2fly/domain-list-community | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: v2fly/domain-list-community | |
| path: community | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./custom/go.mod | |
| cache-dependency-path: ./custom/go.sum | |
| - name: Generate `clash` geodata | |
| run: | | |
| mkdir -p ./community/mydata/ | |
| archs1=(private ads microsoft-cn apple-cn google-cn games-cn ai networktest netflix disney max primevideo appletv youtube tiktok bilibili proxy cn) | |
| archs2=(private ads microsoft-cn apple-cn google-cn games-cn ai networktest proxy cn) | |
| # Generate `clash` geosite-all.dat | |
| for ((i = 0; i < 18; i++)); do | |
| curl -sSL "${domains_download_url}/${archs1[i]}.list" | grep -E 'DOMAIN,|DOMAIN-SUFFIX,|DOMAIN-KEYWORD' | sed -e 's/^DOMAIN,/full:/' -e 's/^DOMAIN-SUFFIX,//' -e 's/^DOMAIN-KEYWORD,/keyword:/' > "./community/mydata/${archs1[i]}" | |
| done | |
| curl -sSL "${domains_download_url}/fakeip-filter.list" | sed -e 's/^DOMAIN,/full:/' -e 's/^DOMAIN-SUFFIX,//' -e 's/^DOMAIN-KEYWORD,/keyword:/' -e 's/^DOMAIN-REGEX,/regexp:/' > ./community/mydata/fakeip-filter | |
| curl -sSL "${domains_download_url}/fakeip-filter-lite.list" | grep -Ev 'DOMAIN-REGEX,|ntp|time|stun' | sed -e 's/^DOMAIN,/full:/' -e 's/^DOMAIN-SUFFIX,//' > ./community/mydata/fakeip-filter-lite | |
| cat <<EOF >> ./community/mydata/fakeip-filter-lite | |
| keyword:ntp | |
| keyword:time | |
| keyword:stun | |
| EOF | |
| cd ./community/ | |
| go run ./ --datapath=./mydata/ --outputname geosite-all.dat | |
| # Generate `clash` geosite-all-lite.dat | |
| rm -f ./mydata/ads | |
| go run ./ --datapath=./mydata/ --outputname geosite-all-lite.dat | |
| # Generate `clash` geosite.dat | |
| rm -f ./mydata/* | |
| for ((i = 0; i < 10; i++)); do | |
| curl -sSL "${domains_download_url}/${archs2[i]}.list" | grep -E 'DOMAIN,|DOMAIN-SUFFIX,|DOMAIN-KEYWORD' | sed -e 's/^DOMAIN,/full:/' -e 's/^DOMAIN-SUFFIX,//' -e 's/^DOMAIN-KEYWORD,/keyword:/' > "./mydata/${archs2[i]}" | |
| done | |
| curl -sSL "${domains_download_url}/fakeip-filter.list" | sed -e 's/^DOMAIN,/full:/' -e 's/^DOMAIN-SUFFIX,//' -e 's/^DOMAIN-KEYWORD,/keyword:/' -e 's/^DOMAIN-REGEX,/regexp:/' > ./mydata/fakeip-filter | |
| curl -sSL "${domains_download_url}/fakeip-filter-lite.list" | grep -Ev 'DOMAIN-REGEX,|ntp|time|stun' | sed -e 's/^DOMAIN,/full:/' -e 's/^DOMAIN-SUFFIX,//' > ./mydata/fakeip-filter-lite | |
| cat <<EOF >> ./mydata/fakeip-filter-lite | |
| keyword:ntp | |
| keyword:time | |
| keyword:stun | |
| EOF | |
| go run ./ --datapath=./mydata/ --outputname geosite.dat | |
| # Generate `clash` geosite-lite.dat | |
| rm -f ./mydata/ads | |
| go run ./ --datapath=./mydata/ --outputname geosite-lite.dat | |
| - name: Generate `sing-box` geodata | |
| run: | | |
| cd ./community/ || exit 1 | |
| go install -trimpath -ldflags="-s -w -buildid=" github.com/metacubex/geo/cmd/geo@master | |
| for file in $(ls *.dat | sed 's/\.dat$//'); do | |
| geo convert site -i v2ray -o sing -f "./${file}.db" "./${file}.dat" | |
| done | |
| - name: Get geoip relative files | |
| run: | | |
| mkdir -p ./clash/ ./sing-box/ | |
| # Download Clash geoip files (.mmdb) | |
| clash_mmdb=($(curl -sSL https://api.github.com/repos/DustinWin/geoip/releases | grep '"browser_download_url"' | grep '.mmdb' | awk -F'"' '{print $4}')) | |
| count_clash_mmdb=${#clash_mmdb[@]} | |
| for ((i = 0; i < count_clash_mmdb; i++)); do | |
| wget -P ./clash/ "${clash_mmdb[i]}" | |
| done | |
| # Download Clash other geoip files (except .mmdb) | |
| clash_others=($(curl -sSL https://api.github.com/repos/DustinWin/geoip/releases | grep '"browser_download_url"' | grep '/clash/' | grep -v '.mmdb' | awk -F'"' '{print $4}')) | |
| count_clash_others=${#clash_others[@]} | |
| for ((i = 0; i < count_clash_others; i++)); do | |
| wget -P ./community/ "${clash_others[i]}" | |
| done | |
| # Download sing-box all geoip files | |
| singbox_all=($(curl -sSL https://api.github.com/repos/DustinWin/geoip/releases | grep '"browser_download_url"' | grep '/sing-box/' | awk -F'"' '{print $4}')) | |
| count_singbox_all=${#singbox_all[@]} | |
| for ((i = 0; i < count_singbox_all; i++)); do | |
| wget -P ./community/ "${singbox_all[i]}" | |
| done | |
| - name: Move `clash` and `sing-box` geodata files | |
| run: | | |
| cd ./community/ || exit 1 | |
| for file in $(ls *.dat *.metadb); do | |
| install -Dp "./${file}" ../clash/ | |
| done | |
| for file in $(ls *.db); do | |
| install -Dp "./${file}" ../sing-box/ | |
| done | |
| rm -rf ../custom* ../community* | |
| - name: Generate `clash` rule-set | |
| run: | | |
| mkdir -p ./clash-ruleset/ | |
| domains=(private ads microsoft-cn apple-cn google-cn games-cn ai netflix disney max primevideo appletv youtube tiktok bilibili proxy cn) | |
| archs=(applications networktest) | |
| ips=(netflixip telegramip privateip cnip) | |
| # Generate `clash` rule-set (yaml) | |
| for ((i = 0; i < 17; i++)); do | |
| echo 'payload:' > "./clash-ruleset/${domains[i]}.yaml" | |
| curl -sSL "${domains_download_url}/${domains[i]}.list" | grep -E 'DOMAIN,|DOMAIN-SUFFIX,' | sed -e "s/^DOMAIN,/ - '/" -e "s/^DOMAIN-SUFFIX,/ - '+\./" -e "s/$/'/" >> "./clash-ruleset/${domains[i]}.yaml" | |
| done | |
| echo 'payload:' > "./clash-ruleset/fakeip-filter.yaml" | |
| curl -sSL "${domains_download_url}/fakeip-filter.list" | sed -e "s/^DOMAIN,/ - '/" -e "s/^DOMAIN-SUFFIX,/ - '+\./" -e "s/^DOMAIN-REGEX,/ - '/" -e "s/'\./'/" -e "s/$/'/" >> ./clash-ruleset/fakeip-filter.yaml | |
| echo 'payload:' > "./clash-ruleset/fakeip-filter-lite.yaml" | |
| curl -sSL "${domains_download_url}/fakeip-filter-lite.list" | sed -e "s/^DOMAIN,/ - '/" -e "s/^DOMAIN-SUFFIX,/ - '+\./" -e "s/^DOMAIN-REGEX,/ - '/" -e "s/'\./'/" -e "s/$/'/" >> ./clash-ruleset/fakeip-filter-lite.yaml | |
| for ((i = 0; i < 2; i++)); do | |
| echo 'payload:' > "./clash-ruleset/${archs[i]}.yaml" | |
| curl -sSL "${domains_download_url}/${archs[i]}.list" | sed 's/^/ - /' >> "./clash-ruleset/${archs[i]}.yaml" | |
| done | |
| for ((i = 0; i < 4; i++)); do | |
| echo 'payload:' > "./clash-ruleset/${ips[i]}.yaml" | |
| curl -sSL "${ips_download_url}/${ips[i]}.list" | grep 'IP-CIDR' | sed -e "s/^IP-CIDR,/ - '/" -e "s/^IP-CIDR6,/ - '/" -e "s/$/'/" >> "./clash-ruleset/${ips[i]}.yaml" | |
| done | |
| # Generate `clash` rule-set (text) | |
| for ((i = 0; i < 17; i++)); do | |
| curl -sSL "${domains_download_url}/${domains[i]}.list" | grep -E 'DOMAIN,|DOMAIN-SUFFIX,' | sed -e 's/^DOMAIN,//' -e 's/^DOMAIN-SUFFIX,/+\./' > "./clash-ruleset/${domains[i]}.list" | |
| done | |
| curl -sSL "${domains_download_url}/fakeip-filter.list" | sed -e 's/^DOMAIN,//' -e 's/^DOMAIN-SUFFIX,/+\./' -e 's/^DOMAIN-REGEX,//' -e 's/^\.//' > ./clash-ruleset/fakeip-filter.list | |
| curl -sSL "${domains_download_url}/fakeip-filter-lite.list" | sed -e 's/^DOMAIN,//' -e 's/^DOMAIN-SUFFIX,/+\./' -e 's/^DOMAIN-REGEX,//' -e 's/^\.//' > ./clash-ruleset/fakeip-filter-lite.list | |
| for ((i = 0; i < 2; i++)); do | |
| curl -sSL "${domains_download_url}/${archs[i]}.list" > "./clash-ruleset/${archs[i]}.list" | |
| done | |
| for ((i = 0; i < 4; i++)); do | |
| curl -sSL "${ips_download_url}/${ips[i]}.list" | grep 'IP-CIDR' | sed -e 's/^IP-CIDR,//' -e 's/^IP-CIDR6,//' > "./clash-ruleset/${ips[i]}.list" | |
| done | |
| # Ready for generate `clash` rule-set (mrs) | |
| for ((i = 0; i < 17; i++)); do | |
| mkdir -p "./tools/domains/${domains[i]}/" | |
| curl "${domains_download_url}/${domains[i]}.list" | grep -E 'DOMAIN,|DOMAIN-SUFFIX,' | sed -e 's/^DOMAIN,//' -e 's/^DOMAIN-SUFFIX,/+\./' > "./tools/domains/${domains[i]}/${domains[i]}.list" | |
| done | |
| mkdir -p "./tools/domains/fakeip-filter/" | |
| curl -sSL "${domains_download_url}/fakeip-filter.list" | sed -e 's/^DOMAIN,//' -e 's/^DOMAIN-SUFFIX,/+\./' -e 's/^DOMAIN-REGEX,//' -e 's/^\.//' > ./tools/domains/fakeip-filter/fakeip-filter.list | |
| mkdir -p "./tools/domains/fakeip-filter-lite/" | |
| curl -sSL "${domains_download_url}/fakeip-filter-lite.list" | sed -e 's/^DOMAIN,//' -e 's/^DOMAIN-SUFFIX,/+\./' -e 's/^DOMAIN-REGEX,//' -e 's/^\.//' > ./tools/domains/fakeip-filter-lite/fakeip-filter-lite.list | |
| for ((i = 0; i < 4; i++)); do | |
| mkdir -p "./tools/ips/${ips[i]}/" | |
| curl -sSL "${ips_download_url}/${ips[i]}.list"| grep 'IP-CIDR' | sed -e 's/^IP-CIDR,//' -e 's/^IP-CIDR6,//' > "./tools/ips/${ips[i]}/${ips[i]}.list" | |
| done | |
| # Ready for generate `sing-box` rule_set | |
| for ((i = 0; i < 17; i++)); do | |
| mkdir -p "./tools/rules/${domains[i]}/" | |
| curl "${domains_download_url}/${domains[i]}.list" | grep -E 'DOMAIN,|DOMAIN-SUFFIX,|DOMAIN-KEYWORD' > "./tools/rules/${domains[i]}/${domains[i]}.yaml" | |
| done | |
| mkdir -p ./tools/rules/fakeip-filter/ | |
| curl "${domains_download_url}/fakeip-filter.list" > ./tools/rules/fakeip-filter/fakeip-filter.yaml | |
| mkdir -p ./tools/rules/fakeip-filter-lite/ | |
| curl "${domains_download_url}/fakeip-filter-lite.list" | grep -Ev 'DOMAIN-REGEX,|ntp|time|stun' > ./tools/rules/fakeip-filter-lite/fakeip-filter-lite.yaml | |
| cat <<EOF >> ./tools/rules/fakeip-filter-lite/fakeip-filter-lite.yaml | |
| DOMAIN-KEYWORD,ntp | |
| DOMAIN-KEYWORD,time | |
| DOMAIN-KEYWORD,stun | |
| EOF | |
| for ((i = 0; i < 2; i++)); do | |
| mkdir -p "./tools/rules/${archs[i]}/" | |
| curl "${domains_download_url}/${archs[i]}.list" > "./tools/rules/${archs[i]}/${archs[i]}.yaml" | |
| done | |
| for ((i = 0; i < 4; i++)); do | |
| mkdir -p "./tools/rules/${ips[i]}/" | |
| curl -sSL "${ips_download_url}/${ips[i]}.list"| grep 'IP-CIDR' > "./tools/rules/${ips[i]}/${ips[i]}.yaml" | |
| done | |
| - name: Download and unzip `mihomo` core | |
| run: | | |
| wget https://github.com/DustinWin/clash_singbox-tools/releases/download/mihomo/mihomo-alpha-linux-amd64.tar.gz -O - | tar -zxf - -C ./tools/ | |
| mv -f ./tools/CrashCore ./tools/mihomo | |
| - name: Generate `clash` rule-set (mrs) | |
| run: | | |
| cd ./tools/ | |
| chmod +x ./mihomo | |
| list=($(ls ./domains/)) | |
| for ((i = 0; i < ${#list[@]}; i++)); do | |
| ./mihomo convert-ruleset domain text "./domains/${list[i]}/${list[i]}.list" ../clash-ruleset/${list[i]}.mrs | |
| done | |
| list=($(ls ./ips/)) | |
| for ((i = 0; i < ${#list[@]}; i++)); do | |
| ./mihomo convert-ruleset ipcidr text "./ips/${list[i]}/${list[i]}.list" ../clash-ruleset/${list[i]}.mrs | |
| done | |
| rm -rf ./mihomo* ./domains/ ./ips/ | |
| - name: Download and unzip `sing-box` core | |
| run: | | |
| wget https://github.com/DustinWin/clash_singbox-tools/releases/download/sing-box/sing-box-release-linux-amd64.tar.gz -O - | tar -zxf - -C ./tools/ | |
| mv -f ./tools/CrashCore ./tools/sing-box | |
| - name: Generate `sing-box` rule_set | |
| run: | | |
| mkdir -p ./sing-box-ruleset/ | |
| cd ./tools/ | |
| sed -i 's/"version": 1/"version": 2/' ./convert.sh | |
| chmod +x ./convert.sh && ./convert.sh | |
| rm -rf ./sing-box* ./rules* | |
| mv -f ./*.json ./*.srs ../sing-box-ruleset/ | |
| - name: Release and upload `clash` assets | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: clash | |
| tag: clash | |
| overwrite: true | |
| body: | | |
| Clash 和 [mihomo](https://github.com/MetaCubeX/mihomo) geodata 文件 | |
| geodata 文件更新于 ${{ env.update_version }} | |
| file_glob: true | |
| file: ./clash/* | |
| - name: Release and upload `clash-ruleset` assets | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: clash-ruleset | |
| tag: clash-ruleset | |
| overwrite: true | |
| body: | | |
| Clash 和 [mihomo](https://github.com/MetaCubeX/mihomo) rule-set 规则集文件 | |
| 规则集文件更新于 ${{ env.update_version }} | |
| file_glob: true | |
| file: ./clash-ruleset/* | |
| - name: Commit and push `clash-ruleset` branch | |
| run: | | |
| cd ./clash-ruleset/ || exit 1 | |
| git init | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git checkout -b clash-ruleset | |
| git add . && git commit -m "规则集文件更新于 ${update_version}" | |
| git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
| git push -f -u origin clash-ruleset | |
| - name: Release and upload `sing-box` assets | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: sing-box | |
| tag: sing-box | |
| overwrite: true | |
| body: | | |
| [sing-box](https://github.com/SagerNet/sing-box) geodata 文件 | |
| geodata 文件更新于 ${{ env.update_version }} | |
| file_glob: true | |
| file: ./sing-box/* | |
| - name: Release and upload `sing-box-ruleset` assets | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: sing-box-ruleset | |
| tag: sing-box-ruleset | |
| overwrite: true | |
| body: | | |
| [sing-box](https://github.com/SagerNet/sing-box) rule_set 规则集文件 | |
| 规则集文件更新于 ${{ env.update_version }} | |
| file_glob: true | |
| file: ./sing-box-ruleset/* | |
| - name: Commit and push `sing-box-ruleset` branch | |
| run: | | |
| cd ./sing-box-ruleset/ || exit 1 | |
| git init | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git checkout -b sing-box-ruleset | |
| git add . && git commit -m "规则集文件更新于 ${update_version}" | |
| git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
| git push -f -u origin sing-box-ruleset | |
| - name: Purge jsDelivr CDN | |
| run: | | |
| cd ./clash/ || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@clash/${file}" | |
| done | |
| cd ../sing-box/ || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@sing-box/${file}" | |
| done | |
| cd ../clash-ruleset/ || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@clash-ruleset/${file}" | |
| done | |
| cd ../sing-box-ruleset/ || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@sing-box-ruleset/${file}" | |
| done | |
| - name: Delete old workflow runs | |
| uses: Mattraks/delete-workflow-runs@v2 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| repository: ${{ github.repository }} | |
| retain_days: 3 | |
| keep_minimum_runs: 1 |