diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index b59c922..41b4cf7 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -32,6 +32,9 @@ jobs:
           node-version: 16
       - name: install Rust stable
         uses: dtolnay/rust-toolchain@stable
+        with:
+          # Those targets are only used on macos runners so it's in an `if` to slightly speed up windows and linux builds.
+          targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
       - name: install dependencies (ubuntu only)
         if: matrix.platform == 'ubuntu-20.04'
         run: |