From ca63be2da3883c796176c6909ef43e6afe3aa341 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 22 Feb 2024 19:12:41 -0500 Subject: [PATCH] feat: support arm64 --- .github/workflows/cd.yml | 4 ++ .github/workflows/ci.yml | 10 +++-- Dockerfile | 63 +++++++++++++++++++++++++----- dependencies/python/build-venvs.sh | 26 ++++++++++-- scripts/install-glibc.sh | 15 +------ scripts/linterVersions.sh | 12 ++++-- scripts/powershell-arm64.sh | 5 +++ scripts/scalafmt-arm64.sh | 5 +++ 8 files changed, 107 insertions(+), 33 deletions(-) create mode 100755 scripts/powershell-arm64.sh create mode 100755 scripts/scalafmt-arm64.sh diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 67c5e6a3160..56c3cceabc0 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -71,6 +71,9 @@ jobs: sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/share/dotnet + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -125,6 +128,7 @@ jobs: cache-to: type=registry,ref=${{ env.CONTAINER_IMAGE_ID }}-buildcache,mode=max load: false push: true + platforms: linux/amd64,linux/arm64/v8 secrets: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} tags: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6629fd7499b..47bcdc13d92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: name: Build and Test runs-on: ubuntu-latest permissions: - contents: read + contents: read concurrency: # Ref: https://docs.github.com/en/actions/learn-github-actions/contexts#github-context # github.head_ref: head_ref or source branch of the pull request @@ -82,6 +82,9 @@ jobs: sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/share/dotnet + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -95,8 +98,9 @@ jobs: BUILD_REVISION=${{ env.BUILD_REVISION }} BUILD_VERSION=${{ env.BUILD_VERSION }} cache-from: type=registry,ref=${{ env.CONTAINER_IMAGE_ID }}-buildcache - outputs: type=docker,dest=/tmp/${{ env.CONTAINER_IMAGE_OUTPUT_IMAGE_NAME }}.tar + outputs: type=oci,dest=/tmp/${{ env.CONTAINER_IMAGE_OUTPUT_IMAGE_NAME }}.tar push: false + platforms: linux/amd64,linux/arm64/v8 secrets: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} tags: | @@ -218,7 +222,7 @@ jobs: if: github.event_name == 'pull_request' runs-on: ubuntu-latest permissions: - contents: read + contents: read steps: - uses: actions/checkout@v4 with: diff --git a/Dockerfile b/Dockerfile index d6649a33f60..55e66d7b3eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,13 +20,13 @@ FROM koalaman/shellcheck:v0.9.0 as shellcheck FROM mstruebing/editorconfig-checker:2.7.2 as editorconfig-checker FROM mvdan/shfmt:v3.8.0 as shfmt FROM rhysd/actionlint:1.6.27 as actionlint -FROM scalameta/scalafmt:v3.8.0 as scalafmt +FROM scalameta/scalafmt:v3.8.0 as scalafmt-source FROM zricethezav/gitleaks:v8.18.2 as gitleaks FROM yoheimuta/protolint:0.47.6 as protolint -FROM ghcr.io/clj-kondo/clj-kondo:2024.02.12-alpine as clj-kondo +FROM ghcr.io/clj-kondo/clj-kondo:2024.02.13-SNAPSHOT as clj-kondo FROM dart:3.3.0-sdk as dart FROM mcr.microsoft.com/dotnet/sdk:8.0.101-alpine3.19 as dotnet-sdk -FROM mcr.microsoft.com/powershell:7.4-alpine-3.17 as powershell +FROM mcr.microsoft.com/powershell:7.4-alpine-3.17 as powershell-source FROM python:3.12.2-alpine3.19 as clang-format @@ -62,11 +62,17 @@ RUN apk add --no-cache \ SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] +ARG TARGETARCH + COPY dependencies/python/ /stage WORKDIR /stage RUN ./build-venvs.sh && rm -rfv /stage -FROM python:3.12.2-alpine3.19 as npm-builder +# On arm64, `npm install` gives ECONNRESET errors: +# ERR! network Client network socket disconnected before secure TLS connection was established +# +# Since we only need node_modules, we can build it on an amd64 image instead +FROM --platform=$BUILDPLATFORM python:3.12.2-alpine3.19 as npm-builder RUN apk add --no-cache \ bash \ @@ -108,12 +114,43 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R / RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R -FROM powershell as powershell-installer +############################################# +# Copy scalafmt binary only on amd64 image # +############################################# +FROM python:3.12.2-alpine3.19 as scalafmt-arm64 + +COPY scripts/scalafmt-arm64.sh /bin/scalafmt + +FROM python:3.12.2-alpine3.19 as scalafmt-amd64 + +COPY --from=scalafmt-source /bin/scalafmt /bin/scalafmt +ARG TARGETARCH +FROM scalafmt-${TARGETARCH} as scalafmt + +############################################## +# Copy powershell binary only on amd64 image # +############################################## +FROM python:3.12.2-alpine3.19 as powershell-arm64 + +COPY scripts/powershell-arm64.sh /usr/bin/pwsh +RUN mkdir -p /opt/microsoft/powershell \ + && touch /tmp/PS_INSTALL_FOLDER + +FROM powershell-source as powershell-installer # Copy the value of the PowerShell install directory to a file so we can reuse it # when copying PowerShell stuff in the main image RUN echo "${PS_INSTALL_FOLDER}" > /tmp/PS_INSTALL_FOLDER +FROM python:3.12.2-alpine3.19 as powershell-amd64 + +COPY --from=powershell-installer /tmp/PS_INSTALL_FOLDER /tmp/PS_INSTALL_FOLDER +COPY --from=powershell-source /opt/microsoft/powershell /opt/microsoft/powershell +RUN touch /usr/bin/pwsh + +ARG TARGETARCH +FROM powershell-${TARGETARCH} as powershell + FROM python:3.12.2-alpine3.19 as base_image LABEL com.github.actions.name="Super-Linter" \ @@ -345,7 +382,7 @@ COPY --from=kubeconfrm /kubeconform /usr/bin/ ##################### # Install clj-kondo # ##################### -COPY --from=clj-kondo /bin/clj-kondo /usr/bin/ +COPY --from=clj-kondo /usr/local/bin/clj-kondo /usr/bin/ #################### # Install dart-sdk # @@ -403,6 +440,8 @@ ENTRYPOINT ["/action/lib/linter.sh"] FROM base_image as slim +ARG TARGETARCH + # Run to build version file and validate image ENV IMAGE="slim" COPY scripts/linterVersions.sh / @@ -467,17 +506,22 @@ RUN dotnet help ######################################### # Install Powershell + PSScriptAnalyzer # ######################################### -COPY --from=powershell-installer /tmp/PS_INSTALL_FOLDER /tmp/PS_INSTALL_FOLDER +COPY --from=powershell /usr/bin/pwsh /usr/bin/pwsh +COPY --from=powershell /tmp/PS_INSTALL_FOLDER /tmp/PS_INSTALL_FOLDER COPY --from=powershell /opt/microsoft/powershell /opt/microsoft/powershell +ARG TARGETARCH # Disable Powershell telemetry ENV POWERSHELL_TELEMETRY_OPTOUT=1 ARG PSSA_VERSION='1.21.0' -RUN PS_INSTALL_FOLDER="$(cat /tmp/PS_INSTALL_FOLDER)" \ +RUN if [[ "${TARGETARCH}" == "amd64" ]]; then \ + PS_INSTALL_FOLDER="$(cat /tmp/PS_INSTALL_FOLDER)" \ && echo "PS_INSTALL_FOLDER: ${PS_INSTALL_FOLDER}" \ + && rm /usr/bin/pwsh \ && ln -s "${PS_INSTALL_FOLDER}/pwsh" /usr/bin/pwsh \ && chmod a+x,o-w "${PS_INSTALL_FOLDER}/pwsh" \ && pwsh -c "Install-Module -Name PSScriptAnalyzer -RequiredVersion ${PSSA_VERSION} -Scope AllUsers -Force" \ - && rm -rf /tmp/PS_INSTALL_FOLDER + && rm -rf /tmp/PS_INSTALL_FOLDER \ + ; fi ############################################################# # Install Azure Resource Manager Template Toolkit (arm-ttk) # @@ -487,6 +531,7 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-arm-ttk.sh && rm -rf /install-a # Run to build version file and validate image again because we installed more linters ENV IMAGE="standard" +ARG TARGETARCH COPY scripts/linterVersions.sh / RUN /linterVersions.sh \ && rm -rfv /linterVersions.sh diff --git a/dependencies/python/build-venvs.sh b/dependencies/python/build-venvs.sh index 4b1fcfa7367..d916c9ef380 100755 --- a/dependencies/python/build-venvs.sh +++ b/dependencies/python/build-venvs.sh @@ -8,11 +8,21 @@ ##################### set -euo pipefail -apk add --no-cache --virtual .python-build-deps \ - gcc \ - linux-headers \ - musl-dev \ +packages=( + gcc + linux-headers + musl-dev python3-dev +) + +if [[ "${TARGETARCH}" != "amd64" ]]; then + # libffi-dev is required for building wheel for cffi, + # until https://github.com/python-cffi/cffi/issues/69 is merged + packages+=(libffi-dev) +fi + +apk add --no-cache --virtual .python-build-deps \ + "${packages[@]}" ############################ # Create staging directory # @@ -24,6 +34,14 @@ mkdir -p /venvs ######################################## pip install virtualenv +if [[ "${TARGETARCH}" != "amd64" ]]; then + # Install Rust compiler (required by checkov on arm64) # + # remove this once https://github.com/bridgecrewio/checkov/pull/6045 is merged + apk add --no-cache curl + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + export PATH=$PATH:$HOME/.cargo/bin +fi + ####################################################### # Iterate through requirments.txt to install binaries # ####################################################### diff --git a/scripts/install-glibc.sh b/scripts/install-glibc.sh index d594242a33d..4fbef71162b 100755 --- a/scripts/install-glibc.sh +++ b/scripts/install-glibc.sh @@ -2,20 +2,7 @@ set -euo pipefail -url=$( - set -euo pipefail - curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/sgerrand/alpine-pkg-glibc/releases/tags/${GLIBC_VERSION}" | - jq --arg name "glibc-${GLIBC_VERSION}.apk" -r '.assets | .[] | select(.name | contains($name)) | .url' -) -curl --retry 5 --retry-delay 5 -sL -o "glibc-${GLIBC_VERSION}.apk" \ - -H "Accept: application/octet-stream" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "${url}" -apk add --no-cache --force-overwrite "glibc-${GLIBC_VERSION}.apk" -rm "glibc-${GLIBC_VERSION}.apk" +apk add --no-cache gcompat # Install zlib apk add --no-cache --virtual .glibc-build-deps \ diff --git a/scripts/linterVersions.sh b/scripts/linterVersions.sh index a785ca0676d..88e7beff197 100755 --- a/scripts/linterVersions.sh +++ b/scripts/linterVersions.sh @@ -58,7 +58,6 @@ LINTER_NAMES_ARRAY['R']="R" LINTER_NAMES_ARRAY['RAKU']="raku" LINTER_NAMES_ARRAY['RENOVATE']="renovate-config-validator" LINTER_NAMES_ARRAY['RUBY']="rubocop" -LINTER_NAMES_ARRAY['SCALAFMT']="scalafmt" LINTER_NAMES_ARRAY['SHELL_SHFMT']="shfmt" LINTER_NAMES_ARRAY['SNAKEMAKE_LINT']="snakemake" LINTER_NAMES_ARRAY['SNAKEMAKE_SNAKEFMT']="snakefmt" @@ -77,15 +76,22 @@ LINTER_NAMES_ARRAY['TYPESCRIPT_STANDARD']="ts-standard" LINTER_NAMES_ARRAY['XML']="xmllint" LINTER_NAMES_ARRAY['YAML']="yamllint" +if [[ "${TARGETARCH}" == "amd64" ]]; then + LINTER_NAMES_ARRAY['SCALAFMT']="scalafmt" +fi + if [[ "${IMAGE}" == "standard" ]]; then LINTER_NAMES_ARRAY['ARM']="arm-ttk" LINTER_NAMES_ARRAY['CSHARP']="dotnet" - LINTER_NAMES_ARRAY['ENV']="dotenv-linter" - LINTER_NAMES_ARRAY['POWERSHELL']="pwsh" LINTER_NAMES_ARRAY['RUST_2015']="rustfmt" LINTER_NAMES_ARRAY['RUST_2018']="rustfmt" LINTER_NAMES_ARRAY['RUST_2021']="rustfmt" LINTER_NAMES_ARRAY['RUST_CLIPPY']="clippy" + + if [[ "${TARGETARCH}" == "amd64" ]]; then + LINTER_NAMES_ARRAY['ENV']="dotenv-linter" + LINTER_NAMES_ARRAY['POWERSHELL']="pwsh" + fi fi echo "Building linter version file: ${VERSION_FILE}" diff --git a/scripts/powershell-arm64.sh b/scripts/powershell-arm64.sh new file mode 100755 index 00000000000..6acb8523abb --- /dev/null +++ b/scripts/powershell-arm64.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "pwsh is not supported on arm64" diff --git a/scripts/scalafmt-arm64.sh b/scripts/scalafmt-arm64.sh new file mode 100755 index 00000000000..d6a20b1d34e --- /dev/null +++ b/scripts/scalafmt-arm64.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "scalafmt is not supported on arm64"