diff --git a/.github/workflows/ecr-image-build-w-arm-runner.yml b/.github/workflows/ecr-image-build-w-arm-runner.yml new file mode 100644 index 0000000000..d0f4a268fa --- /dev/null +++ b/.github/workflows/ecr-image-build-w-arm-runner.yml @@ -0,0 +1,231 @@ +--- +name: AWS ECR Build Image with ARM Runner + +on: # yamllint disable-line rule:truthy + release: + types: + - "released" + push: + branches: + - "main" + - "*-rc" + tags: + - "v*" + +jobs: + build: + strategy: + fail-fast: false + matrix: + platforms: + - [linux/amd64, ubuntu-latest] + - [linux/arm64, ubuntu-arm64-runner] + runs-on: ${{ matrix.platforms[1] }} + steps: + - name: Prepare + run: | + platform=${{ matrix.platforms[0] }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-central-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata + tags: | + type=ref,event=branch + type=ref,event=tag + + - name: Setup SSH Agent and add Github to known hosts + env: + SSH_AUTH_SOCK: /tmp/ssh-agent.sock + run: | + ssh-agent -a $SSH_AUTH_SOCK >> /dev/null + ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY }}" + mkdir -p ~/.ssh + ssh-keyscan github.com > ~/.ssh/known_hosts + + - name: Get the version + id: get-version + if: github.event_name != 'push' + run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Get the branch name + id: get-branch-name + if: > + github.event_name == 'push' + || github.event_name == 'workflow_dispatch' + run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + + - name: Get docker repository URL + id: get-repo-url + run: | + echo "docker_repo=${{ steps.login-ecr.outputs.registry }}\ + /onaio/onadata:${{ env.version || github.ref_name }}"\ + | sed 's/ //g' >> $GITHUB_ENV + + - name: (Ubuntu) Build and push + id: docker-build-ubuntu + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/onadata-uwsgi/Dockerfile.ubuntu + platforms: ${{ matrix.platforms[0] }} + cache-from: type=registry,ref=${{ env.docker_repo }} + cache-to: type=inline + ssh: | + default=/tmp/ssh-agent.sock + build-args: > + optional_packages=PyYAML + django-redis + ${{ secrets.ECR_OPTIONAL_PACKAGES }} + push: true + labels: ${{ steps.meta.outputs.labels }} + provenance: false + outputs: > + type=image, + name=${{ steps.login-ecr.outputs.registry }}/onaio/onadata, + push-by-digest=true, + name-canonical=true, + push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.docker-build-ubuntu.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: + - build + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-central-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata + tags: | + type=ref,event=branch + type=ref,event=tag + + - name: Get docker repository URL + id: get-repo-url + run: | + echo "docker_repo=${{ steps.login-ecr.outputs.registry }}\ + /onaio/onadata:${{ steps.meta.outputs.version }}"\ + | sed 's/ //g' >> $GITHUB_ENV + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ steps.login-ecr.outputs.registry }}/onaio/onadata@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.docker_repo }} + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: | + ${{ env.docker_repo }} + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan result to Github security lab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' + continue-on-error: true + + - name: Run Trivy vulnerability scanner for Slack + uses: aquasecurity/trivy-action@master + with: + image-ref: | + ${{ env.docker_repo }} + format: json + output: 'trivy-results.json' + + - name: Create summary of trivy issues + run: | + summary=$(jq -r '.Results[] | select(.Vulnerabilities) | .Vulnerabilities | group_by(.Severity) | map({Severity: .[0].Severity, Count: length}) | .[] | [.Severity, .Count] | join(": ")' trivy-results.json | awk 'NR > 1 { printf(" | ") } {printf "%s",$0}') + if [ -z $summary ] + then + summary="0 Issues" + fi + echo "SUMMARY=$summary" >> $GITHUB_ENV + + - name: Send Slack Notification + uses: slackapi/slack-github-action@v1.26.0 + with: + payload: | + { + "text": + "Trivy scan results for ${{ steps.meta.outputs.version }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "[Ona Data] Trivy scan results: ${{ env.SUMMARY }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "View scan results: https://github.com/${{ github.repository }}/security/code-scanning?query=branch:${{ env.version || github.ref_name }}+is:open++" + } + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/docker/onadata-uwsgi/Dockerfile.ubuntu b/docker/onadata-uwsgi/Dockerfile.ubuntu index 24c2d2cc2d..ed4b2d3e1b 100644 --- a/docker/onadata-uwsgi/Dockerfile.ubuntu +++ b/docker/onadata-uwsgi/Dockerfile.ubuntu @@ -3,11 +3,11 @@ FROM onaio/python-deps:3.10.14-20240703 AS base ARG optional_packages # Silence configuration prompts -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive -ENV PYTHONUNBUFFERED 1 +ENV PYTHONUNBUFFERED=1 -ENV DJANGO_SETTINGS_MODULE onadata.settings.docker +ENV DJANGO_SETTINGS_MODULE=onadata.settings.docker USER root @@ -35,10 +35,10 @@ RUN python -m pip install --no-cache-dir -U pip && \ python -m pip install --no-cache-dir -r requirements/azure.pip && \ python -m pip install --no-cache-dir pyyaml==6.0.1 uwsgitop==0.12 supervisor==4.2.5 -FROM base as docs +FROM base AS docs ENV PYENV_ROOT="$HOME/.pyenv" -ENV PATH $PYENV_ROOT/versions/3.10.14/bin:$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +ENV PATH=$PYENV_ROOT/versions/3.10.14/bin:$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH COPY --from=base /home/appuser/.pyenv/ /home/appuser/.pyenv/ COPY --from=base /srv/onadata/ /srv/onadata/ @@ -53,9 +53,9 @@ RUN python -m pip install --no-cache-dir -r requirements/docs.pip && \ make -C docs html -FROM debian:bookworm-20240701 as runtime +FROM debian:bookworm-20240701 AS runtime -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive # Install prerequisite packages RUN echo "deb http://deb.debian.org/debian unstable main non-free contrib" >> /etc/apt/sources.list \ @@ -64,8 +64,8 @@ RUN echo "deb http://deb.debian.org/debian unstable main non-free contrib" >> /e # # Generate and set en_US.UTF-8 locale RUN locale-gen en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -ENV LC_CTYPE en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV LC_CTYPE=en_US.UTF-8 RUN dpkg-reconfigure locales @@ -94,9 +94,9 @@ RUN chown -R appuser:appuser /srv/onadata /home/appuser/.pyenv USER appuser WORKDIR /srv/onadata -ENV HOME /home/appuser -ENV PYTHON_VERSION 3.10.14 +ENV HOME=/home/appuser +ENV PYTHON_VERSION=3.10.14 ENV PYENV_ROOT="$HOME/.pyenv" -ENV PATH $PYENV_ROOT/versions/3.10.14/bin:$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +ENV PATH=$PYENV_ROOT/versions/3.10.14/bin:$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH CMD ["uwsgi", "--ini", "uwsgi.ini"]