build

#!/usr/bin/env bash

# This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
# arg2: artifact, e.g. rel | relup | zip | pkg

if [[ -n "$DEBUG" ]]; then
    set -x
fi
set -euo pipefail

PROFILE="$1"
ARTIFACT="$2"

# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"

PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
export PKG_VSN

SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"

##
## Support RPM and Debian based linux systems
##
if [ "$(uname -s)" = 'Linux' ]; then
    case "${SYSTEM:-}" in
        ubuntu*|debian*|raspbian*)
            PKGERDIR='deb'
            ;;
        *)
            PKGERDIR='rpm'
            ;;
    esac
fi

if [ "${SYSTEM}" = 'windows' ]; then
    # windows does not like the find
    FIND="/usr/bin/find"
else
    FIND='find'
fi

UNAME="$(uname -m)"
case "$UNAME" in
    x86_64)
        ARCH='amd64'
        ;;
    aarch64)
        ARCH='arm64'
        ;;
    arm*)
        ARCH=arm
        ;;
esac
# used in -pkg Makefile
export ARCH

log() {
    local msg="$1"
    # rebar3 prints ===>, so we print ===<
    echo "===< $msg"
}

make_rel() {
    # shellcheck disable=SC1010
    ./rebar3 as "$PROFILE" do release,tar
}

relup_db() {
    case "$PROFILE" in
        *-ee*)
            ./scripts/relup-base-vsns.escript "$@" ./data/relup-paths-ee.eterm
            ;;
        *)
            ./scripts/relup-base-vsns.escript "$@" ./data/relup-paths.eterm
            ;;
    esac
}

## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
make_relup() {
    local lib_dir="_build/$PROFILE/rel/emqx/lib"
    local releases_dir="_build/$PROFILE/rel/emqx/releases"
    local zip_file
    mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
    local releases=()
    local OTP_CHANGED='no'
    if [ -d "$releases_dir" ]; then
        for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do
            OTP_BASE=$(relup_db otp-vsn-for "$BASE_VSN")
            if [[ "$OTP_BASE" != "$OTP_VSN" ]]; then
              OTP_CHANGED='yes'
            fi
            zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip"
            if [ ! -d "$releases_dir/$BASE_VSN" ]; then
                local tmp_dir
                tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
                unzip -q "$zip_file" "emqx/releases/*" -d "$tmp_dir"
                unzip -q "$zip_file" "emqx/lib/*" -d "$tmp_dir"
                cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true
                cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true
                rm -rf "$tmp_dir"
            fi
            releases+=( "$BASE_VSN" )
        done
    fi
    if [ ${#releases[@]} -eq 0 ]; then
        log "No upgrade base found, relup ignored"
        return 0
    fi
    RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
    export RELX_BASE_VERSIONS
    if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then
        echo "EMQX 4.[3,4] specific, overwrite OTP app versions"
        local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
        if [ ! -f "${emqx_rel_file}" ]; then
            ./rebar3 as "${PROFILE}" release
        fi
        scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}"
    fi
    ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"

    # assert that there is no 'restart_emulator' in relup
    # EMQX wants hot upgrade without VM restart
    if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then
        echo "Error: restart_emulator instruction found in relup";
        exit 1
    fi

    # rollback rel file per releases
    #
    if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then
        echo "restore upgrade base rel files... "
        for rel in ${releases[*]};
        do
            bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak"
            echo "restore $bakfile ..."
            if [ -f "$bakfile" ]; then
                echo "restore from $bakfile"
                mv "${bakfile}" "${bakfile%.bak}"
            fi
        done
    fi
}

## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt
cp_dyn_libs() {
    local rel_dir="$1"
    local target_dir="${rel_dir}/dynlibs"
    if ! [ "$(uname -s)" = 'Linux' ]; then
        return 0;
    fi
    mkdir -p "$target_dir"
    while read -r so_file; do
        cp -L "$so_file" "$target_dir/"
    done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
        | xargs -0 ldd \
        | grep -E '(libcrypto)|(libtinfo)' \
        | awk '{print $3}' \
        | sort -u)
}


## make_zip turns .tar.gz into a .zip with a slightly different name.
## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_zip() {
    # build the tarball again to ensure relup is included
    make_rel
    # use relative path because abs path is tricky in windows
    tard="tmp/zip-wd-${PKG_VSN}"
    rm -rf "${tard}/emqx"
    mkdir -p "${tard}/emqx"
    local relpath="_build/${PROFILE}/rel/emqx"
    local pkgpath="_packages/${PROFILE}"
    local pkgname
    pkgname="${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip"
    mkdir -p "${pkgpath}"
    local tarname="emqx-${PKG_VSN}.tar.gz"
    local tarball="${relpath}/${tarname}"
    local target_zip="${pkgpath}/${pkgname}"
    tar zxf "${tarball}" -C "${tard}/emqx"
    has_relup='yes'
    case "$SYSTEM" in
        windows*)
            # no relup support for windows
            has_relup='no'
            ;;
        debian11)
            case "$PKG_VSN" in
                4.4.2*)
                    # this is the first version for debian11, no relup
                    has_relup='no'
                    ;;
            esac
            ;;
    esac
    # shellcheck disable=SC2207
    bases=($(relup_db base-vsns "$PKG_VSN"))
    if [[ "${#bases[@]}" -eq 0 ]]; then
        has_relup='no'
    fi
    if [ "$has_relup" = 'yes' ]; then
        ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
    fi
    cp_dyn_libs "${tard}/emqx"
    case "$SYSTEM" in
        macos*)
            # if the flag to sign macos binaries is set, but developer certificate
            # or certificate password is not configured, reset the flag
            # could happen, for example, when people submit PR from a fork, in this
            # case they cannot access secrets
            if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \
                      ( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \
                            "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then
                echo "Apple developer certificate is not configured, skip signing"
                APPLE_SIGN_BINARIES=0
            fi
            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
                ./scripts/macos-sign-binaries.sh "${tard}/emqx"
            fi
            (cd "${tard}" && zip -qr - emqx) > "${target_zip}"
            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
                # notarize the package
                # if fails, you can check what went wrong with this command:
                # xcrun notarytool log --apple-id <apple id> \
                    #   --apple-id <apple id> \
                    #   --password <apple id password>
                    #   --team-id <apple team id> <submission-id>
                xcrun notarytool submit \
                      --apple-id "${APPLE_ID}" \
                      --password "${APPLE_ID_PASSWORD}" \
                      --team-id "${APPLE_TEAM_ID}" "${target_zip}" --wait
            fi
            # sha256sum may not be available on macos
            openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2  > "${target_zip}.sha256"
            ;;
        windows*)
            pushd "${tard}" >/dev/null
            7z a "${pkgname}" emqx
            popd >/dev/null
            mv "${tard}/${pkgname}" "${target_zip}"
            sha256sum "${target_zip}" | head -c 64 > "${target_zip}.sha256"
            ;;
        *)
            (cd "${tard}" && zip -qr - emqx) > "${target_zip}"
            sha256sum "${target_zip}" | head -c 64 > "${target_zip}.sha256"
            ;;
    esac
    log "Zip package successfully created: ${target_zip}"
    log "Zip package sha256sum: $(cat "${target_zip}.sha256")"
}

## This function builds the default docker image
## based images is by default $EMQX_DEFAULT_BUILDER (see Makefile)
make_docker() {
    EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
    EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
    set -x
    docker build --no-cache --pull \
       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
       --build-arg RUN_FROM="${EMQX_RUNNER}" \
       --build-arg EMQX_NAME="$PROFILE" \
       --tag "emqx/$PROFILE:${PKG_VSN}" \
       -f "${DOCKERFILE}" .
}

## This function accepts any base docker image,
## a emqx zip-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQ X
##
## Export below variables to quickly build an image
##
## Name               Default                         Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE    current os                      centos:7
## EMQX_ZIP_PACKAGE   _packages/<current-zip-target>  /tmp/emqx-4.4.0-otp24.3.4.2-1-el7-amd64.zip
## EMQX_IMAGE_TAG     emqx/emqx:<current-vns-rel>     emqx/emqx:testing-tag
##
make_docker_testing() {
    if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
        case "$SYSTEM" in
            ubuntu20*)
                EMQX_BASE_IMAGE="ubuntu:20.04"
                ;;
            el8)
                EMQX_BASE_IMAGE="rockylinux:8"
                ;;
            *)
                echo "Unsupported testing base image for $SYSTEM"
                exit 1
                ;;
        esac
    fi
    EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
    local defaultzip
    defaultzip="_packages/${PROFILE}/${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip"
    local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}"
    if [ ! -f "$zip" ]; then
        log "ERROR: $zip not built?"
        exit 1
    fi
    set -x
    docker build \
       --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
       --build-arg EMQX_ZIP_PACKAGE="${zip}" \
       --tag "$EMQX_IMAGE_TAG" \
       -f "${DOCKERFILE_TESTING}" .
}

log "building artifact=$ARTIFACT for profile=$PROFILE"

case "$ARTIFACT" in
    rel)
        make_rel
        ;;
    relup)
        make_relup
        ;;
    zip)
        make_zip
        ;;
    pkg)
        if [ -z "${PKGERDIR:-}" ]; then
            log "Skipped making deb/rpm package for $SYSTEM"
            exit 0
        fi
        EMQX_REL="$(pwd)" make -C "deploy/packages/${PKGERDIR}" clean
        EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
        ;;
    docker)
        make_docker
        ;;
    docker-testing)
        make_docker_testing
        ;;
    *)
        log "Unknown artifact $ARTIFACT"
        exit 1
        ;;
esac