jobs/fix-build: new job to fix archived builds

jobs/build-arch.Jenkinsfile

@@ -370,23 +370,10 @@ lock(resource: "build-${params.STREAM}-${basearch}") {
             pipeutils.tryWithMessagingCredentials() {
                 def parallelruns = [:]
                 parallelruns['Sign Images'] = {
-                    pipeutils.shwrapWithAWSBuildUploadCredentials("""
-                    cosa sign --build=${newBuildID} --arch=${basearch} \
-                        robosignatory --s3 ${s3_stream_dir}/builds \
-                        --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG} \
-                        --extra-fedmsg-keys stream=${params.STREAM} \
-                        --images --gpgkeypath /etc/pki/rpm-gpg \
-                        --fedmsg-conf \${FEDORA_MESSAGING_CONF}
-                    """)
+                    pipeutils.signImages(params.STREAM, newBuildID, basearch, s3_stream_dir)
                 }
                 parallelruns['OSTree Import: Compose Repo'] = {
-                    shwrap("""
-                    cosa shell -- \
-                    /usr/lib/coreos-assembler/fedmsg-send-ostree-import-request \
-                        --build=${newBuildID} --arch=${basearch} \
-                        --s3=${s3_stream_dir} --repo=compose \
-                        --fedmsg-conf \${FEDORA_MESSAGING_CONF}
-                    """)
+                    pipeutils.composeRepoImport(newBuildID, basearch, s3_stream_dir)
                 }
                 // process this batch
                 parallel parallelruns

jobs/build.Jenkinsfile

@@ -459,23 +459,10 @@ lock(resource: "build-${params.STREAM}") {
             pipeutils.tryWithMessagingCredentials() {
                 def parallelruns = [:]
                 parallelruns['Sign Images'] = {
-                    pipeutils.shwrapWithAWSBuildUploadCredentials("""
-                    cosa sign --build=${newBuildID} --arch=${basearch} \
-                        robosignatory --s3 ${s3_stream_dir}/builds \
-                        --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG} \
-                        --extra-fedmsg-keys stream=${params.STREAM} \
-                        --images --gpgkeypath /etc/pki/rpm-gpg \
-                        --fedmsg-conf \${FEDORA_MESSAGING_CONF}
-                    """)
+                    pipeutils.signImages(params.STREAM, newBuildID, basearch, s3_stream_dir)
                 }
                 parallelruns['OSTree Import: Compose Repo'] = {
-                    shwrap("""
-                    cosa shell -- \
-                    /usr/lib/coreos-assembler/fedmsg-send-ostree-import-request \
-                        --build=${newBuildID} --arch=${basearch} \
-                        --s3=${s3_stream_dir} --repo=compose \
-                        --fedmsg-conf \${FEDORA_MESSAGING_CONF}
-                    """)
+                    pipeutils.composeRepoImport(newBuildID, basearch, s3_stream_dir)
                 }
                 // process this batch
                 parallel parallelruns

jobs/fix-build.Jenkinsfile

@@ -0,0 +1,165 @@
+node {
+    checkout scm
+    // these are script global vars
+    pipeutils = load("utils.groovy")
+    pipecfg = pipeutils.load_pipecfg()
+    libcloud = load("libcloud.groovy")
+}
+
+properties([
+    pipelineTriggers([]),
+    parameters([
+      choice(name: 'STREAM',
+             choices: pipeutils.get_streams_choices(pipecfg),
+             description: 'CoreOS stream to sign'),
+      string(name: 'VERSION',
+             description: 'CoreOS version to sign',
+             defaultValue: '',
+             trim: true),
+      string(name: 'ARCHES',
+             description: "Override architectures (space-separated) to sign. " +
+                          "Defaults to all arches for this stream.",
+             defaultValue: "",
+             trim: true),
+      booleanParam(name: 'SKIP_SIGNING',
+                   defaultValue: false,
+                   description: 'Skip signing artifacts'),
+      booleanParam(name: 'SKIP_COMPOSE_IMPORT',
+                   defaultValue: false,
+                   description: 'Skip requesting compose repo import'),
+      booleanParam(name: 'SKIP_TESTS',
+                   defaultValue: false,
+                   description: 'Skip triggering cloud and upgrade tests'),
+      string(name: 'COREOS_ASSEMBLER_IMAGE',
+             description: 'Override coreos-assembler image to use',
+             defaultValue: "",
+             trim: true)
+    ] + pipeutils.add_hotfix_parameters_if_supported()),
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+])
+
+def build_description = "[${params.STREAM}]"
+
+// Reload pipecfg if a hotfix build was provided. The reason we do this here
+// instead of loading the right one upfront is so that we don't modify the
+// parameter definitions above and their default values.
+if (params.PIPECFG_HOTFIX_REPO || params.PIPECFG_HOTFIX_REF) {
+    node {
+        pipecfg = pipeutils.load_pipecfg(params.PIPECFG_HOTFIX_REPO, params.PIPECFG_HOTFIX_REF)
+        build_description = "[${params.STREAM}-${pipecfg.hotfix.name}]"
+    }
+}
+
+// no way to make a parameter required directly so manually check
+// https://issues.jenkins-ci.org/browse/JENKINS-3509
+if (params.VERSION == "") {
+    throw new Exception("Missing VERSION parameter!")
+}
+
+// runtime parameter always wins
+def cosa_img = params.COREOS_ASSEMBLER_IMAGE
+cosa_img = cosa_img ?: pipeutils.get_cosa_img(pipecfg, params.STREAM)
+
+def basearches = []
+if (params.ARCHES != "") {
+    basearches = params.ARCHES.split() as List
+} else {
+    basearches += 'x86_64'
+    basearches += pipeutils.get_additional_arches(pipecfg, params.STREAM)
+}
+
+// make sure there are no duplicates
+basearches = basearches.unique()
+
+def stream_info = pipecfg.streams[params.STREAM]
+
+currentBuild.description = "${build_description} Waiting"
+
+// We just lock here out of an abundance of caution in case somehow
+// two jobs run for the same stream, but that really shouldn't happen.
+// Also lock version-arch-specific locks to make sure these builds are finished.
+lock(resource: "sign-${params.VERSION}") {
+    cosaPod(cpu: "1", memory: "512Mi", image: cosa_img,
+            serviceAccount: "jenkins") {
+    try {
+        currentBuild.description = "${build_description} Running"
+
+        def s3_stream_dir = pipeutils.get_s3_streams_dir(pipecfg, params.STREAM)
+
+        // Fetch metadata files for the build we are interested in
+        stage('Fetch Metadata') {
+            def ref = pipeutils.get_source_config_ref_for_stream(pipecfg, params.STREAM)
+            def variant = stream_info.variant ? "--variant ${stream_info.variant}" : ""
+            def arch_args = basearches.collect{"--arch=$it"}
+            pipeutils.shwrapWithAWSBuildUploadCredentials("""
+            cosa init --branch ${ref} ${variant} ${pipecfg.source_config.url}
+            cosa buildfetch --build=${params.VERSION} \
+                ${arch_args} --artifact=all --url=s3://${s3_stream_dir}/builds \
+                --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG}
+            """)
+        }
+
+        def builtarches = shwrapCapture("""
+                          cosa shell -- cat builds/builds.json | \
+                              jq -r '.builds | map(select(.id == \"${params.VERSION}\"))[].arches[]'
+                          """).split() as Set
+        assert builtarches.contains("x86_64"): "The x86_64 architecture was not in builtarches."
+        if (!builtarches.containsAll(basearches)) {
+            echo "ERROR: Some requested architectures did not successfully build"
+            echo "ERROR: Detected built architectures: $builtarches"
+            echo "ERROR: Requested base architectures: $basearches"
+            currentBuild.result = 'FAILURE'
+            return
+        }
+
+        // Update description based on updated set of architectures
+        build_description = "[${params.STREAM}][${basearches.join(' ')}][${params.VERSION}]"
+        currentBuild.description = "${build_description} Running"
+
+        def src_config_commit = shwrapCapture("""
+            jq '.[\"coreos-assembler.config-gitrev\"]' builds/${params.VERSION}/${basearches[0]}/meta.json
+        """)
+
+        for (basearch in basearches) {
+            if (!params.SKIP_SIGNING || !params.SKIP_COMPOSE_IMPORT)  {
+                pipeutils.tryWithMessagingCredentials() {
+                    def parallelruns = [:]
+                    if (!params.SKIP_SIGNING) {
+                        parallelruns['Sign Images'] = {
+                            pipeutils.signImages(params.STREAM, params.VERSION, basearch, s3_stream_dir)
+                        }
+                    }
+                    if (!params.SKIP_COMPOSE_IMPORT) {
+                        parallelruns['OSTree Import: Compose Repo'] = {
+                            pipeutils.composeRepoImport(params.VERSION, basearch, s3_stream_dir)
+                        }
+                    }
+                    // process this batch
+                    parallel parallelruns
+                }
+            }
+
+            if (!params.SKIP_TESTS) {
+                stage('Cloud Tests') {
+                    pipeutils.run_cloud_tests(pipecfg, params.STREAM, params.VERSION,
+                                              cosa_img, basearch, src_config_commit)
+                }
+                if (pipecfg.misc?.run_extended_upgrade_test_fcos) {
+                    stage('Upgrade Tests') {
+                        pipeutils.run_fcos_upgrade_tests(pipecfg, params.STREAM, params.VERSION,
+                                                         cosa_img, basearch, src_config_commit)
+                    }
+                }
+            }
+        }
+
+        currentBuild.result = 'SUCCESS'
+        currentBuild.description = "${build_description} ✓"
+
+// main try finishes here
+} catch (e) {
+    currentBuild.result = 'FAILURE'
+    throw e
+} finally {
+    pipeutils.trySlackSend(message: ":key: fix-build #${env.BUILD_NUMBER} <${env.BUILD_URL}|:jenkins:> <${env.RUN_DISPLAY_URL}|:ocean:> [${params.STREAM}][${basearches.join(' ')}] (${params.VERSION})")
+}}} // try-catch-finally, cosaPod and lock finish here

utils.groovy

@@ -657,6 +657,31 @@ def AWSBuildUploadCredentialExists() {
     return utils.credentialsExist(creds)
 }
 
+// Calls `cosa sign robosignatory --images ...`. Assumes to have access to the
+// messaging credentials.
+def signImages(stream, version, basearch, s3_stream_dir) {
+    shwrapWithAWSBuildUploadCredentials("""
+    cosa sign --build=${version} --arch=${basearch} \
+        robosignatory --s3 ${s3_stream_dir}/builds \
+        --aws-config-file \${AWS_BUILD_UPLOAD_CONFIG} \
+        --extra-fedmsg-keys stream=${stream} \
+        --images --gpgkeypath /etc/pki/rpm-gpg \
+        --fedmsg-conf \${FEDORA_MESSAGING_CONF}
+    """)
+}
+
+// Requests OSTree commit to be imported into the compose repo. Assumes to have
+// access to the messaging credentials.
+def composeRepoImport(version, basearch, s3_stream_dir) {
+    shwrap("""
+    cosa shell -- \
+    /usr/lib/coreos-assembler/fedmsg-send-ostree-import-request \
+        --build=${version} --arch=${basearch} \
+        --s3=${s3_stream_dir} --repo=compose \
+        --fedmsg-conf \${FEDORA_MESSAGING_CONF}
+    """)
+}
+
 // Grabs the jenkins.io/emoji-prefix annotation from the slack-api-token
 def getSlackEmojiPrefix() {
     def emoji = shwrapCapture("""