feat: embed k8s rpms in container (#1140)

* fix: adds ssm from docker image directly

* fix: suggestion from review

* feat: embed k8s rpms

* fix: handle fips case as well

* fix: clean ups for k8s rpm handling

* fix: includes and installs cri-tools

* fix: adds missing gpg key

* fix: crictl install

* fix: use correct crictl file path

* fix: rocky9.1 remove kubernetes from package bundle

* fix: kubernetes_cni version

* fix: typo in package name

* fix: skip broken kubernetes_cni

* fix: adds missing packages and installs cni w/ kubelet

* fix: allows setting k8s versions

* fix: small fixes to fetch different k8s packages and ansible fixes

* chore: apply suggestions from review

* fix: spacing in kubeadm file

* fixup: echo k8s version to force rebuild

Dockerfile

@@ -31,9 +31,8 @@ ARG BUILDARCH
 # we copy this to remote hosts to execute GOSS
 # Packer copies /usr/local/bin/goss-amd64 from this container to the remote host
 COPY --from=devkit /usr/local/bin/goss-amd64 /usr/local/bin/goss-amd64
-
-COPY --from=devkit /opt/amazon-ssm-agent.rpm /opt/amazon-ssm-agent.rpm
-
+COPY --from=devkit /opt/*.rpm /opt
+COPY --from=devkit /opt/d2iq-sign-authority-gpg-public-key /opt/d2iq-sign-authority-gpg-public-key
 # we copy this to remote hosts to execute mindthegap so its always amd64
 COPY --from=devkit /usr/local/bin/mindthegap /usr/local/bin/
 COPY --from=devkit /usr/local/bin/packer-${BUILDARCH} /usr/local/bin/packer

Dockerfile.devkit

@@ -79,6 +79,33 @@ RUN chmod +rx /usr/local/bin/goss-amd64
 ARG BUILDARCH
 RUN ln -s /usr/local/bin/goss-${BUILDARCH} /usr/local/bin/goss
 RUN curl -o /opt/amazon-ssm-agent.rpm https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
+COPY ansible ansible
+# Fetch nokmem rpms
+RUN \
+  export KUBERNETES_VERSION=$(awk -F': ' '/kubernetes_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '2p' | xargs) && \
+  echo ${KUBERNETES_VERSION} && \
+  curl -o /opt/kubectl-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubectl-${KUBERNETES_VERSION}-0.x86_64.rpm && \
+  curl -o /opt/kubeadm-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubeadm-${KUBERNETES_VERSION}-0.x86_64.rpm && \
+  curl -o /opt/kubelet-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubelet-${KUBERNETES_VERSION}-0.x86_64.rpm && \
+  export CRICTL_TOOLS_VERSION="$(echo ${KUBERNETES_VERSION} | cut -d. -f1-2).0" && \
+  curl -o  /opt/cri-tools-${CRICTL_TOOLS_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/cri-tools-${CRICTL_TOOLS_VERSION}-0.x86_64.rpm && \
+  export CNI_VERSION=$(awk -F': ' '/kubernetes_cni_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '1p' | xargs) && \
+  curl -o /opt/kubernetes-cni-${CNI_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubernetes-cni-${CNI_VERSION}-0.x86_64.rpm
+
+
+# Fetch fips rpms
+RUN \
+  export KUBERNETES_VERSION=$(awk -F': ' '/kubernetes_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '2p' | xargs) && \
+  echo ${KUBERNETES_VERSION} && \
+  curl -o /opt/kubectl-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubectl-${KUBERNETES_VERSION}-0.x86_64.rpm && \
+  curl -o /opt/kubeadm-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubeadm-${KUBERNETES_VERSION}-0.x86_64.rpm && \
+  curl -o /opt/kubelet-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubelet-${KUBERNETES_VERSION}-0.x86_64.rpm && \
+  export CRICTL_TOOLS_VERSION="$(echo ${KUBERNETES_VERSION} | cut -d. -f1-2).0" && \
+  curl -o  /opt/cri-tools-${CRICTL_TOOLS_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/cri-tools-${CRICTL_TOOLS_VERSION}-0.x86_64.rpm && \
+  export CNI_VERSION=$(awk -F': ' '/kubernetes_cni_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '1p' | xargs) && \
+  curl -o /opt/kubernetes-cni-${CNI_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubernetes-cni-${CNI_VERSION}-0.x86_64.rpm
+
+RUN curl -o /opt/d2iq-sign-authority-gpg-public-key https://packages.d2iq.com/konvoy/stable/linux/repos/d2iq-sign-authority-gpg-public-key
 
 COPY --from=packer-amd64 /bin/packer /usr/local/bin/packer-amd64
 COPY --from=packer-arm64 /bin/packer /usr/local/bin/packer-arm64

ansible/roles/kubeadm/tasks/redhat.yaml

@@ -14,9 +14,44 @@
       not 'kubeadm-' + package_versions.kubernetes_rpm in exportedversionlocklist.stdout
       )"
 
+
+- block:
+  - name: copy cri-tools rpm
+    copy:
+      src: "/opt/{{ 'cri-tools-' + critools_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+      dest: "/opt/{{ 'cri-tools-' + critools_rpm }}.rpm"
+
+  - name: install cri-tools rpm package
+    yum:
+      name: "/opt/{{ 'cri-tools-' + critools_rpm }}.rpm"
+      state: present
+      update_cache: true
+      enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}"
+      disablerepo: "{{ '*' if offline_mode_enabled else '' }}"
+    register: result
+    until: result is success
+    retries: 3
+    delay: 3
+
+# If the rpms for the kubernetes version provided by the customer
+# exists on the current container, we should copy it to the remote
+# and install it with the file.
+- name: check kubeadm rpm exists for provided version
+  stat:
+    path: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+  delegate_to: localhost
+  register: haslocalkubeadm
+  become: false
+
+- name: copy kubeadm rpm
+  copy:
+    src: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+    dest: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}.rpm"
+  when: haslocalkubeadm.stat.exists
+
 - name: install kubeadm rpm package
   yum:
-    name: "{{ 'kubeadm-' + package_versions.kubernetes_rpm }}"
+    name: "{{ '/opt/' if haslocalkubeadm.stat.exists }}{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalkubeadm.stat.exists }}"
     state: present
     update_cache: true
     enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}"

ansible/roles/packages/tasks/redhat.yaml

@@ -65,9 +65,48 @@
     - versionlock_plugin_enabled
     - item in exportedversionlocklist.stdout
 
+# If the rpms for the kubernetes version provided by the customer
+# exists on the current container, we should copy it to the remote
+# and install it with the file.
+- name: check kubernetes rpms exist for provided version
+  stat:
+    path: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+    path: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+  delegate_to: localhost
+  register: haslocalk8srpms
+  become: false
+
+- block:
+    - name: copy gpg key
+      copy:
+        src: /opt/d2iq-sign-authority-gpg-public-key
+        dest: /opt/d2iq-sign-authority-gpg-public-key
+
+    - name: import key
+      ansible.builtin.rpm_key:
+        state: present
+        key: /opt/d2iq-sign-authority-gpg-public-key
+
+    - name: copy kubectl rpm
+      copy:
+        src: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+        dest: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}.rpm"
+
+    - name: copy kubernetes_cni rpm
+      copy:
+        src: "/opt/{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0{{ '-fips' if fips.enabled else '' }}.rpm"
+        dest: "/opt/{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0.rpm"
+
+    - name: copy kubelet rpm
+      copy:
+        src: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm"
+        dest: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}.rpm"
+  when:
+    - haslocalk8srpms.stat.exists
+
 - name: install kubectl rpm package
   yum:
-    name: "{{ 'kubectl-' + package_versions.kubernetes_rpm }}"
+    name: "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalk8srpms.stat.exists }}"
     state: present
     update_cache: true
     enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}"
@@ -77,15 +116,17 @@
   retries: 3
   delay: 3
 
-- name: install kubelet rpm package
+- name: install kubernetes_cni and kubelet rpm packages
   yum:
-    name: "{{ 'kubelet-' + package_versions.kubernetes_rpm }}"
+    name:
+      - "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0{{ '.rpm' if haslocalk8srpms.stat.exists }}"
+      - "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalk8srpms.stat.exists }}"
     state: present
     update_cache: true
     enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}"
     disablerepo: "{{ '*' if offline_mode_enabled else '' }}"
-  register: kubelet_installation_rpm
-  until: kubelet_installation_rpm is success
+  register: installation_rpm
+  until: installation_rpm is success
   retries: 3
   delay: 3
 

bundles/redhat8.6/bundle.sh.gotmpl

@@ -46,21 +46,26 @@ subscription-manager release --set=8.6
 subscription-manager refresh
 subscription::defer_unregister
 
-ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms"
+ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms"
 EUS_REPOS=${EUS_REPOS:-""}
 if [[ -n "${EUS_REPOS}" ]]; then
   #disables the standard repositories which should not be enabled when using EUS
   subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms
   subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms
-  ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms"
+  ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms"
 else
   subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms
 fi
 
+KUBERNETES_REPOS=${KUBERNETES_REPOS:-""}
+if [[ -n "${KUBERNETES_REPOS}" ]]; then
+ENABLED_REPOS="${ENABLED_REPOS},kubernetes"
+fi
+
 yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools
 yum clean all
 TMP_DIR="$(mktemp -d repodata-XXXX)"

bundles/redhat8.6/packages.txt.gotmpl

@@ -15,16 +15,18 @@ yum-utils
 cloud-init
 cloud-utils-growpart
 container-selinux
+{{ if .FetchKubernetesRPMs -}}
 kubectl-{{ .KubernetesVersion }}-0
 kubelet-{{ .KubernetesVersion }}-0
 kubeadm-{{ .KubernetesVersion }}-0
+cri-tools
+{{- end }}
 conntrack
 ebtables
 ethtool
 iproute
 iptables
 socat
-cri-tools
 gcc
 make
 elfutils-libelf-devel

bundles/redhat8.8/bundle.sh.gotmpl

@@ -46,21 +46,26 @@ subscription-manager release --set=8.8
 subscription-manager refresh
 subscription::defer_unregister
 
-ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms"
+ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms"
 EUS_REPOS=${EUS_REPOS:-""}
 if [[ -n "${EUS_REPOS}" ]]; then
   #disables the standard repositories which should not be enabled when using EUS
   subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms
   subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms
-  ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms"
+  ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms"
 else
   subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms
   subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms
 fi
 
+KUBERNETES_REPOS=${KUBERNETES_REPOS:-""}
+if [[ -n "${KUBERNETES_REPOS}" ]]; then
+ENABLED_REPOS="${ENABLED_REPOS},kubernetes"
+fi
+
 yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools
 yum clean all
 TMP_DIR="$(mktemp -d repodata-XXXX)"

bundles/redhat8.8/packages.txt.gotmpl

@@ -15,16 +15,18 @@ yum-utils
 cloud-init
 cloud-utils-growpart
 container-selinux
+{{ if .FetchKubernetesRPMs -}}
 kubectl-{{ .KubernetesVersion }}-0
 kubelet-{{ .KubernetesVersion }}-0
 kubeadm-{{ .KubernetesVersion }}-0
+cri-tools
+{{- end }}
 conntrack
 ebtables
 ethtool
 iproute
 iptables
 socat
-cri-tools
 gcc
 glibc-devel
 make

bundles/rocky9.1/bundle.sh.gotmpl

@@ -8,6 +8,7 @@ echo skip_missing_names_on_install=False >> /etc/yum.conf
 yum -y install epel-release gettext yum-utils createrepo dnf-utils
 yum clean all
 TMP_DIR="$(mktemp -d repodata-XXXX)"
+chmod 777 -R "${TMP_DIR}"
 cp packages.txt "${TMP_DIR}"
 pushd "${TMP_DIR}"
 #shellcheck disable=SC2046

bundles/rocky9.1/packages.txt.gotmpl

@@ -2,7 +2,6 @@ audit
 ca-certificates
 conntrack-tools
 chrony
-ebtables
 open-vm-tools
 python3-pip
 python-unversioned-command
@@ -15,17 +14,21 @@ yum-utils
 cloud-init
 cloud-utils-growpart
 container-selinux
+{{ if .FetchKubernetesRPMs -}}
 kubectl-{{ .KubernetesVersion }}-0
 kubelet-{{ .KubernetesVersion }}-0
 kubeadm-{{ .KubernetesVersion }}-0
+cri-tools
+{{- end }}
 conntrack
 ebtables
 ethtool
 iproute
 iptables
 socat
-cri-tools
 gcc
 libseccomp
 nfs-utils
 sssd-kcm
+iptables-libs
+libnftnl

bundles/ubuntu20.04/bundle.sh.gotmpl

@@ -25,6 +25,7 @@ sed -i 's/cri-tools/cri-tools='"{{ .CRIToolsVersion }}-${CRI_TOOLS_DEB_BUILD_VER
 sed -i 's/cloud-init/cloud-init='"23.1.2-0ubuntu0~20.04.2"'/' /tmp/packages
 
 TMP_DIR="$(mktemp -d repodata-XXXX)"
+chmod 777 -R "${TMP_DIR}"
 pushd "${TMP_DIR}"
 #shellcheck disable=SC2046
 apt-get download $(< /tmp/packages)