pipeline-deployment.yaml

# Copyright 2021 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#############################################################################
# CI/CD steps for Cloud Build to test and deploy a TFX pipeline to Vertex AI.
#############################################################################

steps:

# Clone the repository.
- name: 'gcr.io/cloud-builders/git'
  args: ['clone', '--single-branch', '--branch',
         '$_BRANCH', '$_REPO_URL',
         '--depth', '1',
         '--verbose']
  id: 'Clone Repository'
  

# Run datasource_utils unit tests.
- name: '$_CICD_IMAGE_URI'
  entrypoint: 'pytest'
  args: ['src/tests/datasource_utils_tests.py', '-s']
  dir: 'mlops-with-vertex-ai'
  env: 
  - 'PROJECT=$_PROJECT'  
  - 'BQ_LOCATION=$_BQ_LOCATION'
  - 'BQ_DATASET_NAME=$_BQ_DATASET_NAME'
  - 'BQ_TABLE_NAME=$_BQ_TABLE_NAME'  
  id: 'Unit Test Datasource Utils'
  waitFor: ['Clone Repository']


# Run model unit tests.
- name: '$_CICD_IMAGE_URI'
  entrypoint: 'pytest'
  args: ['src/tests/model_tests.py', '-s']
  dir: 'mlops-with-vertex-ai'
  id: 'Unit Test Model'
  waitFor: ['Clone Repository']
  timeout: 1800s


# Test e2e pipeline using local runner.
- name: '$_CICD_IMAGE_URI'
  entrypoint: 'pytest'
  args: ['src/tests/pipeline_deployment_tests.py::test_e2e_pipeline', '-s']
  dir: 'mlops-with-vertex-ai'
  env: 
  - 'PROJECT=$_PROJECT'  
  - 'REGION=$_REGION'
  - 'MODEL_DISPLAY_NAME=$_MODEL_DISPLAY_NAME'
  - 'DATASET_DISPLAY_NAME=$_DATASET_DISPLAY_NAME'  
  - 'GCS_LOCATION=$_TEST_GCS_LOCATION'
  - 'TRAIN_LIMIT=$_CI_TRAIN_LIMIT'
  - 'TEST_LIMIT=$_CI_TEST_LIMIT'  
  - 'UPLOAD_MODEL=$_CI_UPLOAD_MODEL'
  - 'ACCURACY_THRESHOLD=$_CI_ACCURACY_THRESHOLD'
  id: 'Local Test E2E Pipeline'
  waitFor: ['Unit Test Datasource Utils', 'Unit Test Model']
  timeout: 1800s


# Build the image that encapsulates the pipeline.
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', '$_TFX_IMAGE_URI', '.']
  dir: 'mlops-with-vertex-ai'
  id: 'Build TFX Image'
  waitFor: ['Local Test E2E Pipeline']
  
  
# Compile the pipeline.
- name: '$_CICD_IMAGE_URI'
  entrypoint: 'python'
  args: ['build/utils.py',
          '--mode', 'compile-pipeline',
          '--pipeline-name', '$_PIPELINE_NAME'
          ]
  dir: 'mlops-with-vertex-ai'
  env: 
  - 'PROJECT=$_PROJECT'  
  - 'REGION=$_REGION'
  - 'MODEL_DISPLAY_NAME=$_MODEL_DISPLAY_NAME'
  - 'DATASET_DISPLAY_NAME=$_DATASET_DISPLAY_NAME'  
  - 'GCS_LOCATION=$_GCS_LOCATION' 
  - 'TFX_IMAGE_URI=$_TFX_IMAGE_URI' 
  - 'BEAM_RUNNER=$_BEAM_RUNNER'
  - 'TRAINING_RUNNER=$_TRAINING_RUNNER'
  id: 'Compile Pipeline'
  waitFor: ['Local Test E2E Pipeline']
  
  
# Upload compiled pipeline to GCS.
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '$_PIPELINE_NAME.json', '$_PIPELINES_STORE']
  dir: 'mlops-with-vertex-ai'
  id:  'Upload Pipeline to GCS'
  waitFor: ['Compile Pipeline']
  

# Push TFX Image to Container Registy.
images: ['$_TFX_IMAGE_URI']