diff --git a/CHANGELOG.md b/CHANGELOG.md index b319d32..d261acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Change Log All notable changes to this project will be documented in this file. +## [1.2.7] - 2023-11-06 + +### Added + +### Changed + +- Updated state_bucket naming schema +- Updated unit tests to reflect state_bucket naming schema +- Updated outputs.tf block to not include csr if use_ci=False + +### Fixed + + ## [1.2.6] - 2023-10-05 ### Added diff --git a/examples/training/00_introduction_training_example.ipynb b/examples/training/00_introduction_training_example.ipynb index 2d1f767..304aaeb 100644 --- a/examples/training/00_introduction_training_example.ipynb +++ b/examples/training/00_introduction_training_example.ipynb @@ -1105,7 +1105,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip3 install kfp<2.0.0 google-cloud-aiplatform" + "!pip3 install \"kfp<2.0.0\" google-cloud-aiplatform" ] }, { @@ -1118,7 +1118,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "id": "36329f82", "metadata": {}, "outputs": [], @@ -1139,7 +1139,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "id": "cf82f09f", "metadata": {}, "outputs": [ @@ -1166,7 +1166,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "89b82a23", "metadata": {}, "outputs": [], @@ -1176,7 +1176,8 @@ " 'google-cloud-bigquery', \n", " 'pandas',\n", " 'pyarrow',\n", - " 'db_dtypes'\n", + " 'db_dtypes',\n", + " 'appengine-python-standard'\n", " ],\n", " output_component_file=f'{AutoMLOps.OUTPUT_DIR}/create_dataset.yaml'\n", ")\n", @@ -1235,7 +1236,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "id": "4f1a3660", "metadata": {}, "outputs": [], @@ -1295,7 +1296,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "id": "aa8047d6", "metadata": {}, "outputs": [], @@ -1363,7 +1364,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "id": "01996d4c", "metadata": {}, "outputs": [], @@ -1399,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "id": "bc244ca7", "metadata": {}, "outputs": [], @@ -1424,7 +1425,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 22, "id": "6f8282e3", "metadata": {}, "outputs": [ @@ -1434,10 +1435,12 @@ "text": [ "Writing directories under AutoMLOps/\n", "Writing configurations to AutoMLOps/configs/defaults.yaml\n", - "Writing Kubeflow Pipelines code to AutoMLOps/pipelines, AutoMLOps/components, AutoMLOps/services\n", "Writing README.md to AutoMLOps/README.md\n", + "Writing kubeflow pipelines code to AutoMLOps/pipelines, AutoMLOps/components\n", "Writing scripts to AutoMLOps/scripts\n", - "Writing CloudBuild config to AutoMLOps/cloudbuild.yaml\n", + "Writing submission service code to AutoMLOps/services\n", + "Writing gcloud provisioning code to AutoMLOps/provision\n", + "Writing cloud build config to AutoMLOps/cloudbuild.yaml\n", "Code Generation Complete.\n" ] } @@ -1461,7 +1464,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 23, "id": "2d66e453", "metadata": {}, "outputs": [ @@ -1470,46 +1473,49 @@ "output_type": "stream", "text": [ "WARNING: Provisioning requires these permissions:\n", - "-cloudfunctions.functions.get\n", - "-serviceusage.services.use\n", "-serviceusage.services.enable\n", - "-cloudfunctions.functions.create\n", - "-pubsub.subscriptions.list\n", - "-cloudscheduler.jobs.list\n", - "-pubsub.topics.create\n", - "-source.repos.list\n", - "-artifactregistry.repositories.create\n", + "-serviceusage.services.use\n", + "-storage.buckets.get\n", + "-storage.buckets.create\n", "-resourcemanager.projects.setIamPolicy\n", - "-iam.serviceAccounts.listiam.serviceAccounts.create\n", + "-iam.serviceAccounts.list\n", + "-iam.serviceAccounts.create\n", + "-iam.serviceAccounts.actAs\n", + "-pubsub.topics.list\n", + "-pubsub.topics.create\n", + "-pubsub.subscriptions.list\n", "-pubsub.subscriptions.create\n", - "-cloudscheduler.jobs.create\n", - "-storage.buckets.create\n", - "-source.repos.create\n", "-artifactregistry.repositories.list\n", - "-cloudbuild.builds.create\n", + "-artifactregistry.repositories.create\n", "-cloudbuild.builds.list\n", - "-pubsub.topics.list\n", - "-storage.buckets.get\n", + "-cloudbuild.builds.create\n", + "-cloudscheduler.jobs.list\n", + "-cloudscheduler.jobs.create\n", + "-cloudfunctions.functions.get\n", + "-cloudfunctions.functions.create\n", + "-source.repos.list\n", + "-source.repos.create\n", "\n", "You are currently using: srastatter@google.com. Please check your account permissions.\n", "The following are the recommended roles for provisioning:\n", + "-roles/serviceusage.serviceUsageAdmin\n", "-roles/resourcemanager.projectIamAdmin\n", - "-roles/cloudfunctions.admin\n", - "-roles/artifactregistry.admin\n", "-roles/iam.serviceAccountAdmin\n", - "-roles/serviceusage.serviceUsageAdmin\n", - "-roles/aiplatform.serviceAgent\n", - "-roles/cloudscheduler.admin\n", + "-roles/iam.serviceAccountUser\n", + "-roles/storage.admin\n", "-roles/pubsub.editor\n", - "-roles/source.admin\n", + "-roles/artifactregistry.admin\n", "-roles/cloudbuild.builds.editor\n", + "-roles/cloudscheduler.admin\n", + "-roles/cloudfunctions.admin\n", + "-roles/source.admin\n", "\n", "\u001b[0;32m Setting up API services in project automlops-sandbox \u001b[0m\n", - "Operation \"operations/acat.p2-45373616427-990bb410-2998-4b37-b37f-09ed724e9519\" finished successfully.\n", + "Operation \"operations/acat.p2-45373616427-447dd1e9-9ed8-4831-9b57-ff4a2e50abf5\" finished successfully.\n", "\u001b[0;32m Setting up Artifact Registry in project automlops-sandbox \u001b[0m\n", "Listing items under project automlops-sandbox, location us-central1.\n", "\n", - "dry-beans-dt-artifact-registry DOCKER STANDARD_REPOSITORY Artifact Registry dry-beans-dt-artifact-registry in us-central1. us-central1 Google-managed key 2023-09-05T11:25:48 2023-09-05T14:47:39 3200.712\n", + "dry-beans-dt-artifact-registry DOCKER STANDARD_REPOSITORY Artifact Registry dry-beans-dt-artifact-registry in us-central1. us-central1 Google-managed key 2023-09-05T11:25:48 2023-11-02T09:28:28 15534.510\n", "Artifact Registry: dry-beans-dt-artifact-registry already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up Storage Bucket in project automlops-sandbox \u001b[0m\n", "gs://automlops-sandbox-dry-beans-dt-bucket/\n", @@ -1519,7 +1525,7 @@ "Service Account: vertex-pipelines already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up IAM roles for Pipeline Job Runner Service Account in project automlops-sandbox \u001b[0m\n", "\u001b[0;32m Setting up Cloud Source Repository in project automlops-sandbox \u001b[0m\n", - "dry-beans-dt-repository automlops-sandbox https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n", + "dry-beans-dt-repository automlops-sandbox https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n", "Cloud Source Repository: dry-beans-dt-repository already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up Queueing Service in project automlops-sandbox \u001b[0m\n", "name: projects/automlops-sandbox/topics/dry-beans-dt-queueing-svc\n", @@ -1527,12 +1533,13 @@ "\u001b[0;32m Deploying Cloud Functions: dry-beans-dt-job-submission-svc in project automlops-sandbox \u001b[0m\n", "Deploying function (may take a while - up to 2 minutes)...\n", "..\n", - "For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/0dcd7601-a440-4c7e-8ea9-28d1cf927991?project=45373616427\n", - "........................................................................done.\n", + "For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/127ff0be-d768-4d55-994f-0312a72e76df?project=45373616427\n", + "...................................done.\n", "availableMemoryMb: 512\n", - "buildId: 0dcd7601-a440-4c7e-8ea9-28d1cf927991\n", - "buildName: projects/45373616427/locations/us-central1/builds/0dcd7601-a440-4c7e-8ea9-28d1cf927991\n", - "dockerRegistry: CONTAINER_REGISTRY\n", + "buildId: 127ff0be-d768-4d55-994f-0312a72e76df\n", + "buildName: projects/45373616427/locations/us-central1/builds/127ff0be-d768-4d55-994f-0312a72e76df\n", + "dockerRegistry: ARTIFACT_REGISTRY\n", + "dockerRepository: projects/automlops-sandbox/locations/us-central1/repositories/dry-beans-dt-artifact-registry\n", "entryPoint: process_request\n", "eventTrigger:\n", " eventType: google.pubsub.topic.publish\n", @@ -1546,11 +1553,11 @@ "name: projects/automlops-sandbox/locations/us-central1/functions/dry-beans-dt-job-submission-svc\n", "runtime: python39\n", "serviceAccountEmail: vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com\n", - "sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/a2ee896a-a9dc-41ee-a129-fc000fc59dc9.zip\n", + "sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/b846a071-0e81-4549-be94-6ed6c7181aa8.zip\n", "status: ACTIVE\n", "timeout: 540s\n", - "updateTime: '2023-09-08T03:06:05.246Z'\n", - "versionId: '2'\n", + "updateTime: '2023-11-02T13:40:13.103Z'\n", + "versionId: '6'\n", "\u001b[0;32m Setting up Cloud Build Trigger in project automlops-sandbox \u001b[0m\n", "name: dry-beans-dt-build-trigger\n", "Cloudbuild Trigger already exists in project automlops-sandbox for repo dry-beans-dt-repository\n", @@ -1574,7 +1581,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 24, "id": "e92f3e51", "metadata": { "scrolled": false @@ -1585,28 +1592,28 @@ "output_type": "stream", "text": [ "WARNING: Running precheck for deploying requires these permissions:\n", - "-artifactregistry.repositories.get\n", - "-cloudbuild.builds.get\n", + "-serviceusage.services.get\n", "-resourcemanager.projects.getIamPolicy\n", "-storage.buckets.update\n", - "-serviceusage.services.get\n", - "-cloudfunctions.functions.get\n", - "-pubsub.topics.get\n", "-iam.serviceAccounts.get\n", - "-source.repos.update\n", + "-artifactregistry.repositories.get\n", + "-pubsub.topics.get\n", "-pubsub.subscriptions.get\n", + "-cloudbuild.builds.get\n", + "-cloudfunctions.functions.get\n", + "-source.repos.update\n", "\n", "You are currently using: srastatter@google.com. Please check your account permissions.\n", "The following are the recommended roles for deploying with precheck:\n", "-roles/serviceusage.serviceUsageViewer\n", "-roles/iam.roleViewer\n", - "-roles/pubsub.viewer\n", "-roles/storage.admin\n", - "-roles/cloudbuild.builds.editor\n", - "-roles/source.writer\n", "-roles/iam.serviceAccountUser\n", - "-roles/cloudfunctions.viewer\n", "-roles/artifactregistry.reader\n", + "-roles/pubsub.viewer\n", + "-roles/cloudbuild.builds.editor\n", + "-roles/cloudfunctions.viewer\n", + "-roles/source.writer\n", "\n", "Checking for required API services in project automlops-sandbox...\n", "Checking for Artifact Registry in project automlops-sandbox...\n", @@ -1620,13 +1627,11 @@ "Checking for Cloud Build Trigger in project automlops-sandbox...\n", "Precheck successfully completed, continuing to deployment.\n", "\n", - "[automlops 1fa5664] Run AutoMLOps\n", - " 7 files changed, 34 insertions(+), 228 deletions(-)\n", - " delete mode 100644 AutoMLOps/components/component_base/src/custom_train_model.py\n", - " delete mode 100644 AutoMLOps/components/custom_train_model/component.yaml\n", - "remote: Waiting for private key checker: 5/5 objects left \n", + "[automlops dffe270] Run AutoMLOps\n", + " 2 files changed, 2 insertions(+), 1 deletion(-)\n", + "remote: Waiting for private key checker: 2/2 objects left \n", "To https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n", - " 70a23dd..1fa5664 automlops -> automlops\n", + " 9002cdb..dffe270 automlops -> automlops\n", "Pushing code to automlops branch, triggering build...\n", "Cloud Build job running at: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n", "Please wait for this build job to complete.\n", @@ -1658,6 +1663,14 @@ "AutoMLOps.deploy(precheck=True, # precheck is optional, defaults to True\n", " hide_warnings=False) # hide_warnings is optional, defaults to True" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41d0309f", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/google_cloud_automlops/__init__.py b/google_cloud_automlops/__init__.py index c13eb2e..79743f0 100644 --- a/google_cloud_automlops/__init__.py +++ b/google_cloud_automlops/__init__.py @@ -23,6 +23,6 @@ series of directories to support the creation of Vertex Pipelines. """ # pylint: disable=invalid-name -__version__ = '1.2.6' +__version__ = '1.2.7' __author__ = 'Sean Rastatter' __credits__ = 'Google' diff --git a/google_cloud_automlops/provisioning/terraform/templates/environment/outputs.tf.j2 b/google_cloud_automlops/provisioning/terraform/templates/environment/outputs.tf.j2 index 90ba201..3b7aa51 100644 --- a/google_cloud_automlops/provisioning/terraform/templates/environment/outputs.tf.j2 +++ b/google_cloud_automlops/provisioning/terraform/templates/environment/outputs.tf.j2 @@ -24,12 +24,12 @@ output "create_storage_bucket_names" { description = "Bucket names." value = google_storage_bucket.storage_bucket.name } -{% if source_repo_type == 'cloud-source-repositories' %} +{% if use_ci %}{% if source_repo_type == 'cloud-source-repositories' %} output "create_cloud_source_repository" { description = "Cloud Source Repository name." value = var.source_repo_name } -{% endif %}{% if use_ci %}{% if pipeline_job_submission_service_type == 'cloud-run' %} +{% endif %}{% if pipeline_job_submission_service_type == 'cloud-run' %} output "cloud_run_id" { description = "ID of the created service." value = module.cloud_run diff --git a/google_cloud_automlops/provisioning/terraform/templates/environment/versions.tf.j2 b/google_cloud_automlops/provisioning/terraform/templates/environment/versions.tf.j2 index 93a8eed..53dd3b8 100644 --- a/google_cloud_automlops/provisioning/terraform/templates/environment/versions.tf.j2 +++ b/google_cloud_automlops/provisioning/terraform/templates/environment/versions.tf.j2 @@ -12,7 +12,7 @@ terraform { } } backend "gcs" { - bucket = "{{storage_bucket_name}}-bucket-tfstate" + bucket = "{{storage_bucket_name}}-tfstate" prefix = "terraform/state" } } \ No newline at end of file diff --git a/google_cloud_automlops/provisioning/terraform/templates/state_bucket/variables.auto.tfvars.j2 b/google_cloud_automlops/provisioning/terraform/templates/state_bucket/variables.auto.tfvars.j2 index f6b3681..c2b4524 100644 --- a/google_cloud_automlops/provisioning/terraform/templates/state_bucket/variables.auto.tfvars.j2 +++ b/google_cloud_automlops/provisioning/terraform/templates/state_bucket/variables.auto.tfvars.j2 @@ -1,4 +1,4 @@ {{generated_license}} project_id = "{{project_id}}" storage_bucket_location = "{{storage_bucket_location}}" -storage_bucket_name = "{{storage_bucket_name}}-bucket-tfstate" \ No newline at end of file +storage_bucket_name = "{{storage_bucket_name}}-tfstate" \ No newline at end of file diff --git a/setup.py b/setup.py index 1f4b04d..9686adf 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name='google-cloud-automlops', - version='1.2.6', + version='1.2.7', description='Build MLOps Pipelines in Minutes.', long_description=readme_contents, long_description_content_type='text/markdown', diff --git a/tests/unit/provisioning/terraform/builder_test.py b/tests/unit/provisioning/terraform/builder_test.py index 29c5b26..a531ecb 100644 --- a/tests/unit/provisioning/terraform/builder_test.py +++ b/tests/unit/provisioning/terraform/builder_test.py @@ -422,7 +422,7 @@ def test_create_environment_variables_tf_jinja( @pytest.mark.parametrize( 'storage_bucket_name, is_included, expected_output_snippets', - [('my-storage-bucket', True, [GENERATED_LICENSE, 'bucket = "my-storage-bucket-bucket-tfstate"'])] + [('my-storage-bucket', True, [GENERATED_LICENSE, 'bucket = "my-storage-bucket-tfstate"'])] ) def test_create_environment_versions_tf_jinja( storage_bucket_name: str,