From 8bcf1876595f7cf16d279c97c7bb4ab3f25e7697 Mon Sep 17 00:00:00 2001 From: burbanekGCP <145143298+burbanekGCP@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:50:38 -0600 Subject: [PATCH] Initial commit, cloned from vertex AI qwikstart folder. with modified requirements.txt --- .../vertex-ai/train-deploy-tf-model/README.md | 119 + .../train-deploy-tf-model/images/clv-rfm.svg | 1 + .../images/vertex-ai-overview.png | Bin 0 -> 76160 bytes .../train-deploy-tf-model/lab_exercise.ipynb | 1062 +++++++++ .../lab_exercise_long.ipynb | 1931 +++++++++++++++++ .../online-retail-clv-3M/Dockerfile | 21 + .../online-retail-clv-3M/cloudbuild.yaml | 5 + .../online-retail-clv-3M/requirements.txt | 3 + .../online-retail-clv-3M/trainer/__init__.py | 0 .../online-retail-clv-3M/trainer/model.py | 149 ++ .../online-retail-clv-3M/trainer/task.py | 34 + .../train-deploy-tf-model/requirements.txt | 12 + .../utils/data_download.py | 188 ++ .../utils/dataset_clean.py | 49 + .../train-deploy-tf-model/utils/dataset_ml.py | 72 + .../utils/dataset_schema.py | 27 + 16 files changed, 3673 insertions(+) create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/README.md create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/images/clv-rfm.svg create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/images/vertex-ai-overview.png create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise.ipynb create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise_long.ipynb create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/Dockerfile create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/cloudbuild.yaml create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/requirements.txt create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/__init__.py create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/model.py create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/task.py create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/requirements.txt create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/utils/data_download.py create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_clean.py create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_ml.py create mode 100644 self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_schema.py diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/README.md b/self-paced-labs/vertex-ai/train-deploy-tf-model/README.md new file mode 100644 index 0000000000..94888f9a8b --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/README.md @@ -0,0 +1,119 @@ +# Vertex AI: Qwik Start + +In this lab, you will use [BigQuery](https://cloud.google.com/bigquery) for data processing and exploratory data analysis and the [Vertex AI](https://cloud.google.com/vertex-ai) platform to train and deploy a custom TensorFlow Regressor model to predict customer lifetime value (CLV). The goal of the lab is to introduce Vertex AI through a high value real world use case - predictive CLV. You will start with a local BigQuery and TensorFlow workflow you may already be familiar with and progress toward training and deploying your model in the cloud with Vertex AI as well as retrieving predictions and explanations from your model. + +![Vertex AI](./images/vertex-ai-overview.png "Vertex AI Overview") + +Vertex AI is Google Cloud's next generation, unified platform for machine learning development and the successor to AI Platform announced at Google I/O in May 2021. By developing machine learning solutions on Vertex AI, you can leverage the latest ML pre-built components and AutoML to significantly enhance development productivity, the ability to scale your workflow and decision making with your data, and accelerate time to value. + +## Learning objectives + +* Train a TensorFlow model locally in a hosted [**Vertex Notebook**](https://cloud.google.com/vertex-ai/docs/general/notebooks?hl=sv). +* Create a [**managed Tabular dataset**](https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets?hl=sv) artifact for experiment tracking. +* Containerize your training code with [**Cloud Build**](https://cloud.google.com/build) and push it to [**Google Cloud Artifact Registry**](https://cloud.google.com/artifact-registry). +* Run a [**Vertex AI custom training job**](https://cloud.google.com/vertex-ai/docs/training/custom-training) with your custom model container. +* Use [**Vertex TensorBoard**](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview) to visualize model performance. +* Deploy your trained model to a [**Vertex Online Prediction Endpoint**](https://cloud.google.com/vertex-ai/docs/predictions/getting-predictions) for serving predictions. +* Request an online prediction and explanation and see the response. + +## Setup + +### 1. Enable Cloud Services utilized in the lab environment: + +#### 1.1 Launch [Cloud Shell](https://cloud.google.com/shell/docs/launching-cloud-shell) + +#### 1.2 Set your Project ID + +Confirm that you see the desired project ID returned below: +``` +gcloud config get-value project +``` + +If you do not see your desired project ID, set it as follows: +``` +PROJECT_ID=[YOUR PROJECT ID] +gcloud config set project $PROJECT_ID +``` + +#### 1.3 Use `gcloud` to enable the services + +``` +gcloud services enable \ + compute.googleapis.com \ + iam.googleapis.com \ + iamcredentials.googleapis.com \ + monitoring.googleapis.com \ + logging.googleapis.com \ + notebooks.googleapis.com \ + aiplatform.googleapis.com \ + bigquery.googleapis.com \ + artifactregistry.googleapis.com \ + cloudbuild.googleapis.com \ + container.googleapis.com +``` + +### 2. Create Vertex AI custom service account for Vertex Tensorboard experiment tracking + +#### 2.1. Create custom service account +``` +SERVICE_ACCOUNT_ID=vertex-custom-training-sa +gcloud iam service-accounts create $SERVICE_ACCOUNT_ID \ + --description="A custom service account for Vertex custom training with Tensorboard" \ + --display-name="Vertex AI Custom Training" +``` + +#### 2.2. Grant it access to GCS for writing and retrieving Tensorboard logs +``` +PROJECT_ID=$(gcloud config get-value core/project) +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member=serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT_ID.iam.gserviceaccount.com \ + --role="roles/storage.admin" +``` + +#### 2.3. Grant it access to your BigQuery data source to read data into your TensorFlow model +``` +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member=serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT_ID.iam.gserviceaccount.com \ + --role="roles/bigquery.admin" +``` + +#### 2.4. Grant it access to Vertex AI for running model training, deployment, and explanation jobs +``` +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member=serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT_ID.iam.gserviceaccount.com \ + --role="roles/aiplatform.user" +``` + +### 3. Creating a Vertex Notebooks instance + +An instance of **Vertex Notebooks** is used as a primary lab environment. + +To provision the instance follow the [Create an new notebook instance](https://cloud.google.com/vertex-ai/docs/general/notebooks) setup guide. Use the *TensorFlow Enterprise 2.3* no-GPU image. Leave all other settings at their default values. + +After the instance is created, you can connect to [JupyterLab](https://jupyter.org/) IDE by clicking the *OPEN JUPYTERLAB* link in the [Vertex AI Notebooks Console](https://console.cloud.google.com/vertex-ai/notebooks/instances). + + +### 4. Clone the lab repository + +In your **JupyterLab** instance, open a terminal and clone this repository in the `home` folder. +``` +cd +git clone https://github.com/GoogleCloudPlatform/training-data-analyst.git +``` + +### 5. Install the lab dependencies + +Run the following in the **JupyterLab** terminal to go to the `training-data-analyst/self-paced-labs/vertex-ai/vertex-ai-qwikstart` folder, then pip install `requirements.txt` to install lab dependencies: + +```bash +cd training-data-analyst/self-paced-labs/vertex-ai/vertex-ai-qwikstart +pip install -U -r requirements.txt +``` + +### 6. Navigate to lab notebook + +In your **JupyterLab** instance, navigate to __training-data-analyst__ > __self-paced-labs__ > __vertex-ai__ > __vertex-ai-qwikstart__, and open __lab_exercise.ipynb__. + +Open `lab_exercise.ipynb` to complete the lab. + +Happy coding! \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/images/clv-rfm.svg b/self-paced-labs/vertex-ai/train-deploy-tf-model/images/clv-rfm.svg new file mode 100644 index 0000000000..d86723d0c1 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/images/clv-rfm.svg @@ -0,0 +1 @@ +Timeline1Layer 1HistoricalNowTimeUnknown \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/images/vertex-ai-overview.png b/self-paced-labs/vertex-ai/train-deploy-tf-model/images/vertex-ai-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..277f151656f0802f969170da3bbbf5eb99450bf6 GIT binary patch literal 76160 zcmeFZWl&t(ex#r7)vH(UFgY1fWCT2f7cX8Qi;D> z#l^++*~$7j@T+t}t@IMbic6Wg;N?lnMP}iBrbBr^c6K(gnwT%_e|U9Dz_H6gDlsH& zaVYqq|9*wP=tUrc`}=#~8x>(+{UGJf0Rf~?7$kqazNmdO_|JDNfdg(x>tM_9AFqlD zc14PP0sr?)4FnYqQZYh73I0#>@YR1A^Vb;uF{)BB{;IB$k#FDrz9jIC9AusGKh2xs zlQJx|idqr}JC5X=MxSS>`s4X9#X?!Zr^lbe*gY}Cs5FXS`E#IO$`we_P>5&DA6e_R zd%{-)Mbuqy^+trSmT9K7@qy$2nn@}ZFdSy`*~*X5Ywcd$SD$ODth+_-e*JpI5wJIz zmzO0;SZle8DI+68Wzy`~OFNRvBU-UaR^fKX;>DS8Uk&}&()hXoBTelOtFsL4tT7p% zD17GEYVqzqlK(NB{@Os)t!r{}a*tP9v$n5nsp0xyy5i73`gkdA1^YjY4(Ba@mA`1H zHac2NI7(*o`bARdq8p`T8m2O>s!%~B?#5BZiwfP2Z=Y*Ss?07`THWrkP*70zR_i{P zE0vlj2G#B_mJ5F2gwM>Znl%Yh&%$+nnV#^cefTi|TS6AuS-lWS%3yb)1kvvJxa5{J zlr&LRgP$uLfn~~ny4uz)hbo&$Cr^+BR$<`yv{HY$GZ-MLTF$>1%d|KJ6NG@pu<#1} z_n!WHeW9Sey}ey8w&4%htb>X9yo=DW!5XmS!N^qudmUcS(Rg)QXE)EALeIQyXXAs4 z^K~W_<{hq8xo0?>^JHFC=zrRD=n=5YLO0}{GQ0H9z11Kvqe)guS{%i&R*gLe2Bm~) z9PSC9E!v4-u-VdTr7>tMgQxKEop~a)TF30`^Uc0GF(1poKTVhH17Or~3%{J1Z%@}c z^42;(-re89vy}S(yfhdoLdQzap*Z4pKeB17@9=&MFJU=c-|CM!xGm3Ik!vwVj<_BS z{_{+eRAYQmLsx%iV{)0?llD$~Y2zl7%iOq{pjb1++0VGi{g_j$(Ux&wfs36}f5O+B zcE6yY)$;%uTNgeYMbZ@t{^=h!WIH$Y zdk^O->>m51+dc!Qg4pNbDSe^l^7^v-^9K`BH#lZW^Xa02taqggR+aj9Xm@AlP+PBS9`{&o978{fdl5J<5kr#zmh?wFFci8R%gX*vxrls!Hima7^_D#Ut`lT zNk!R1uGf{lK@L{=`RUilf_XO1WS+s)FP2br^59?hcVRvgHL`qG9ucj(8*^9tBe+b5 zf13yF$KONtNqL{+(2#X^u@g~03=6+$(2YmT&dHf(6gp)x_C@km$KC03=LDVOhK^bO zxVcn1BUrBE(fwA4@`K9!#8IQ;0>VfZmuy95U1>78cszyi=>Af}>E3i5{`)5v@tF#> zc7k8G$AqT3H9g^Vys4#u@+D+(>r_C{i%p@D&7#kzAZ36~mJ zEJh$ge=ab^r&QnG-f+IyyKAnjRJP1F6*}!@*_=Z&P_%C&EEwcv5Lh z$cJp!3SQy!mBusnLSY!fHb~=HJl!p~#8=)ew&!Gh(b+}b3BOtzY~2O1IYiK7T3XAKz-0vS>qfUIW`_N`B;^F>9?i;k;N}Y18&+(#Dt>qhb`amQZ zdtS(yPGlmDLMqEFF!uKLc5@o}mDork?UpYh*CkZ<#EkaL_0N2#fvz$m8La9p<6n!X zDvgib6Rh@BLZ$loD?2Nh{`8!w;(#p#ZGVofSe~(HT0d@aS=8$Zd-EE9t}K}D+!V6+ z9=*6&wN%8(@$mcL2u+Y~-N1VlsVq*y_)4at@2zHUYRz-#cVU&;YW0I^!A&x04UWBl zi&bWu%oXkqtjdep<>YYPV|ayxmrJN((tu)HtS$NY4jegMq=;&>+A8n!Ry;$EJ|*Zx z>twA1b#RR|h*WaJ1*Q7!A8UF~@yV`AAV7Q>@MNpGqE^$P8`N4wU*zK-S6**TY>>M< z8IPxicQ6@Mb0P{ESCOqpwBm@i(l{pMQpsi!lqSEM)Kn*@>qRiTADSuMnno<$yYDCRQCXPX;SuIC7nTR1bkjiAoZNEEUZHwTr z>!ufW%Ty-~+jMQ=g+_0!Rmx?|LGs6FF)Cj|>rts>+~4XpI_;PDf3e(Gz+tZRiPc(d z^C;wIqQDL;c!=k$bN8$5!}#1A?n{56PYmCJ!?$k&7xcjD)}^*R^m*#*Pr42DoP8ky zPHxet$u}|-O%~zXgBwRjkqtI$o2!ra?n}Lh1{Z;L1MxrUkJTs>_s=Ht4o-W*F!Z9v z5!est%u2Xu&UZ~0bL(yj|K%dL|bHqg1M(UU&QR(gawR){@K3Uc^X(4>C5E7&oyr z1|r8cG>QejaPL1R#YR2@4xZg-BUdM3b0!vSf)e1yv)%i!(JktJZO{{19{(x0_6naJ zE+pozV?a_ojTzokg-$R^<2_o%OKBS2a=KSHhX&R7lHs853th@X%Q3cp{N_QIaVYux6_2NqN$x$wk`3XV@oU z35EK{zmOLnOI6GK>&wg5Iou(u5sgZ*a{kU%8e3y49;VBXpoRW;V?N1f-M}wIZTBUC z@y+?dH3g;%t;MT0WR}d7l3_$4=h##q8)Or>558>lic9zUKl}cMn?xC z*t&y+la5KFASVX557qhj7buAQJ#JEi6n5{?1Eh?fJ^!#u<2p+ z4kzl7@un+EZ#@9?_Q_vtWhfZ~Ovm9Z2rdkjBlQo0!~U+Ceq+4|8_$#-2>(XEYEBD4 z1bjUq$#8OHLT2HR_$v1qd&*bENuSK;W-bi@Q~)lJ&ce~t7Y=HuPf}llP_}WUW!WVhrK5S1gIiC{{Z--$Yn! zchKw4E%JpWi~c=bV8ujMf~BvzxDY^ezLQx*i|Y99Q16v<*@ID>=g4cmo3Ip&rsh9N zx4x!|#t$G^fO#AKaJBzt(RuIdj~P<>^Z2=sivAd2&U%|t4BDaLwknL}8Gbq*Z``Gj z7~&*Xk_;B%MkQ)htn>!!DSU73l*PU29Iv8i*1;_jJ(6alZ>|B7B=)EsN057F5g?3CoRt-xr0i`e;avUKol? zX1Fst1OhQ!{cA8)X4y^zHcztl?{)E90tO0Cfj(so46hhN+#+sj(pDQ@RiYS zISlfEC3v>!cnWW-DG=nA640RN$urJ$cL!U3l}$m3z}^cv6H@%lD_3ujGT?AX5=5>IoEAyxI#JAkh!Vp_DPA8N4o4{Wdr#-Vy`h zm#;InYGd6N=J0-S{iv}soI>wj8Wh_9Z8dYLmVP9-CpqA9F1x!q4 zr1+}@k}H<$&Ds{UVMQTu5$A^}SNFH4wC6?E(FmhAC)!+j#@FV?q5__ zYMl8B)ZTEkH`~#FX5MHVaDfqOq|_d_guVv-F&4!`2ZWkz<=x9a5lNZ|9EbnqCfz$S zxt;J<+;33olvyF_`6@jO6eC{#SSxbe?5Cm!FQdtrE2_G|2c_yC#|ms0K|DnAPuP1SO|sk*p4oEy=`J+Row+{} z&u>t%VC!3V$l@9$te^lRbh2a$ed)|Q#|VlM$ES{zz$W8(WkKc*N+=$%VI6{cUDq-O zK@}NYUj>(p`p>W0{W9*W;Iw0Prstr2MJh8@Nk$j%=4+J-?T|sm!Vp8%HZNviMEJRC z=;pPSe>wf5VE)E1P&S{B$V{3O$@5$ypZ7EJT>08Wz(RqOB(Tp6XzQrNykGQZ@Z3wo zuZ~thGo>oj$9`9BhQGc-*{#S4>8@R^Ny+6i>i|Oci*EwgrCf zD3A?V_)sX=g@}|?D4Pvr3vNX~Ug7g_?(u~-2LvYKi8oTmKdf8Iv%sD%(qYl+tJpU@ zi2SU!lh=_^lotTw`S?(~6}z45U@Y}v;M?@1*X8azW{aH?RcWCR8A7hqMbH~6>Dijk zMWY{;#BH<@&u<9Wi+>9Z(;;Mn98t6-W^fZ{Lh+*6Uw2!n-d; zV88SCgZcFmgYIO_=Y=?&+u@CsvTiG38k<#a3L<%}elK9kNev;^aUeFTDB=hpD-lq6 z(R8=wqalkZ8Ke7)$KyoT@nlwTd`n1(03Dz}Z<_8OR;{SL7mC7o`8y?nWa*c*x<6`A?p|GA&!~XL zanHY?%cKd6BPdcdW&Xu1p!(T@z_5pR;t5sW-39!CdCxxiq4QPxxcSd_gKugbtwLN* zS6>cG^oC=}=lY!>R#WzbQ?3p(m}IHO8cc$~U*jqJ>2;_!%g_^XNV}w`u*#IP)5!Gu zw!^mK$URtV?g%;Dk-T0v9?rZ|XhainNBUslQ#_rgs++;?rclxDn!P=il%OB9fTp_e&X?gBII&G7HV`Xj>p*~atsYUL5UZ$?_4Jpc_VQ!i(L`n_jvyb!KX4%?fiI& z4cPQI*za|iU8i9&e~+$@ez&F*_EVOQ*a+WS*LGS^Kp@tyY2y0c^`==)g; zz3zIQEhDS-D)gX14h_b8`3h^oS|=;QqUVW7-7d+rdeb0mdYywh{0YsEHFj6?=ku*j z={hUnprJgUY#k9dNj?_yroEu4o_}T{!LD7YnmxSL7?BW-T$*@Hr$-IqvQJ$MXbxt7 zk9(fa&lVd@7p0`*OF6h5mCkm4bF4W~f(_6V@*MbvX47YM@hKZ)D=p>(RR;Yc2TH|g zq81-RwW^w>9lIYS_{-~VQG+m8kp{-$?geMjGX zQPaDm+Wyn#e&emq_TSVqy@x&Pp8*h{fInRV{N8*kJjee|^M7~qe~tG4sk4cN4ff0? zhHmGsrY7qYoTJ$VYJ;e+KB8u zVz_a6pVlbQ$e@8+v}aqxcciS6;vu<^{pBPok)T2n+cvH&Jb+y0+n`F<=NG-`Q51^yQ+AbJD?lnmOYLR;aGG&hz$nFh;O3cM0AAjT7bWY@c_0D4_#1h5GEil!G&;09`r`r= zXs|$h;AXZ2mI^_$o~U2ue~y43n4>GwJSp%aXHzAaus<)qr~b{L^Z)na{No5jFwSC~ zB|QiouFC6I7TBHRsO{Jq(0nH$JQvK}8>y8qJUh6d-m}b}(u=%jHzp|azHH+`bZvVj zHV+qVmezgyWWU~|Iv*3m^Jk#phl87_A%mT0jF>^Iq^*MXDf2fNI(XXb^f~G5X8Z=d zlX{DJwguxUi~Yf;W5i-3HMhFtc}wjjyXSo2${gv68vDcQf)Do(j~B89P&6Lh=M@`g zZA=J_Z1WG^Y6*YNvn%emDFu#})5&zY_aGD|+S1(+KeY3P2wrV>rX!y5D3hqYai)8M z^}yLFI&XYYI2u06R{K(2DPst%`RajyuW_9R4M^vO{}rvv{+AUm^u?`?zc zsmmSFFT%6?7ZgRv{oKPZe)33gyQ>6V!EJf}VI=4Rz!3=iWKxJl_K8J5$e>)1zsvKC zM*|~N7cKfT&xVHstL+pYQa9;6D?0_v%e8KOL*Sl3+crELL@rfoV0tfoQwFK=mF5R6% zHM?d?0*A4H9LZ6N|Iz}i)p(t?%wt6NzK0^V25)qq$DCY2;7ZA3kN>DJ2f_oVMbAGe zD)#}9m{!Kb4foue`bZ#&p`Hk3Y*pB=sM98a2h14es`NKhjU{owt=du_vm z;dN{Ya&LGj2kQd(f8tJlYB;qBwW?_kTvnUaTw{d}Zyq4um9|m&U^3e02lM*;N4Agt zUhK!5RJ)#|#UgpdhqFgNYwZ>ipC?Z&!k)o|w~W>gLcf8xgYme~bdfwv-CCW63%Bfc_uG@On`7^CYqLo|A$|jNunIdTjE~H% ztmV&at~1+6UTEof$%PwBLIQ^}sLiM1gS%C2g#N@CUl-LnJrc3`7t!i(e?=hQw)%nJ zZD1E-W-!TqKY3f~ai>FgUqdhLKbmLc8x!J-M^ zr>?Ls?gHh1?Xqv(o4bp5oH9DHfyi(5H#+X4?`~E|r9zI@U-U#lVGi99CfRp<%P!6e zG4rv^03iP_IJG=A(!?H^PKO@ba1a&B=l+=NFJhCM5^xKRdhe=@MMcXxyp3K69oKF2y}u#-TjS~R-3zYkR4Em5_U^Y)ufWj z2rO1CB-gCBoUinjj%~rcgfOsH06g<1z;SK{Wz7xf}tw~7FtxzX;;LXAoHA`9<95Ic3%W>-*z`BD;Cwe~^3H$05BynJuEIB$`y$o}St z!E?l^%NpZ94C+aP2bbw>)LyuY))8SUZ!M?Xkd^+*zWTj-6b5p zu|H*k!{*FvK35Lf)&eN#-4ToSIcQf`*AizygAq_`_Pxlb!D0=a%lRlUA>oZoCU-fA zPD|@xYr9x+AdZ}CwoIFFx1YH;oI!L1<=?S@jLVN}93$suugCSCk^6xLY0|l;V(_7| zeGR!M^oFN8J>c@KpO|3hVo&&TlYNe6!+X=^ub->GiOHfS*-pi(k&+449k868Z@s0Y z`zoD|MX%G^+2V3K-Cm|y*W(NIh_tA{O<_I_fjLJ*lV1lohx5%H$yT3dotczJL!Wb^ zBAvYY4$+riT4^30JkPfW6|Wmw6Yn7!e$?{0>m#3QY>PE@gAB#enX&+SaI-rE)!dWJ z(bQfg^fsv02i2sdty2S05&=;;N&q69 zv{HO8RuS<#LU&TBeD5_v>elQN->*3Ozk@*2~S040c`YN@W_tbvr-_ z{MXffA#7-sfk>WAriSVFS`HLq-rPm4T2m;dhw~5&LHFyEh)&I<2O$4gE3ImA z@bgnkAi($-^hKJ5r=Lw>9-n&L<|rSs9{hN`cLrBcK@fYxE2&0q{tfKOc>FYFF>)U! zdfW*%`(=IW%hL0Vqe^f%+__n;Z+zwnS@fwf-b#ed9zUdLHz@t2tb$Rw_VxuR%MQBg z-)syYl*JqH_Di4s=rY%bJ4T5Vn!@(g;kgGKM*YrViyRdQ+G$Ya=jx)0yZBH=414o% zI4rkIP|s3>4WnTtR~m~&iHX?4?Rezpq}PFcnk3Yc!|{2dF@%D3CCUtL7ea03?=PRTZ2Xz+Iy6y4uA4OdHf#js=GL6H5vq0)|a99FWsa(udq0Fbw&W{pE7*tHN zYE}9hBl|uK?jveOP0Jt$lX1;E+hGtK39A z{+aP5z=sJ)r-)z~iBxsvvM=Dk&%^%q%do2ub%$xh%0MN|_gZivD1@`&pbDKff%4^_ zm|LLXwm80y=MEO(E#&pxSty=nm#L+KmWqa3eN0+41~JVr5|S?dx%Ziq`4R2>+{*@m z9lr5zQH{sbqLbi>?&tW z{O?#4m*&e@mFrl%8H&wDqyo{1(jymnjR77?qREGiDw@pI-zq3m+podmX>Q&)Y_Lh? z^kV&NB_e;Xb-8H(B)du&$V!pD7RcsiPv& z>g@5)TvCB-dhX)vk#4mR{LxYe zamVu$zVq=iitM<2Dwj<=hBR}SK!erFR{{I3ynm+N0sjS^NEF^MP^n-JWMd!B+6c^z z5z6NWBQ`tjhkg~u0FQ?w>bxTS#ka#Z3T%`JX;Sw|0qI6fH zb-vDnc+qh$^13u-0}zmyPJGQLasX6)ztFa~$!VZ0u)sYOWUox)t%lZst$!*@pVL2C zUWnb!h~KvBeBlCFtCVj$BIayFrBV0_XSi~6%%y)Hok?rb^Wau9^F14*d=dbl%WW>G zgo_P|;^ZlG*L`0;3MRUeY*0BJHWzV&B$HhY_9{0@wBb?XV&SBTBIm0Nz~~=ac>t*g z%wU<_#kRPzD!B@sPOH$)Z0s4g5Q|*6JZ9FurokZqxn4Vg`03(`c8964-pwhOA4Imjjdh9kS)@3^vdl7IALZUM(t(Kfw zbhN8BSKIE3G|GZCgasvXg~AfoNxI4~4L0BagA=`f#r#eE-pS9aWk==7olwkaW6Qk!W3^Q5dRp@ zRy{6|%_eFGG^DWzG-QbbwD`f{RDpT2r#hx{V|+RJ#TWn^zpO2ZoGgE>E;JdFzk*+EQW*u6e`Q0>JE;>b!PuvaM9(y|SWE zYHSi0tL1!A;A@&{iI50ME2OsQtB6SXS)B)n1) zNURkZ1PPBDGo96HHh5cHQ#y@VpdyQB+P(-XqE@%#Bl*npT0cr@Q<@^+|EgtecPXKz z{rrQH)ZT+r1{8|zPvgG(_;MxF>*hSQIY-6lbjdmNVWn~YBNOQ+Z33@jzGZe;Z{HB_ zJjH(?8hhpWI7h_V!k!-(;6TaVH*9|M9)eQ^>)Fxfw-%w@*)RBwC{%D0i$7z;)-e+A z$g4@P2kq{B0RPSt_sqlM=>@?iRMllSO|g#x71rU~_4}<4XB$ipS@dvEpA2>b6cL80 zJdvJAEL~zWQFZd9z+SSirTwMPA!t)J64`K%G4 zrZ@X3N(oTbqW4S*2rw`i-U)`m?qZ31x;;O6)h!0m>6wdB#MV=M-P#>ZFHs*{AQ_Ip zqusQ!Jziu{<`WD?GXJS;urpIS*JyY7DwkbY{&AR~t0tAJ;*21n$x9fbwvmV-Z*LPM z*ebOfopMc<)_IG1=geSm)*Ei>^KHMu&sIvAE7zp+X_R99_9wcx*3rujRbJzh^O`ip zzFu1?c|zN8^nWe7cM0x;jt%l^K)IqlJ4JB}wYrGt$d?6+Gk8y*fYS!-?o;RSPp60L zTB!2&^e8vp%ipp)%a&Z^N;29jO~fF8pa$e@r5OWOt)Q(EATlqMA_!2R3@>(muifs? zdd#);6c9qKT>`s1nubsjB+0PsfOIfd77#}iz-atHF_OWEfyu%)`0~8-r48zK76e_TV*MPK?)V&tR_3Z4U zt@Bi=WEEM@9>iM|Xlk$_i$IRm59*bxCdDTKxRh6shRT9p)%28i;fB8f5lgapc8;|d z;jl(LH8NCRIQ0`t;Uu7f-M!K)LFq+M477CN>P1+p)3z0rsUmCaQCs^LDJtAEPop9P zdkwdr@S8)6sgwae;`};0PSqE_yO`ZSK3VZ7(kVpO=|Cw+Z+|J8@xB;_6}2_URQ}<^ z{%d8NSNb3KHl{HF@)6qV;KVEa=>xTGvy8w&k%n+hS%#sfURfjNsj5KDVj0URuyx&W_>(*%cu%K%0l0n8k(j2Z^? ztfcqH2jr>giKor3VFAn95GO1mR@Fs@Be*y)SFvUbC=x7Js|iyobh=Y2`nnkf7GoC2 z$FWh?vdCxWN(YipxQ2E-gA&J1(R=q>290S{QiTkiKtQ)hrg)bnKQJkzclh-JzX|#Y zY84&|U?f&Vk|@!D>d#EWM-%vZO#};H6SV~J_fYO4I#RTT#?^<-&VKiKhPW4@hCyAK61bdEGy@?LFtOg@zOu;Q6Z;T4gBqeA zG?PA{00Y?%U>Hl4ibbO0RbbB2y_KE#^3hAFLS<&#l+9+{*Z|dEs}b5S_1pX{tpW2dkah+TO5?QE7uqadC+M-Uv47l~QX;W0v&CidpFm`G z&(vf*D?&c6A6EUx#J5MWOwOdF@p&ItOt58_lnS#Luq=j=m4paD(zIh?Ri+53WQ_=% zbOn-6bTzPLvVITawKCrvq!3Cq@;=rxj zjx!D~O(cf4Yo(`_$Gw2)8j6#S#J1?eOls{6WuP*@H8Z z#>?pGJMTt+L!z?W>V*Hncr+km2}*J-V_{=AEbMyhNyew^F9LWa|3;@p%%%xXu zAbi{h6BMB+yqPaX+q@L?j|%#IkzqjgYkDZ2t)Sl@9R;LV#Xk6@^cy(9W+~700GfKC zi|xVdUk`WZ8lMS}k2nFg5Wi5Ov5Mpf3eN zg>OmS!NyRKqKd#seTW{aelH*kw%eY!8#w<>I7wx(zRBu-4etIblY|k32!i`A_A(;* zTaO*NJc*A^fRmI}1McAxz;vPNu~{z*L(HBvU3^okW4f z#C5Wm6Ur5WxBEGyJo32%Xa9RaUj}YM9X;X(;!hKsh?MvJ>v~!LLe#}B9%d%RR%JrG+a4T;4rvf^?sZ6dtChXwk2UY2RH%KL=O{q*=T zrkzNwiQ7<~#Wu*k?&iL|w`R<9m~P3AL_1%tRyAb8A`*pHb}LT`P)|4;j0E`-j}MJ4 z-jA7(H~~$!#oj`nIGnFyOQ<%5S3O_OJX9MFMoQz)m9FjJ z(s@KL-`yPJFV@lfmAMItGsc zyZLZF_4WY>IK}dP;TdA5Yd*d;4$uL#36?4K7=XrHOEXy3i|Yb1@uF0|kd!tN8~}6_ znBxv%$4YC~>rFK$y=oeQ2X4xB-DxNWnXpVj3?OBhoVG*b?PFez;0XogD)>C!yc@V2 zoc~F6P@hq%zqO84^$Y}tnfBnLXTbDl4CUq!6}M1MTt|v+Q1ydOxB*hAT*tmbm$Q`Z z@rz|1I{C$qVHk9tZM?Gk{Qg)qy%Cw+)i)zmQNoAw07oH__;B8>>ZOcm`a#?Jf>P(U z-TZ(6;9Wt0*U4|k+Q)KZPq2c==ux$wlsAK8rffh79jIaX# zVvkzSk=T^*R7M|fVV0T3P|zW(Wnb^}`;E?%GO@VLAF5L`7HprM{PP<xeiuSDk}fP zyo5LUy^&@IvGC{4njo-H(cV=5bG}Y_8PMF)^LT$dBU#ppR-t(qV7XBJF)-YE#l!qC zUxMqo!QfSG-W)F#Y2|Zn;|!ej z6VnQ9e%}Gd#-i{0>Ct1l zLo^0ALcL~}TK3hxDs0dJi}}5W(u)HlT@}B0R9Xf5?E#c*W$^T)5II1J`!k@U3qGxs zC?h8238+GBtyT~SIi0)8wOc^L3pK{ndO|`$K2IFQ*RSI2gu*rh&87-StTWjCdJxd9 zN|Y=7fZ9s&j~T$D(E5pSQ++iJOa=0(T-8(Fg9e@!m(4ldDX?`~E6K(EB>C$&GGPP) zc3ZSj&AW)o!?`kffnd^cTnAY|QvoFwR@L*ez^qFB@~QwwcOgw1dL{UR*YRGK3Y2<)MjnGz|K5YIST>Oopdr2Sc9@|Wf%=$qb`T=gJLRRV-l)

zpjWQwLNZV>p0Bfzv3Koz+RASq%oVx=i9ptq_piP^yg?+JPl-ic#X#fDP<83qP^I0A+g^XiMD%Fpue%1Z=SB0 zNWHhv_92b+!bLbSgaH06yX{_OGZYA{fD`uSZI?|)fogvm96{T!CWF&byo zxQx=uR7KO7N+qn&o_@ppURC>cl zKTo$r(>xDd7N^mbMmKz*b<-clqaFw8mfhh-m)Uw2uRrEUIj8xU(^=Jt#pw8#^ntLt zz^ka0JeJ$wK4k4_FK+GT+Eh+GB#m0T?pfqgG`{xdIoYNTCCIM7@TCd$xv=7Pkms{s zcPIu6LiIxueCEocSwj=EX>f@$ZCAGglCzaI@qRCYAjB*A9m2VnX0#ts+w0B!n!C5oO5e&v(29GJaeV*n%j3B)w<_3&ZWMgpZ?ZL}hfu?Gp{aTI?8=uvx zH>>UbvEt2=%O+5oEK(@;znDdgK%)fmCm};xxynj+6k4T9663Lq0-Zj}>54iF{Oz$! z{CAqHqC=Q8a$i27U)Sk$aTt$goJG1dJ7wACVJ&sfOi3fa+}rD5-V2mAA< z*FUY!aMjy<3`WA8mh1?OHx0kJHD@7R#b+r|?d}QNVqCL;;^yX7*4<2!uQLbLMO?k@ zisDX>B>aHqbTIuL;JUQA1)H7sA*+-^iAiz@QEHQInyOljuIK=XrqH#p+*BjiC`Wp% zCNxi@F~TQhzZG0Jkj7H_t09u@6HaKKfk0rYFk)LQP_Cy$7?hv|t3Z?dEhR0c?-qC=50lahRA?F}c%dNrh49)>lFm*>5*7>jTA zP2Y&o(w6V~20j19nAuhN7KKU?;zM7 zE=o3`0ggwZE#qjx;|0*Q(MMLHE*4K6hagO?p37b!jfbeaMr=COj`+P$w(xzRit>>h zBsL|&zjx@K&e_aB;bQsYFCpD3u+s72>k>Eg(c>ykWrs`_s(9bt<1Gk;)YRrpi*2lVG%^`iyFh ziq*0li4Rd@y?J?^>Z(9KgVUxdB~vr%Ie))%`N18#*DgU?y=BC2GFx(2%5<_hK;cQ$ z14UWaYR46i#TmNU#ZD3RtZdO);L22<(RtcdSvIihI=xpiC8?@fG{QAQWSG}=F@qq_b5vOD8shoeXz9IExlVv7+mP=KLY9b2$GTVwtv>CV125UgGhXP&hu#C|B=j=XfxWbulRbJ6@eH37ps;rn4oP z2;xQZW!3iQx9aU`@W~enHD3MJkykj8A*jT9o$DJMzs?ypHK`owQcr5y2v7yyF?_n#Nrjq28I^+?xxNoPB zHCl~&VeESq=<{w(n{JY6bwNKbcY0Ify3e-mT_|a%i`OuBhLeIVBz@+re|D16f4%r7 z(J|4er?@C9xPp#`=Em{FAm$ zF3;Quq~P%5geMZ0PaSCI;q?YiU|$xZP%4~j77>rLz&(UsckOW6!he!Hr1;GGe0^`l z1`SC1!h3aU64e&gyoxXycQi_(&!?%8ZjJ$x0=Wur!61Q4qg6EQ zbA%$zaUW1jn};)**RQKZb26prqJ3hScCRI4N>66fd9}wjo74ASC2tkscwNuX(pYY9 zQ?aT90^@$y()3kMw3-svb7(GzY`6F1lN>fSgB<*VDK`~?5~PxAccCqOQnnIStz%{aDtj(bJNzv~S<&beNpB1XOY58*NGHw5#EgN|m~Op0yS32ikMq z0i-qwb;Ec15!a%NUUK7#(M_+JCbdsFkSCvZ@}lay4mN5 zh1s(^+p@(|h(nI$-xoBTE(>Q9X;1e$u8qAr*bxh-xJ-OjsZb-(z}Iy4;lc#N#-3mk z|MVx<<4+5eYDQJ298&t2t)nep)%qL}g(<&!OU(YY2Vp^HJfA8Y8w^?Doi<>;*c}V& zi#ovPK{1xXwNcj$s%Fu|0vjIKzu|H}V(0bw=`V%*;fwcHvRaKje6Dbm`IRSFr353m z@%CeJd~Y4BXjEn>BKFe<8Wpj2m?m^0PJ1ag?XXWQ;}(oSrW*d8G9t(Ivjn@_k?##+ zrXz`-n2|)1oUR@sh&}=^C1tG~On58EQYEgjY<8ffdJU|^FMNqNhsV~Z`ax2x)JtOT zq0q^ye_5<@h;gR@%I9QKcr^(g zSf_ETC#=lNl-ubSo%M2SP~h0u(9*$B-Jyimh;Ae1Vy$_3bhdAf!ESR($b7-aW1Me5 z(#Nx@N3wI$b`Ny7LmA7Ye+l$hxi2ZUu=cst8xph*^;jc8AN)z<3_Mk#uLr^4))c*I zTix>ZCTeI&TCv`0EeJq+2B&~4WT#fldFAeSj6=;>AOcZiaCsG;2nl7(5>(0^jWeJs zny9wR<2myvAixB_FzxyPU5&Eu)nfNS)yd~*CNE+rESCnVo8RAD7o+19x2#k4^$t!X z9wkCB%1MQb6M12zavgU7%@l+4c3PHKr3fr!O%<8nHmo6gv-gwVQ8%-2~3td7cIh>d3O$c6#vru|D_XgP7ITD2ki_He?|o?WHW z);I%Zf{8V|Ql(MVW)ENCM1i0xCcTCjeBclIhmbeV8A`126B)kn4UXe%f~?6EROzfX zz(E_BQ%ngSjinVv*fAnS-2+Nz{1qpe%B&_0?|ERfUsFJY;%|nqmx84=Nk!|A{2+(3 z-#`F2IP->s&I|50&IiN88sVSE$E6U8r)6-dqP)`(pIxZAeR2Jw1RcCh+LxM%02(nl z%Oc5Q$_ZPjO>MIR>MpbY&I(&&-H_p29g`SBy;)E>xI8!Z2STqO9MP%AaZXupGN}oQuaNrI`GH~ z4)8WilxuoeNI>y@$q9=p_eA~aJWMtSvCcT|^#?<3~>0(r=|#o_i$2*3#~Ltg;gwzi&i8sB-hBsJF7n8u@i>&M-x{Xvu~MhJF8Y| zRQP6#mYL+p5I!z=SM!R?PIpe&^Ok>a@1B>em&rGcuUNOe9Gx zxd=l0-MRMaCV}B&xqip5#&yzS_dN@;pIQ0oC_f3}k72ZS;7|z4`oxL(o}-0hkZSA5 zHq14@eo8Ln5V`s_qHzA7qE4oDOBJk5B$UqG}J@j<%x+gHmSxK#Gl*B_PM@%rVu-=vXR_^_T(9>yp4 z#yB)P#^a)sKK7y<&JqxawITgz<7`lw=+6t_ap{C=7l%`KIuap^8)+9MP{ssGa9ZMM zdGpxoWQ~bQx8c{UYWPdL(Fm&NOySB4S9Y_T|%y zD=Bf@D3FrKCgc%i3j(O@quIihq+IzTfzmc|XICG+>!jT&z#59vr!(sUT!YeZ<{OCr zv{Tm`fn~!w=|yJMxxsO9DQSdW2t#)aRTQzDvTo0~RKUgO{EhHp&o(t-?^868tXspP z8|l}=yD*qO-V;So@H+3%kZ(|?$)7Wsap8izQR52bhK$bobn@L^Hff!-saS3O}utd>wJfkMyb_J%=oA{)^YY2U)bXo>yy{75G^Kb?kLVxJbI-8ajEE6^LZlo z37=@}WKpr<2UY1H5efz$Cn-Wvur;aa2^=(=t7A@|=zr{?kcq&)>P%P3E&t5VZ$Vqy zgR#oEjFah4Ht~jWcCs=VTCJb-#FKsV-AB)gS2m1Y;AE;38H+}Xh|*$t|0Yh!N%*B} z9+Zj6ygI%&h9uwaJAH;=07_rN!=o*&nrh5vai-D;Ou0wcC>rqRlsE~C+4->E?KU1Y^gGr3*tBz?|tmx^wg$aTO? z1mSQOj;O}>$rp1y=J}xezL=Dv0-~#u{TOkV^Y|8 z6a!sHaVHxtr7uD2?pt~QzGSnL{`Qpjf#E&&IMo5vIa4*ajUb1Mk8qboYI z7YN8F%B4)&vx)omHzX#1tcVvFZO31)U*!#6b9lUclTR#40I?Z4(K3xo-qGm5Q{2(P zoWWt$ZQ${}+9~Xod?^kId&|3E((($uYW|Z{2Pl)`>hlQyMfv{R$9$Dtf^yR~_{)R& z!fMP@t;O2+5O2SW^gbnD7{dyGMJ=uM%w5+ApyapVAHnGT~=$d`Kd24Nt<; z|Nhy8UKmSOnk`fCGC}@7PEE$Rc+2wOg>IOl`7IWad+Ta z`%<9T^xhH8q)s|8J$>D>Up?PDq1x_yWzq+7Q0B#&abfcl_bL|Mpt_Z_ayp7tQ5$(A zorS{bNZ3Y4yVsOf>U({{`@5Igg6ZjlhMrl4Y6lLirp1ZVrG4|Hpo*5y?bS)jd^U0B zZ|ze}#cuIU#K>1_0|C;v>FXe`efQhM)`-zKAV}+I#=D_g;MHlGE^ltWUYeg8$iCrD z6aHv&7r*E2bC^^jSCT6>Hu#fX$&NtD*N?kj()*#L$~w#cd-|tSHD>8BB4AU|^(vlM zRTH=?z3%*TTS1`C|E(yVTAaYs1i^3G-lp^5O-*|f*pt)N_wRi5%&j_2zFdX?iG*6e2~l++*3?Z9d3Iog|7iJ)3Yi}#Vl z2i=MS6LDE5dVl_JT}nIAVYPNc=a2Y!T^5lp%3i%fGqPYCRQZ-xQ(m>bDWKKi5hYq2iJRM!@0eDOD^IP(S|-Q}nzB5+Ktg*~0S(Zp-?(^Yy;BPkr#WTQ^@ zE+C$WDa6QFCJm2Jpiy5%Upep3HlhLqQir{jQDF*fe(bF~pqdpCL3fInfxG|B>7z!6 z=U)pLE;liB8(xYV-F@1k|5DCJIASb3}?~C%<5+><#d)*ExkJ7@o-MA|y_UZdmJ&a?3!& z-+QQ-KIv&w8|~Z%EC0BCknmXrse-}xZ?+ePOlq&MuMNWADNA zdD+D{UQO&)mFOBMH!|DeRq{zKOgYNDPC7ZuAzG|OuR;qS@%Bc5}mTpza1fi9m$~ptj$lpB-6+1D} zRD%rH*K#S@L=0l`QtnE2(UsM!yCTK`Q!}aUepeQ=Jh+Y0MWw$l7i!oUrA&`KzpeEu z^H{7|9bh&`QXC#4j#p(pxc9-hx18Kl_qW(VVOe(3pLebfeqc5_AP1IC9@qjFc3Oq> z5L&{j-R$^Nx)Me^PW|#oUsV1LV!AP!J_OF&hJug)(|Q)rIhMh{J9~vQq{b5S?VYva`T}TD2@DvISSHyrZPW&G3Z=C_}ZXC z=#>ewaa=c0u0h?fwAH2YLDs^aqW*-WCpzq*iulf}!nHPa-IRWV;YPZs3W>B*wQu3x zp51`jBYXaVSBLGsc<%t!8&FM(FItk)ix&V(^VRc{e02Q{$7S4~G`@=S#qs{{(|&y| z8~bCB-COm%V_6y6Og?p=UY|_QEqWTRK@dJD++8%ArH@Zk4+5U=xdjOpqD^`w`jj-h zFh<4c#GEQ-B4GJ6ACnj}aH%Euzu9bMTLR{RW`EFU9u%=ox&6d+GMSk`A4P%_X@Xx$ zx;{(vGgxj6Bx5zM5PT2Kx_}+BcO#`NnFi9Y;W}fCny^V)N-|x~5J{sSxvIBh`n}c_Gi-=!jIt zK2j;8&#m+yW(`N9gR(cWK`qr!7(bI8D-YSJKY4C$K7cA}Lw8b0$oP;D_T;Ozp|pzY zKkK3&Z3dyIbBdpFL3zd+F?>L!WKa>;6R>>ing&9{AD+H`xEHFrl9Q#9hrdh9CR($n zgNE|drOx8b-)L5%aef(SdJ1E(xq%fR?ZE2B`LotO>$E%ZH&XIRr$Ge{nK-o8)X86A zKAneFv%>n3@yQS#?uX#eh>GGW`e<3JC=RHc?Bk~Bjsw8F!>FDXMx(%obf0EAvek1K ze#G*-q^MI+^HP4&mhwf8@4Q=vJGZUn>yo7)j1M%83t*}>6RDpmqh`nn&hdju|P110+SifYa< zmftv>lWZkgf}@KS&y(Kh0dwmI5iM8_`j!eTQ5+*g9#9XDU(LB5fI^(JwY$>%FQbX+ zx^4VKK~KfA!e90XGV6xD%v?Nr;I19RS?Ry=4LYh=p^N(8Xq>Q&;sV=#d`a*FNwRsD zi%oa7;Ug z8W+&Q7bctf3m{-6=)EgQnMTVxIFPODzH}m)zz1|wHj%8w1ZcQwCZ}vD5QE-naqt=X zczyRXY*LG_e`x%+;i=`m_nkZ(GSL-;g)Pe*wWDjLF>4iLMia@&N@=j2MwJs`Djc=5 zwliQvpL4a7vP~!xpedr!(aqiGdJB5DDz~+7BrlpQ`%qISx6Qrt4y=Edb8{qmRI8ul zvh(8;lms%7`RaDMA1Zfys$M^DP^rK-xoe6aL%h3;Y;O-2Lu{`o8t@BRM;~zGabT|; z)EnKfdGNcjPJi`xKzThfhwcv2CjFd#rE5}r!!#BNq!Qmf=g6SaHK$|pI{^vgrKEcj z2ZwHpg1kH5OMHdbb%paNwDoyhmLe&Lc#$|VldF2WiM9)|C#<+KQ8l#Nj+P6a*`iBm zXmx9+3guFc&P`u5E=dP`jNm?&jm(@P@5<2by)-LVZ-#e>3`E4b0mvpov%BTF*;kPt zX+h+~(L16Q`OB2hda7h~mKGf&yE9LJsVDdvrjqEhaXT?4&VrJsyI3-+6D;|xCX>2E1f2Fna@$wnLJRV2|D6Lua-5yt*ZdsaO+C&TJ}`ewsGQZI0{ z2Z=ID@<|Nw4K3-TJq58@5(v`sO{PiFb*I;4Rt%k@pDdf zsr4+0t-`dAoI2ieE6^Z(c(9>ei*^RZY_YAXk=eE#cP;s697N%n2+%r_FHW=;YAz~q z*wj%MYStkW5kchGtoRy7_vU{0xdODCPVi)p7XmrWk|dCJj-I$8=f9cLeO;=Q=uZy+ z?sU$JYFiCd&-2cI^V7VJ{@yppQB4M67ouBiA=Apv+=Y|@M`(_q5`@kJ5BrDYGy1Mv z;z#eBO=_yZD4FKD$Nc|40DIcJF60G*Eprz%obQTyVf24%0*+823``S$M`1pA5=eII z)vVY5<1?_ozX#>Nn;@X;f+GJU|9@%>rsE(Pd{7{u;8SRv z#s3=8|9)E^sP#WX{onr&#@Lk=fkRjLxpyt^|GDkIL-x-k`QP3&N5J_j_JCIzm>DHa_APD;JT}-(1sGwCUq~3)= z`;A}`>$a~XUGqCfVX4WCD|q@V!MG@wwZ7z{YXods%Xs5b^n=9!1@mLYFBAwZ&yeq} zDVb{uRAhKmC$7>ZfWAYOZd`w{$ZpF5-l)H#T^K`p%nD}oW4nHj`gcKygyU_jU^Ie7 zlThc$nWy@}Go2y6bM zQLCm(*9C0-+y)GDS|N;r?-vI3V%YvNXaXPOhHqa_K5I(k^tJ}0-r|1bzK{`wNeI3a zE}EXzhm@Tfv@8c7L##Ic#YvFRRT{y!pNYkZJS!U>4eS%}Pu^t>AQ>ltDN#LJvB=E4 zvWRrfg#p&mVW7>gcR_skK${}kS+mozGGKp*{V}ts#k`^gTWs$SH~M(s(`kV=f!35? z3JHav3nRm%(dK^>H|z?Y8E`cj#kN7HmIC6=?8v(y6S^`X_>zL1rA4d8!zd%jk`UIs z7i9KRtP-sG`LjhA=9LXt^QSmgXYYbA3I5RyvS?lwFwUSaZi7-Iw)KyNVfwqq?i#U0 z7zM9l*dB;9Wudol{iAy0E9#ZEynlz<*qy&^j+$8#HvV_81ycqyHQHVj6eCE+ng8g9 zD}cgv9k%3P(xiTO!Xk|PyEZYv+!SU4U&{WtFl)!WGV!l&b{B%cq@s+5NmCGOW0~=v z6}2)96Mbvha^YQ&#aJ&B}oLYp^ts=;`Sb*|eD>K1Rj4slZmO zO{W!6MPXr~F`^8r@XEw8D6vXvWd zg?>6kxh#DuP51eut_iH>-W1Qug>md zP1fiv@MxLra#JJ1nX|p5^7(rVJA`zI9v7u>?U;o#)7K6P0St?=?(sTp)JdO`c$tO0 z4}HKR`5^A-*LJwv9s`bl>!9d@qN#C(F6ZytUs?;0NRQsh_u9OEm&PZ%C;8IStm6KM@HTUPq)=k zdDzPH=d_IBLx8Qk;`BJ&VW;J}@r#4z^~Mi2RpfS)wMjC~Ui0U>vkh*1x92x-N1uO; z6o3?mGOk<+>^|?GficxfYiC||>n1(Ie{I9TcYx2#{TI7MuIJ^txMl7Gm2g31Fhq9Wz^f(Tf>jUuyLdcLXb5JqMVs83H%pZz+@<_t4+ zg4sEBB)3LKlZu2)f-Z}2^M5Q<&YP(o;ln4W81!t0^hR2V66en zHb+}^d3Gr$XZlr{Ha>f0p=H*2W$GPEc9JHCnJeZK3ld0z#FRmvYp8TMhM_+V3CWxE zq$H^6*6uF)eBkByPnzE7+k%(76C^$<1Q0uF&(&_^uyAqt?}dRN)jD<8@a%?Wq<(l? zE_K&_TpR8~#(E+)|FoOC1kAQ?J3cbX#s?iaygzT_;O7L(uby0wd2S&dc31Ivh&jNVS|N(EVB7wal#%!xeDT z3E5+-T35HvK4+!Z+0^AfACASlzviyr(FEdQA}ck7Ta9KLl!JKTVEh>=m-+0tMQJnT zd3a3?cL&7+yE}fjc3KO83Xj zZ{d-SDj;&-yQBHD;1bfHK*9O53l!|=xqkK@2nrI| zxfE2kHs=4`1>)|y<`;iuz3BX!Iprv6S_2Zc2@9E`atH%+?RZDW&noSIFkk<)AyG-p}{#3aEVhXbwo+WNcSP#Yr+tUrU>x&udQW46{Dvdh& zzWZ|qrMDcxpayS|K|M(~^Ql8g>swxv4u84fM{M7-WB1QaPTb<+znNFTJRTm7MPnAw z_wbaLsRDd)xGG2c%deWVHc2o-vnWuocDSSU+ia@vXjn&b#kDzZ@6LB2fn{AD;=Z3F zz)Z~Q!SD@Aj?~PT3|wsY8{-eo<8ebUZ1&YQzPh6^0yQT{#^1Z-e1bv)SGwTo9>d5^ zpok5AF8Z^&kiFt?xU?(=d7M;s9RffQmC=GCzrQ)>24$SOaWZ$l%jC&out)3_jIrr_ zFh+NrsU&&&^OGm$!GbqYE&*fmc<#lwhd&Tw{`a4)zXeKN>`ig@VBBUq4X3a)zvpsi z_*rKOOE?Zj#w4K?7M7ird|~rG&@db8;Z*r#D_DCT4>(@V!lIU(pdQ=0uxt#Pw~+Vw z-V$MN3~_&19uASIm1>rN3`x!KY@G)x0s=y~M%(&PXV{e-5HpU3QEOa(e{(lVKH)cV z_l&7y>|FgiX3pq9y0cpLzlPV1)PN8YPa_(Ccyxgmx71+IF!Yuau_#E&QsZoEq(?cv zv+0GrokF?XFnW=`)Yy|xW(pY^sdl{7nuVm&06HW-O9SJ(Zkw9k$B?%#k)N*H_fyDu zF$e(}Ba+j3q3ND2EHdEkGExM)MDu4{-;hp4&o<-G#P+`UUiMWL@ESyn`gPl${N0iw zQZP|)w2w5Px|vBo1+@K%8;LOo>K6aC-|IgMWZIQ>T5O{>I~-)i4K|PU z5K-N1+2;m41*J;bqc@(8dZEj%F3Sy%%y4GtZ-JpykOerkj&_;q*L?#ISqiZ9>B2&T z^Kg1DJG~=mGcZXLk7Ia>m{goBx4`-xdG-*0INoR{;OXKY&<-0s##5N&P5FB ze5XpvN|pJJhK;_2;F1o)5LQkV1;*04%k$k7u{rO_tUfibUD)1l!LqVz94k;4vZbe* zPhJwO4+Ao9?|bSkD6)_A_^iwoc8=s@--fLK z2g;PSXkw5lM6jtzKZXUm+4N^(_F5_rw^wl5>tF$0UF+;5cF(6`$*3E#+2ZRnsGmz` z!tGl9*i&WO3F81IG;p}k(n2-+hNKk(_T%?*f^pvjBzWwljR6}pW?poz2y!=V<71l6*lTIExiYm6E~n!cbQBfr}c04Ar&Y&e5=iT*_bb-U~Nj%I`X z0?N|_8q%?P=yc^=hM@D4)2)%vr8aMaSPx7RjvjCYM$!_C=pPYx{Z!?Jc{}5xBHfuv z({PZcw)!jSCt9%O_GqW-f+{7FXqE*7>+Z#z$PB!B1;XL?W>Jl(HpZBNw{Dbg zIi})|e@)f1ym>fM)+X_&3lV!9%N3luda^D#SuvxrVbbovL@nVPCKZ~jyJ-Jn^VGUC zvj-!y9H8%l%M^XZJ<($lG+}`>0gQVk*N?!Z7KBgLvm~%bvIK~qW-YXPY~xrB?uS3| z{ACdSdbL{w2?Zq+s+(Ab5=GJXyg-_%F3)4F2YoG$jS1j!P}K`8a7=yzEvu4D(3dir zCgC=)`~J*O;rw7h{daqcEs2l+_q8u%ej#gy=l9278O0$D&FQwPn-g#Oit_zdx=gQc zw}5+Yf5h?Y&9A@fC8vaMqgL=LQ`{D(RP7z}pDkQ!v7p@_rIt6*#EOCk>*}`_<6Xhb zV;JNfcKf{QL@gduLsO;B3~u6(zF2C=-g}ca!N42qsNP0EoR@w0lS#@k@_ImU9z}XV zinYqDQ|(oliSTPvr>hd@&g|?g*rBM^XmGlX{jYhmtrF)M$?^e#p;c--tV~5{&n@;Z zm0+F`()(xLvg;T8?;ch{aeHe-y?$fAevpT@p|*-1&DDQr5s*3iNvaIH(Z60G zS0aF&1Z%n{x@DaL0IG9myJqQs&)d5{^)72^-?x&ED1P#Ot?tCjz#R=}LS3w;8dz1T zSwe;x=GlDT{iAYUJR8GU_tH5VHfd`Q1$C8TI4L%;$@MBT_`JDFRdU%w;u+BABQcl-!th>^d7AXVB80p2@ui*-y20-=*uqLuBKx ze!ezCiY@xsRHV&6z*is_vvqyaPa{VeJwC2+@+LX{fI`?VzZ_E48y$j-X+93F z%X7T@*}V*sH0Z#YqI-)b@ERTXRDD6gh&^mHP0_?7S)FtUI+|IXcb61MNGOGjCmWXS z6Z_oJz0v$K2QP`4!1xSYH3W=^u}ef@A9)?o_2_}4_o1$WN9Y9%lTbLTR!qZKA8ju; zlabC_y*{x7$FTZHDJdffK_`8drjCi}SwZJzf+j2qK_n24B1CyJw9osGOb>4~nGsV-3c)kLpeHk!k;R`C!enJfPw z60LrVi??9EE6ey~b0C9+(-g}gf%j2!STTk$7r!bDIhvSct&T$|H87-GFG`FU*L73pi*OQ|jt zp>Rw$XRN`4RU6_?O<&8z>Juj3-`HA)&=NpoHa+Mgn6)2cqU*q^1Tb((u@ft~YYYkfTi z_%bm*`k>xlt;^DrlZk#--Epzlj-s2i53JqVp#(Yp&(4HzB?^8>#;!lwEa8olC7hpr z7A3|d=xoRxl0re)ZdI7B&hK4h6}V@_p1)52BM~frHf+_k4;eu&0G-%*wmJID%ry5;KYQ7EB0jQ$B!T1UcO{QZlo0YV_VQoLF4+ehmGxKE9z2m zBJ}$!f7Sqb`Iq147KtIcLNOQ%q3?o8i+K z;?Y+^#Z@DJVXR{3&SG6>^pW-fNTJ3TX}kH)Y|Dp>2HB-HxvVIoi>+t1uHT_2^x*{U zl7;JW{jraj)aZ+>h#))RLti5jWL?$PTVxl^!2dId@vt&6>W+&GzWC$MVcdd7_l1s` z&@|veC-xB^`9=V_5H{6gBBQ_unMl?)uLGjxrtIuInDM+Z8sc=7gCT#=n$lX;tz6=U zsnEhLvyPYe)VaB=+MgPZH56FCBVOOeQK0$$M)En^4A09m!{Dqt+Ppek>ThrJmgD~e zYC_F34N!5qC=37>Bq1PfbE1*}7(jOV(L@yl#!t(g-nfBs_re%~k004ge*23M4wQQ< zIJFI0{sQU4k+1OKI}7DFK2w6(jj^u^-vJI#QE(4%L?;mFj}}?Cj-3+K>$7cE<6_ss z8Lf`KI5W~^(ohMC$b`F#bv1Wlvzy(Z3&41_?2}EE(9#~yP z7ibG#0W$t~y874XGnk)O(vGP{t0v7ysgV|FKePKbuq@uTvi>V;UdR2$UKYSPsn51t z1ys72fr4hkAeL6(N=WGyad;RdIumMyvh)IzY{(lpY$L3t0@`*oM9ECt1 zfZ+_jXEE>TWb)AaGLv;2!ZzMXQyCL9L}_~MNKf(B}e0M{(r z7Zl7yZ{hz}%3HyeK3@0;dVrU^vlh*fG2;PP7Ndh0hWaZ3@r13L>%OaWm7xEFd`&dN z3&o=g6Tw^>nMWf4uO)$8Q^R2V{|d;=F#ZGKI$l0)jz$-zhdsWfWl@-UMFR#Q;#l)jeElax1fn9@@4x~bUG#GjC3IE&7f7~w@TkZLVt}FkB83M| zAlmXDIsUKo@t-dKR}9QJ<==zf+`}R}^GYO4j(9uKg`WQ3E@ZO++=pU~C*Upda5 zyW%WC_ypEPjc4^B^zxrB{$HuO|DrMyBbdlN*Dn$Py?XStgBp)XLsQvaL>r}XLFKGy z@d7Ed&SQuc#*jheW`!1s5s_9=Bfz0-mKFpX+M=;W#~>8rf`oOGPjrXQI?mS1TpXi+VnO32uC8>y-W^gc>QF0=;X60|P@DCi#|G zBjz>o)KwZ0MX|TBlA9cOTLBqNg<^!~T*)99Bnz9^%_yMB_-A0w%KyS|iU3{xXF`Sm zh59pqo+Wz0d~VTcV_4Yi7Q9lBK)4M>P@z2lzmvQwTbchWtYaJtUgMfBSfpfLL4f%@ z!D<`KJn*D2YZ2qgD#X$XYm7^z0|cYs7r;!j5asI#0kTXo0g!0#baMj4^lEcJBTank z5M$V=a?MpOvNEsa!&)1Ux2VsML;+zWIg~?xmnVd-8nCHt*I#R3uL_HnGml>I2FI$v zA{UEyi(bqt8L-!e11%~*^cD8nv%>(Fwnwl&dVxjvzxn`Fb`EfHX&RQ%UJ#s`2fyhL z^gUE@K(iPThtz`c=q+S0kGN8%RYau=8T5S>A!k6<97LbM6QsX1T-4k_f0B&d{i%SW znqcuBFB^Yk@cTV1kUzQktptQx%|$^YPPTn||HP8&jIeKl_El*~cS{=8bP8m{;l`>F zaP{EvZ6~UyHBpDs@FlM`7xn~9{oHu)e5RWm*_!382R2!xecX{+#0ie?@ZFOR zHAZoeLWJf`sm+qyVMCm4@r;|!12KQ6EK%H_kg;Gf6BVhdF)mqNL}5w@&O)ys*U|Cb z(CEyFJISM%1;{j!(%~^h#mCJ%Qbms#j7kwy_G^CuCoeuR<}%dXR^<3qRKOOu80yCt zG$mg4-~)mdTt5|IT41EIPI>1d7bIpvmrg_Y@NvhLmGlql_1LtYRfklf5>}Y+raY}{ zaEIftht4dV$ZLo!jXmb>oLQ3zjYpMxEiYc+EUZg}y zz6id1I`?0FLgAJg`9p%0W()W3l#xR>n+8QJ+kuctz>$w^?>CMtisYQ$z+Pk1RDOg9gsJ zxpEQbYWyaO8G%BO`1IDUVbkZg#K$FRzB=ifE8CSrD_U(f+L^3GWgG;?N8cYB!ikzh zo1B^p;jC!)-dG2OcLW=IA*F_qz(tu@0%p-R(&8yxk{-NlucLbVB-%tiXE zYx=Qvar*bX!xuu=Gi85DWnKy{S_9ZBBx%C(NjgCu{vMy0OZ6tRN&;&t9Zm@;wf%-8 zX`mx$!&;=qmAhz!rI4ntpQSHzZT?kS8R>>niV9S&@QhRjUUC%SbMcL~Q!@KK4%-Vp z*1KPD_+<^;1X+lR%RzqlE1Qki9t@UA{OWwyaNISv11{?6&+7G)xMo|m`eF89{X%hw z^D!x;`Zv}UGkv+T42;*nxB?XDn&HVrydqS<1bQmdd z##_8HlIWNRZz>Ud`kQ$}+5}!Pr#>Oo-)$H+!>gteNLS*zK`;`S#D~M}!Vi)$NalzE&l+Ll7l&@=f#5dtWQB zt-syJZ2f~=uC6X6zJM_xis$}rRl|iV#CSuzi!bl^Q>4TXvskKHfu{z#=42aMq43@` zE4_%Ue-`Mr9kEZI3BG#qx%U}+(fZm5utHz1LQN6ShwV2q<$HyT_PJ0}f5W>kZ)vy; z2OIBUcC8>=%QJU1LyFytXsx+0$}uK`Of(gSq66aw!9Bc;AS>!GI1wW$u30&;u#*k4)Z>M& zv_Fzxq@v6Sw-1UmT@_p@&<+MG#i)jsF1pSf6EQte{n=$imT=LGod_WLpTkb=M6Irz zB&*-7tyWO_6Kmv3eel3bmRDmHvWY?uGX7k{%t2X+{Oa1k2~ld$ATcHoZd2B~uFS;6 zRo;WCEL^6GdcY?$@@E=f9s>Ls?Tj$m^)wPS%^Lz=eD@9^__5y_U;l*qwp2ovMc)5p z2Jyc-!31s2jl>~?EX@F(8;tf9Zq zU%kG$f0Q(RcZ_J_$F~vBPqoX(TqMP{mfe?6T_KF2EU6SJ7Oysi%2%)Qmkp7u*h+he z{QTjDc5O7;jlOC)RvjNsZatBZtiGMU#$(XcA2yS|bo8L9kcG9WOfH+o}vk3*L+ozg5FE|{?wgMbcK2QiiQM_N?8k$N*Ts{|`< zAR^aGk{L|(N~e;7{UJ<+!=_E{PBQmD)650WiS%8NH$tcYB`-`g3%FD}$lJAa1sTNL zq4)u}G9$l+SZ$2Skr(fz9JS}nXi0r3hSJ<5uV2r<(%L|zl~-lG`vA|>iNB0J)LPJ0 zWIr#kF{kA54#X-FekVfiu{Ni*RvgTdOfFn7+1BFizsZLzNDiCLigjXvD^pa*#V3ALX^y zbyw!YdQt0QF7dYr?5EC9E-Hb7M_`F+V-g~6hJuX^3aaO+}cx&cJxZRyGQNND@yjLa_J5wTFqv3L~_ouoFKAFJHL-Pj37URb;M=e4HUVINzN{{A51bN!+yVjGr^b^1wfn%MvlgqBs| zZUuAr-H#Y&1c65u5J)%?6QLn~Dmk>q(qzbbB?L#>W>0wh2ABT@%4xCd*&E|P{*^-t zi0;N7cIzIchREIzzOKe2+L^g`=K|=F#Ap&MP0nQ6=Z)xz_Yy>RO=*~Bm@^8klMm2K z8AD%PT6V0VC7mE6-2r4##39TB_)4+R-+OfUT}nvdzK@UwYe*TrU$91y!@t>|7KacJ zLaGCi*0azt_M&PX6YZXs8IgeL>}NLN2|Up-YO^e-y6z$U^dV??X6s|D#u>y97y_=` z#VR*F`tnhhy24$~BhYZfmmw(`iCA^gl+DH-#FL~~1!jVR5~yr&eL!y{icn zHFUTKL)%`HPIAYS^-32Wx2#7ux%uhF>CYJC&Pb&DjdCXislAzr4ZZ}-R8Jk?+MexB;1YkaTp*;;b<#Bhb331RPmI1$4;^!72*UJ{1aPC1r7 zQVj8<+8^3AgSX-kI7Wf#5>VGBUUu|^dP7SlcsMY#U-uY&K<>`Y(s|rPVwC^Fh42?s zy}m^b{`NQ;qf8~<%w`ecn`d@q@ZT3V=df9~QKfw^n{~N*QDa2JWrrK7f1|XTmxc94 z=QnE9x5Kw~Th+I!^AJ6`KUig~`mCZut7lZ5g8{|DqU>)=g=#2lmSN&@^L7@(<$2G* zJ>W#2T+af+@2<~J;P&R8x4NLx1Gg8^^Vz3&Yr04dr38uH>PARK^|cWshc5+N$k(X~ zw#mh@lLHETb;5~GXz)DwP9TqULiTbgCeF0c7j|?=KkUDINpU`FnaiThL@V#H@`xY_ir zyiYlGI9+>-gLsXA9|$JGXk>vLt&C+bR;RW%4Mkoi(uN-E?Q4T_+-524b}Y8n6bc5) z4lHrkV`?KGOq!j*@wxd$t92CNz!a+c)H_L8xJgk+zVFtLriO5n2=`+4AlB0KT-ez} z{nl6roSf0YH$sjunQ6gP{33)vs1OTBrDNzqkXRr8cHO$xt8zEzMTWtD&fPingtS6O z1bLAelJw#tD4P?cbzhFM+XF_OU>Na#Kq`P3+H|^CIKiYI8MqF?k`HR?YS~1gujzFO z9dyOppEM_d>?0+|xG5ntQ1!Gopy9-pQ>e`$rSoU@Vx;fux`2b21I!9}Ue%pk!cH>x z`tFL=NMg8A3AOAPfQmGL-*WSHTbY!Zz*ELq5_h5`bmaxFwdhBB_bMZS(?-%#Oy^xGEyV zbLvuZZ6=QcRx9A@CI%D`QD747q(YIB8NV!zz}-H5U;Kd`kRS`n9io5r6gellsE41g zJs_uVkNm_y=QmS&a7dXDUhW z3;;j?P&Uco84P_0-MdqY=A01aDruqT3PLYNfbS=U6n>>M`D4b;|1QP~pfgKo11=h8 z4XEzZ%5~_w^twn?p+wiDc#ksv2jmK3l;KZVpGcq6Ip%${*b-eGJb+_YYE<@{%=c z5Xm?QhEOAC4yn3I!3N8`%smC*PqZ-nX~)`yfzXxfAB<91uk&Lg4EhDlXSc>Vwf%x& z{GNAjjrsr;76yJTwZo9t)PD(1LxdyXXO z-}i{MPB-qqZ??@^Jhr*OkZTkxSTxuzQwH1xq5)FhdxL;C9Cztv=@ygayyaRYkEbh5 z(S?MBmM8%^Z@fc><+~o>Q9b)6WVq8Yz5K|avQ|iwD*?=7ei#KT(|k(_Oqkk#>63zz zo0VX?7e@kJvB{+|x}AU$k|(_@1!y=pYCL-Jlp?jRU&H_#f=Nf^+IK?;&5z{GsL7Mn zsZXBTCGi_XzAIf)m1dpr%gf7T0(%P^L`fS0Ic+b$0yd3YZjNXZtgP#&U&60;@IHAT zv7I&pLfVf~wdsJWcD^Uo`UM?G2wt;o?Z(tVBfE-{$?!E$$>|BUCVG>e=yLOw!2+U% z(Z9g1B+`b8$ ztShs1>pPcf-m>!Y41SmUJJScw&npaxQ9|KCiLfspHz7UV4P-8Jr*V@!@KcVH(i-F1 zu+n!&9SfW~dkSg=oxe{Z9rrH$?wdp5mh!kPWs!_Eb9I(1eX;k{AVA!G?e=u8;owbE z!Z{5O9w-^y)#CQF&?b)Dt6BX;a6Cj;HbUyjN$Ikes{~m2*}*{@=#488lEG?hO@g3< zwt93X`zg?cf1u*XAiqFXWIvgq6n7y#7vG+BkMZILZ~{?T4{peN${+-SU%} z1qe{)Jn@nQcO1b8t1*wcOg_aLHNOCtwN;jDwLS*7(>a`o`-E%w?RXfxbvxhT9oLO= znEF7cp1VkY1S&ea0{-4B>r16ttTTkrkdRv63&U872R`?$;i%5$7ldXh`7(n$;cM&b z4D$i|*XY-`*ZGExA4;Ks)C^;>d_>Zz{Kk4L6{y$oSvdzpfiEn*{gQ(b-@< zxRdGD7{mbPy1Kub^q1?Cr>j1*9u$VY~F{9`Yq;MfUcZso)tsPowl;==XHb! zRa(-ch&(D&c^6jG`R5rL2_uMI@Ti7c0Jy}#RO(4yUXlWvZsLv zzoUu1W{)u6=!Xfa{PNEUtZuuJ z3HVmxJz=%}_EvS)fhcFEex%d+i_lMh7FjB#N!z>tK|Deh2O}lHN;5~g4n{8z5RJGP zd&39prMVDnd&FiU?E(4^0M3U6O}sG%uoPa5D;v;WP~^YPY0jNPzXc7%8~Y1F?_PeX zldB{<-lhG0(yvR0#czN5VqrF5^@e}Ys*q3qU8;18r&H9|1oveQB^%*zb6je zaR8-}^w3HuT~gB0r63_4(sk%AMOsn>B@c}>(v2Y9-6frO_`UbN>;B_%9hZC0o;`c! znP)!F_nW3Y>9W7eM!!y&4}=49bVt8q`kdiDkd3{$wX~E1MIW zw?5v=J1~R!xt#}@1>|>)>O($^nlGL&+=~yS@*!LY z6bzAUiC}tFb~w{0$L}%;+Q|qM{BElZ-5rL>P)!xs%{xMUuKkx^#|sdY(AQ$0 zW{YY6sae>u>@&3g)dVMUq#{5V(^A5?NQVaijYNPPAo;7B0kBA%rT2eU*^)(lzSrI@ z2UnONGE~6@gX=I*!?F63ZlCrN;NXzx^U5LV)Z2;7TaL_*DO+Jwg%V>)F%kkoFHWgY zn}dADyj$^`=1xs}Vtlp~#>(f|DkqR4d0>AvVM&o3>}mY>Awyyl)eN8H-ZXmhecT;RX-jl2UE} z{cM}Mh-J*bv`w@RbX4l@lhz!qy{q&IJ0lt@`Eu3Qtd=c^ z73H31OV6lJR~@kWikA7QH0Tg^Mx2B}9c`Z()$a{8mwEiNcRyUr4y#Zre=pB&colh( z+VG;g@CnfIan6r1xsKp**`I}Wsie1f%`J3njx2=70veuKnIaaSdOFg(z3gMkv7~eg zP(x4Uvwm8Vw-Bu_7j=4beb8)q^a^$ai-i8VV-$QiMhqu%C)x_fsi{~wY}P$INr__Z zz%ODHD4752&)%^DHIFmnSxn^6c+6-0hQ!KNoy`!%#;G4$Xcl4sA(0dHFR&Dt#Xam4 z(=K0f&7x#5Qt098ch958d2{d+uDt~5>s?*)2PT?1`>?9^nc{Ds=EYrI@ydF5VCudE z4p^`0&>#UBX01ItO@hv&B=k#ctlc;2Qebfr$(9eX33i|z3}nI(4v-RBhz<5)A6ZRtZ2 z#~$O0R9C_uvQ)SEt%!rJ7^1&mAQ$P^JO)y`0x6p1Z*)h0aGvf=JVO{(+x#GmkLpPm ziiD2eJ-+Y)A1^VJxMatX^@}oZV=bzB<|SRhtOBc_#Ge^adA&TcRkLp_XTC#!ecYyN zk8CFf8;EIY13?l8ITUicjn+SN^Mam#;qbSMW*@g>6d|nx1E`?%;#DsC=?Fp!D8gwo ztMUKGa_0pRj+MLR>3Qow{x6r9DI(i?BgS&{m)R%R z9qJ?-J8xq0xDAxAWJd~LJ=<=Z-~C3{sdVsd7^U5;Kc(c)xvoD5`_oMl7{L4I12y?z zCeL4P(@u8!*Q&gLnSK_Xw~FNm47zrwD#&rjImj5=ECSI@hO<;;d~JQ8%1al8|P@WE}{NrMi@*B2w&eR{{x4524M+`W+Fq zt|^gFuxLnjC}>Ix#Y_rvEt0=w2V7+1*mT)m*I`d$dgu%0ySbO8Z@b9@RkOXPq{FuX z4xJ0?l`dt{KS}%mF{H_of_@)@#^3g{q&ziL`Q9;Z=dZ4WW?YX>nU09EX&{w*E&8q`WKZ_vHZ0>-pE7-n90eo z6GUuDAb3LXr~^Dtx0jE<_<=Wmj){v~NvK&g=jQ!5j8F1Lgrpk|xf3*^$kn!ZrK)o! z>@Bb6mv2puduc)k(n^PSE6mUQgD`TyzRi8}5iJrrp7!hbl{% zt5f=ke#ov2-Sb(JzB96x)rHwz&2Iy1!0iR%Bf1&R`L*%_8H))3Jiv{hVKJ1p>m3M= z<=1J3?@4fJZt4!rEr-hxAM{bNup}f@IuKtwO(o<3L=_G6lOk56x7MKH1S*;6;h#BJ zl~OOBP1o47_|k~pz#L}_ct(y=amWYB^K}4ijn9ar%0Phe(($(rfroiqD0jLi*nnkWq%f&cVEGqt#ifz^cw${`1uSdqgEW<*je$} z_dD*#vNKvyvMJ>j!GmISgPmXp;El3_f4Q^+%}`A^A%a;d$|Wi9ac|EMg}V}f>dCKt z5MDBWsic1^8%k3SORiT{^}v1O!ekf;7>x7DJQUylDJ7^gXMW3t;!i(S_FmhreYIHB$slJQm>hQ9fb=YU0cd zRIE^7KWglKRJYZRwZb0<^vA;mF79uf$3Aj2?p}Qq_dD6i_pnf9K$s3&WJaz$|FD~9 zi#?jJM6@thdqfL3aTt}5>ukcp!~{927&Y9e(KA_85f2f7y#RvRSJ5;u2+8p(5DXjW z&JgkXWYW>BT^eyg{qf`})5}j^E~2Qf125)?+Of-jFNh(DsU*V<>(9cPij0a=3BPTA zBUI4$oe2VdyE?bd?}xAp!~XC{UGf16BO9`hu^&_YoY6w1JGjbU{>42JVcuLBe|~_m z)ZiRu@7hvJB#FH7zS<+F%W%~UDrCFZ%Ko$!!)_VR6j6;Ya&#xWV?v|;1!-6P$NwMm zfwDGXIbk?vx9#4O|711iHfvTFU)U(86yLh;y>ZKdzTD+J{Czj%bW68jq7w)te9$I) zg!)33de1^LnD`N5@|lG)bXNV#y*mEJ`Xl1dzL_c?hm6y-o~*W|2A`O zjeoVYvzGFw*!O3xM%NjI(9=O5bMpu3ifNHS?K(%{#u{|YWLj&#y0y6327s5CPBBX% zcSFfhKSgccn~P)5+gvuv%4Dqy#5s?JJibM-S~Oa`k`q626XjPnIcilz_U&1cU{u;xDT|yUGKq>EK36E!2UZo|(7MU&*(ucR!mn8N5HP(>vhgPb`SM04Xf8ZWhX5~tN%_N$e z+j(UR{dU$yYp(1agAOwmfhVy?qCveYUTZ6T&VX(OGtG*UipP)m&;Va`ztIoeozfB)loZ4QW+@B-`Llv?=+s@MH0mjyRlVRB3d{ zXxxy5D#+n7a>8#H_-3ehFkak0YuWu|7QP#rgi+dd-l!gOpKl!CEl8Z&p-K%v;m0Vw zh(Ynmj8(m4<_RWd;J{l=5sXdNQZI51d>@Mas_o~3l+ygVit6O+FQ%*n0R=|9awAU^ zMy-OHMdb+%`(reucv>QL+>K>c6T7x{DHduVAvCV&aVX>~a&O-7vRo*^^Q96XYQAmKd~-8;o*M_Vzz62m zXgU?zCWtm8MvXeE*kkdM@#%)3yO5PZ67QGdC3Wi8`}>a11Y4-A8fk7QFzr9{*AroM=+Y&@m`e}K4w|vDHO$G>u9>?ua z6WZUj2lAKbykkY`o`NT76Z5AxV9Xw)#*_P(H;Z$gxOyYnR59a(*+uw2hCzG3WWt3O zB6=F$IO2V4;K)jU^!AeZ8Be-h2lBy7@nQ5Om$*eKqhrWyBr2CZvWKTc$#b?lM(Ii} zT0)O6%J)jO{51@4Vr*6kvW32A8Y}yE_-`NyU2ze2xSG-E`-4b12dV85o4M5V!C!;6 zHObVo=Oqu-ejiAR80Z^Vkn#oT5S>Dt_7;$8Y*dwfkRZj%nUuXDdL$$;*Sjh-P(9cI zgtd?ELNmjNhcX1C)?gEFXCWL46^$S)eMoc?B1ayboT)Dw56C>{K}eSPA{`bZ^WE@+ z1GcQ)!mMcI6Eem=YwodI63j`Qwb#z#$LPHSsNFle+3R|}J$DeWwK`t!3G_$3TKZb@ zm57iqY(l@1$SY%GATF_$w(}Y_Yq^H`^l_zac&p?_ZVV%t? z!OYhb{g*P<_&TLLqgrz-PH&SS%j9q$m-B)0J3k(}WA&Rqn?eH3K^qB1E?K`e0!WX& zZgyt-kks5bj4D%jw^H)+=I7qkd?Eqp%-k%1tpjs0s>tDDx5AH{H@JY@z1dOMde!FN z4b$B@w-!5okj)TCFo>hZio@veIyfa}X;o!VftuRj6Q5|6>iKW~9q3$T+y;VaG1R69 zIhCQfB8=q{rwdr8MQ5*!pT0y^?@{qDAZ!^B(kh}O%AEyV(90pMKV(DNO&BQTAHI*E z=1;l<7Ng2Zs?^dR?O$~m6jl&*{Q0#JC1zdJrlJ&mupNdC5v-u;4-Or<2SM+Vn}7aJ zdo}(GZ6DT9<53#Va<0`Y@+?sirfFsG4hFP`%yBt5xeFrdvy%!MhH^ucTjU{RsY5{;c$4ESum9&CIf8xam#VV zni(!OKTAwDnI?e}Jnn9=K&U+TD2{hf(l2vo8_aBGHBs4}uP4y2v6cNiL78h7`91l$ z%Xjx3@$2TT0ot zqDn&xH`sP&C6`4x@3M|QVglV)!S!fO>+yFd z=PS$Mppd7D6N2YP;C54O2Gc5?g9GQ+heSE|a@q)k&dhnP>~e4OMA>T6T7gbBmFKBD zx$`pR6n8G57{aEPOuRJ;OJHbp z$6V{x*)vm2k(q921rS~iIoOrrYfT7q;=~3R+&pQS(@tP(P$~6hmPY5(`xqL_Et`H$3GGBY&yqZ=Q4&+q7*G&&51prwh-wfX_zrAsBjT)xQRs5XW?dW=TWD2WJdy zy23=~HN6f=Y4Ga$la}8}dp$QhP!Q)=vL&ApZ{xBwi&H#_)`kBR=bYE}qNDkbUzbVO{eT(K3j4mh`<9CHzR$8}2Fo|1dW?%i>_2Y|omI z>~TF?hC@Kgc{;5B2}1v@BXO(Sxv&$Fx{E%cGDmpFgL#_@y+Bkq;ycqwX$ zx@3|~?BAp?wNp(`XU`kr%zk}*ZT^G^fl?L7gBv&Ti!LiaQc^8fh0cE zORqu*`2M#W)Nz2gG0_pDUd>L_5XPx&#%x`XMF-_Tn#oyIPja*64?=2D?`D=)zOu*z zlH3;Sq#wNwe57S$O!s>8Fg>qGtG4;uhPlwQWzXGIbCpr08^Yi@R;?l6bZlBFCcgD` zk?MYNs6p8i(p*{1rXf0QDvZ&MPFuR*=D^(voZf+sHdIN@5nAr|flg)kaTUAg{s>Kt2gHLcXB<3-=h_(Z6)G^_lfQv->C|UJ^tyXHab(M$lb#yd5zR$9#p zU2W~uYmqy)DL|lh8>x}N$`dE5HD?!aFYHJ4QY(;G+YTgOTt6c8oc^hk-Qnr}bXn6f z@+4Pv@RV#nfenNTE9F1ZDXoqZ2<=~X^B)%-EcqH)LX;6A6C!1vqJ+;raF%8iEF08# zqeg5t4Lep0N|bmDXb66kox`^Q9&^ zli>7gr9Gu=$&Fh*8r>fTk1&L%K4=zkcNAqkK0cdhkJXR#F9f1j(A9{`X%52-sMD#B zc#L+|ZZT%VHyu(WlXePvEn%Tjw7#4R;*+79*4<{D?aP|yz)B&ooXhVIwdA!l+eHRF z?Kh_ix;1OGwQa`5@q5tM5la3B%cjZG_PFhr7nlh`igemvQ5$droN7ND2WB%L@GFdK zg7R;AW8Y}5Q16&?FBfwETwmR4h20)umtCZ@T_Jd1(mmf_uaikfVeeJ0dDbGsW_cT; zN5^ZJ{b|prfsBgG>9*!b?N-{+?t27-u3;BBlVpoQ5!}M1+rL9+vtn z2^E1z_b;#_pe9C8Pqf|<>6mgp68RtOeAYOiUWQEHLefMgs^ICdSIu7=#?k@!9uHSx zw<+J5&Z~$j8Ln{R)1s_osMl@3!>I62?#iENQh^N~Q2{wXX1`VZCimJZR@ zH>%<86|Ze2Cs)u@?n23*{Vve+!s>$X=I)x`-P2>a`Q9#D;#ON?#%3s^9=`OXsmc-y zoIVUA48HweA;#Z-C?b1Yrt+q%(e=VGGDDpj_+Jx1kBhpeUQ3sN{NgiPTW(R(Xo*T7?}RXv8Z2$wNXr}2lfjCBEv160f>}@{ z@ngC)E@L6s=~wvpqmR-yRUXshpZvt+fbQNDs?*w5>Un#swttv2;_}V|M}0FBksl>2 zin;d2In)h0@kHzW*)a;*Y=070>Ewjcj5w=1X*l`)wI?aqltzOxTGO$~%=T5kgF#zX z_1ZW>_NK=#uvzY>Vu539#2ygv$^_-(sIf97Z&N$mt@y*qY$Nf(5RLLb4~UVcqhKD)? zwQ`lEB7-4o?+|SgcIejT?NAtsCCsv43zHaCxru=9>DcR{O}xvhF4Z2PF$&h13jt6W z^-7ZSnlAz82^@nY2+ip>4aviXs({A1Esip-5s$DzMtEaoQ0ODTf%CI;!=N=AmbRF6 zkTr^qBFwYj-s$BbQ79bu%Qv(!b&BV=_ez#Dt+St`zEaP3Ake@ zM?C}T6M3J$c%)X@xl|sUhCx?7^s>EQ+*H4VzSku)L9<;EEW75inJ?Gu_m_(*&~1Di zf=la9^}=a!F=H?1M__0);lF$(P`1CoR`vo-P(TGl{nLi6XOn5*5d4hUs_VF)Wug;< z7;5ibrV|ELz+BZP2p2%EBp@O32jwV#e8o=b4=>2GF5Yx^T!-7@xmQ9zsqk^gedaNj zY4_86gw6%3m!y3Dhc0$x*L>0vH}}&bn5^<2KjRejVQ~psX$015i|LiyljmbCjd9p6 zL|z=H?M2+4YO1Pgh`$|k7*75%sp5nrZ#7DODbA2N+j9_llQ}hM!R>8X^RH22{i=LP ze+tGN7LawV^bp--HvWT`mU2vsiZ;?Nf5{nt#z{D(7m^K>e!OU&?X+25q%Fo_a>0v;%mHuQeTl^JkOO2qr)opn5h5 z(#3C)!qWeyat^X&>p0#?6=-T9c|4w3mat*9*5B+pDpI91tQ=qq5`5RsSrZ?TmBNz& z&2G#~`iA+sBr^G3*E|~bNu`G=Ci2p;Q-R4m$uz6EVu?maMDDF=b=lT=4C;BT1l9*6 zii+;eHDSU5`lsqb~SM82M>wAdrwBAj>AxF}bLFEj&SzhpysVg^o ze?vti=e*MF_e4+-_ET({bzK_G%Zyd)+iVeoR$EqU^~J}7SRaBs1(8q7csU&3mA}St z=+^+{co@%&ZMzaUA^u(YBZgXk_hc|J{Mk31rH<%K2Pckm;)u^r#}jL0QmX5BM1+66 zA+W=i3g$8mFz*e%jaM;!GKr%SSgv71RNR5ckJrv#nC3XQQyhx8E?n!=FXl#8aKNHW z+0-A#_jxO{)P_71L6QCoU1S;M<5E0b3@^0j4Yt)O8qXP)JzCXUN`3D9Hz+o}*B!ni z>ZNi#GOv=GE-=Q*@Da%TGIvnsp74M@+sAl&yX=bv6yJ#bmgLH}P@hT^bF2Ll?R_5& zcQxU4xWb!h-ZKfl3B{M53-YyQsT^JCcr_H`{8jkZ$-L9d-?m$ghKuaqzw)ASEwT3^ zw9i^r+p%S%p!?(R&~O$d+-sv0`^EFz(VvZF~f1e%avV+`4umyZ;dN``X} zCYMzi$;kau?vle<5&ynFOBFziu9p=(I_;()KTc$9_z-*2I4G&a?Hs1{C-{EKCeSNtNK z-k!@3MJeW(0F1M}5O}I7k+Dyb`@Xo%j3a2V@DSrRu`EPm|MMv5UQ$ z1qnpkvuI`T$E@yCEO$QSOSFIIZ>e5xJbL!K4Lf@XM`XClIrKdq$aAJtzx%ihSlG2ey-UHF)4z60=}Y^-{O1AJCz-EYnN~BMzo%g z*>i*)HDfGwiVTGn0128F#Xt=4^sod2r;i3s<|JWzq~`}SkVXKB{C-=8^^$>Rc&@%W zD`%Nik*G`R+iD59pAhh%RnC0zWF=h_Y;js!TaR8h`dsrrSNhycerY@ROX;rOeBdzv zxXeFgK$oS->ID>4RSDaE{|1YaCdCVgAa9 zC(z=|vikCr#|6rdIU`MM2E@fWCpmVB@gt=5Y~FK zP?FhVb?o{L)H?ibEfM3Q3}jz66=9<7`&ySK z=yTGX5GNKcI75*I8O-4lrNU$^S4rU*PHa{!ceSo#CXIHryfsk_$?eIam?m4H@sr(| zk{`j7zWPpaQFHX8(YC|*O(;S3E=nGv$vbsBb%&$Pbw@ksPxeFV-3*AkQmU=5pI@E( zAZOFd;}oF$$X)sL?C8G26VJ#SQQCjJ5-TRXpyL6t5)K8^#;c@~p{0|pQNZQW`QCYuNz=&ZNnxCuuJ`N8M?is^ zY#YXkXd3{bOzof&8w+0N)X(kjAVpd*t8q`#8oRky^{%!@9UIjx4UQs$R}B{dmHOZC z9!R5qBDU$@Z^JsHY3;1t%zt0JH6n$1myKBRV$}w=1ROCd-0|Q549gAXwN^)5^5H{Q9z`U!e&QsbkMQf$&u;5m0WX?kJI9| zT<$z&Nay8DL3XC`_$j5%Mn{J!6A_fXpIc=%cDLH_w7v0vUY2(A@eAvAEmBh&x^11C zJ=T(fut|KAr+Aj-e>V!D_y)UglA()_#k$4Z%_e03)jb zD=u&~Orn9B^(H9CCqBHX_=D+2;Bwa#<-9SN>#~!7^f!p>{n=dod3#7S<=PN{uv=I% zy&3&{zx!b9&NpHxr9MxTU?b8;%ZMP)6yqxOzK=6}(ruQ4v5un?llQL%GU{6s9iIjw9E?g@s8Fi6c%kYGag;r>I?MjG5q~G@hJn7*6%q! z5G9jb@I?h`;ZN(jGkJWT0%25GFx+jPI5)rQO6dPUKrM19sNVhH}+NZ z<6d5AU&k8%*BQb@ncl~2Kxm1nBld;ZPyivCYF}E3D&mC8Q=G5kBjTn#}hlvE<8~%$el4`y~H6!Y~zajymO0UwY_Yn3A zP03ECz6SDUAhz!e}30cb5vO^3Vo0&GUO|70y zOHd(VA|46ro3uDV0e^hhVb!r;qN2w^Nm#(2D@NF%?z87*_45$Q4h7E!>#q)BgREj z2pGsZfg=XQhkn3X=cYMYSBm)X1&S!A7N&&O4NDLjy+V|yDdXAxOsLqs!BaF$#M;V) z$G?#pd)lzKNerCrxv}G&voFNt?J*G!YR8XEsMxE-8+Qh=|mY>f8=!I%9ma zJsBr!{2T7BtSJPu{K`#&Cj&+*y8cnzj#GFH)Dp}m$tFyH&NbNsAFX+uUyjmS9ko_8hm zkjzbuHZ-wEK#dlppT|LN!oUNHS-sW@hTH^#zusHQjt4+)BH)<-yH-IX+S35^ID7Ja zAt{4&I`;KZ_Od--^P|J*dEC2I*$sca?hF8W)%I$+o-Q7Hn4?&>=^T@Dr)o60P2mG6Wd zWc!Z8bh2eo61`$lyLZ6TdxyhGI+O9#Kory9E`5ybp+smoTFqV(QdyY~NUwJQ zl2~;m@=^f$LHi{62@UtHf+-yrN3^-7p%$ZzT4C3Bmlk}TF}LCw(k3cQiivSE8ry%P z{kG?JGkp`mT_WmlSC7|V1^T7_q$2HX%*(4J-!CBGq24Njc{ENO8zRi0RVD+TK#mR# z1;iEU>$YB4g+AN|#Evsp7xY9#ncQ z?iWd5IBy)kw*%c{#~_YqQ_9lC1*7FRgvyXqwa9^jjf)YPxSI|c>>zN-e$U9aFoCbW z76N<5giP$f2fVsUnEP+!;gCNHEh93>K)UPG+9|l!7SrNpZNmJG2Q60ZCvXexKW6jb zXU-?!7G{^myX6eK$w=_h92N-}*0m;vF9(y_v9)mdqy_-D%47*EkJHbpg z=Z98MAj^`Yk-~U8Q`|lyVF@a?j5zijQF$7<986gc56Xm5v0}!! zsD&~Q)6zMRni`od=GLJA7mdU!kQl}}L&tXQc95VUg6&fWJ7xKW*jk{2M>CRcNubr! zQF&O^CRzWdp;4)i!1O36_t4PRUH3;=wO<|@nkb|yK=u)aOZE!f(F7&rW;nuc7YzA1 zR8_aXt~`w9D*sKMp5iW+9kLol4lExNWxy}|V8xgXNkyxJgAOGZH%kO}?EziYS|S1d z)eLl$yKdkx6)_!hJ~8OPH|hQT0}n6jkznK#|JO=z6PfRJzFHZOhIsJ*?QNjb4|6m- zxM;e}r%U$$@S}>VMG9k_F4&KE7jX=EeuhsxT&PM(>Ngl;>A*=Yn1qfOC-z~A=Q&|d zhBRch>|sb;*lHl}ikLaXAbuiaa$*>!V}7DTOCDg=|Nh+SO=&~~v@c&TLl(t`*Ief3 zUKEsWhqY; zj2>PMBT7t+qVm(2sW`X>XraSXW%E}kcT$Ttrv?-}A_Vxxs)S~aMvWgGz1gC|c}bT8 zr);DL)9Z$hd%BK#%jnt26cBA5wr%NhE##P+zXD)26{h_|9*FC2PrH~5-2&Dm<5i>J zRB6O4Jmc?sK0!gvN=LovD|(B;^d7S9|3X2M>1f@D_bL7GKC}DXiLh!@K(7g)S~xN; z;y^g(`@=^nj^B+EwfuZx%D3oV4DEn31mi5txy?(@$yzgcJ?X&iN;y^_KEyy-QuDuX za-=Z7x0bH#Xj~!AVh|Lh;5zZd4@5u7(^(>*j&Py#fBoRQ$Na%kte6?BKejqrk&r)% zEhc;>Gb6&A&v=A#iua8OCyNF@F;RdhtZgP&R=r5hUg}ts`0yD@&Avrz1uc2!)K12u zq#$vkd%NQ-+n8ZPvtR3B;5H zZqlh>;eYkeM_oTuvSl-NUZYx)KnqrLc>=P#1)=x6c>P3f^ zu#2}#txDOVyx7x$pIhR6k6@~A1;@NqB!uo^ae0t2hPmuLYV6rNk&cL`nu*8~GFb7yV@U*qj+}#&8DZ9pI-X!JN&dNNME0QrTx#W#u| zN`_jj_w*YOS?!7W!yAUj|03n!w~rbAa7I#akhi2v6=ypoOK6Cd5<0|pSXv>ypvdNL zLP{=|nz?`+zBOEbw`Insq)>#3hu4ZDlG*iaPj4kxMf5kVVxmH!7|l72q?A+c5IiYe z4<`bu+|knyKJ3|t51W#%0yF91!{)6tibD`%vg+p1D|Dndnb^(EV)q6bB0U$c|L{zG z0FB}ws+ue^qw{VcB20<9%qYy5M=j*X^S5;nOKoCpy&nAJ2&9p&Pv;?o9@cKAdx(u= zV3lcShq^ZJ)S`pZo9icq^%P>O^rFRyp;}lC6~2 zx;oM_4%gnnAiVvbX%{zyZ3lg6e=c&^UN7IdudhVMXtn%|8X1f%Y zXFM6CUZ_?G@@(qI80nO-of3>1c=jM|OR>ms^+e(dR`~?7-Cg3Zd@WV#&F`wLZi($1 zmaGug_0=vYk7xuipV9f^q1U$c(1G1a0$d^ZkKlHOFPFFfT3b%%@Btg1&;w?lU||o* z!`HGKo`fJr<5|Hg1twf_U`M)S($YVl_b}m-mY%;ad83EIv_LY$z|R&*$0QArWc*znu?~&NX97gOv5aBUBCivQ*GM+s zke{HU+T;ZHUuzVg&7l5F?qBftwR(J_Vc+XzHm^33C1Vt#clc;=9p6%jB>i2cYLO~R z_V_d|Bne)%f2ArO-){EjQP($^&DoY39&4c?hB1a}N z>ptt94&pe|-6Xg*Yq?`yWMmi7S?hC+@0-kz3Q`6wUI$T~+kO)!Tjfw76^;eYBB(M| zlEXj>r}v2hC@AN4>9o1@^UBd#5?KjV(am`unsKT|vCWQbw;xcwvZPh6u8~EyhqniO z4lfBn=`=x}oSV)$ZV<5tnbjUdGKfNKaM0iPShGM=8)d3bN?(6CL^4WfO$=jqW^A7- z;B*m_)lD1eGDMQ&AA05Jvuv_0bjWI8_3ewr!Jq8`MJoQM4G1=XU8eHCI`zJ>*mX@iXw>SN?C(BKsQjc_#WT0D$5O&0yf@}@`? zztK}Tlx;z&Bc)=45DH~NRxLX87Ry2d%ku{i?=aXlxL3J7d|HNkOT5tcVX`CbZEmz|sUbXYT9FzK5#;v%@@!a zgkjY-b1`kfZK@4AYQpp7c-2M5(U}{Az@z*i=25W4bakS>WN#u{v=&ENuGx;o(Lm6> z{85hL`Fyg-3dUIJHS7GnLxTbL?tByS!s{!tg@?jz^UO4Bd>VhX0AgzZtq~m-fSVf$;(=1t&%B<(DhTHv{AszT3ELFHd~3Svni8{i!~Ge|hM&bGKAmp*jV+LLZ;#4~!{~(iK=b3~$5WUxCfaQ1MnK@R; zLrCmm)A)2l8aO=jj)myM>qE1?mU*bRq?21+B8CSC(C|;aSy@-(uj`4L9%1?*Tm9`B zR(_4F91}Es!Jf4<9OHvEd(q_@ErJS%8x1$Xtqy0hut1hYN4`WCFFtux>D-JI>N&YT_!nvwy*12rkLR;6 zdaUhPov2!_jaC%z)S6AUk8W#0OLvp3R77af-)Y5TEb`^5D;u(2^H<0uZI(HATnB5l zKn`JYX#!4fF+P94fj1}KgxLEfhj3D7cHAtGEk%|xn=A|}P1f6s-5J)XmD7$j6EORo zqvHNV2BD8cATvkV(gIY_cbU;{wlqbV_T_$0Ar8k6zRw>LlI$ zJw@g3Or-O>Ks{6D6BsP}z}Bo#(Ezc$-Y|I;-qMt39O~4N%OND&r-|m@n$)Mt73+(HTThhcs@&60)|oY)AU%5&?Ox>g1fXr|5kF1mslp$u zw^|mHR_L>JmIg$Es=+H~l!2R|T?cbLtQCF5N$4{m-Th)ls`!PhFJ8iZhy2wSzpMj$ zYZ6jv@xIWKA%VM;z z7HQpLU>0+&C>6Nef6-T|tU~)#D${05x7-_wcv5M2oBh5JX)QjRP;BMuWYQ^h9&JA} zs5*~V^@OA#&nPkD(Xp6wL4)#)5N>U~Yy=dG9BNhJuz2TanFi~e)>?)YFkNXBxwC3D z@E>zTdD{vaZT_Vui$r^7F$9b~R60s*cM#@#o>VVCGKS2J+s-&{9Yfh~O-0u^bXaS` z8u*=_H>Dl(k#QMCJd|Shtf(+;W01oB9K!$oX@>+@NfkHa*H%$ZH~!{bk2=Awzxuj2 z4a{6_O2vyW=p9TcE9Y!YApNx_!RosjFZRVF$fiA zGhX~HdAiBay5Xf$B`yt>fLf7*kryuuSkliQHlZ|w>>Yd$OL|D*#9FFc5#iC}$3?Nt zs^qQQs4%3Zv~RcSMT9ulF8n+U@jCPegsM^5wMHD7sWgV+hF1nrtrKU6|b+=+s&9P1jfl4t2(UuCf_kl^34-i8fEc zV-RBs?ASj|XcQghD2r==koC`NurW%FKSJkv$XP@_{{7`UBea5{b9CuOc=61r;8GvC z&Pr-#j-XZ)W`WCsxIA3wAX}clbs-_R|kFJNi94*6lh!t4A8@ zRG);<=@SPV)tm+mVwi)^vyXj-ter&!sM8;Y`D-8yDiXytYMzRg8{|mw6Io>2Z?Dfw znp#d%9;gov^&pmDp8DB!6pa$n4(xq$J56T|CSmTuSfzv1j-Rmvrg> zXmF&NE(Z;nv`5G)B*x(ce`1P|ns8yi-m%A}Q^ zvfaraZW?u-tn%Ul`4oJl6utvVdnpc~Y38WTIZkF@BJH9uL7e}?-djgi-F*L}fS`m( z9zs%5x?JD`6+^ayvY zPfs|Ec%sz(rzQYU;TBH57A-XpD|P9>F3_ol7tjkiyj!6bm;zxo*EW#>TXlBeRM7T+ zPgv&-+a3!a`BS{RTk?V)sj`(SF_+x*=U|l+COAVXBlA54(joL#v%Yw%0wdD5?l6Uv z+Xm6D*X*=7Js>Sfak(^SvR2QVvdt)>)>oHEioNX4c*V66aWehlbK>*6HG5GPg{0HC z{-}DLh|Rcqij|b9ogRa@=U={rN1>rPQ*?(G?jw)YObKJVK!u&+qLU5yvP6|u=;+Q0 zXZAT`zE3H3K*(lSB-(m6HBm72c*19U5_r=^ zswb;Ds3p&`nk=;z7hRyAA3ge-4vmAOuJPRpyTPl#X8)beaD7Ns z?gg#9XCg3eHDbn?(I${?R{|R*E9ab*?`e+*&^6Nd$74>MgblIQ;i&;KLVjw{d)0p6 zs~`A1rRsdhpc-G72#dr^)pg(CWQ06iBj=;IN}e|SOA`f|(c2G#;zDlM;tOO^@$-v* z@O0(8zN`9u8$P$r-1H&Z?4|_b4d7x}u<8U0!4WDJ012JF`!u>EIjfU}!L(0Xz45?t z862sWq69ns06V%#gBVgXSzzr_%}c|4G({EO&QZcHFjNN%@2muZhjjBt!jd0Oq_55? zHo9y0J|@F(%|9kl+TPyFR$sU`&-ez>&8siF{L&BvG-u@HnofbulST1*cusP0q`WP&$EgCqwSX+pakprA82ha)H*= z?x_=_7=i0F!ertJu=2M%0DGM5mIgSncX8q6l-Xam*Dm~_WUi!(!@|iw& zqEe-_82?mbA-^#dD=Vu8+^>*uHb$um_U90oAC#SvXmsKnjNtAScZMI;JQhMi(fpGS zgvS&J`lU2m`$;BkcPG;EJ&-<6vF{ZIA!N3|z4RiS@Oh)pd0)9V1?a|&pSQ7hoe@M5 z0zy#R8$b*#GqmLKgVi?%%gUnEH*;#3u^ZA!X2v~-j8JtNJe@*80;X~Z_@>gC{A%+4}>Zf$!;O>8I5DY!gilw(S!C4X83RE5HfUX zo8Y@YZC@#YO;1~HJUnojzJXCg+#41f>7@m8+A&xIr)PW;tj1Y*b$9GtL7*z?i{;F) zK;fUrpaiMR>aqvS)7hUy{Z4q58r1c(O29%ML01~MDPp=&{ zfRKoYmI#X6;|Lwipr4YxK>;rn*}^Sn3cZl(#$ z*?SM$79dYp`ET;-4a3NbJjwd z3J6!giwzw7sGgSYqk)L^QCqM5&Oyy-dKf`%d12f^xABa}0g zoPPi-jrxCGdMArD>Mtz7&6_Y&8^lPwep}lTAaWt}E8wt@y49~4>BR$M;lB9P3mlYz zavyhCh>I2`Hr*zfgH-clzv3UeKdhU3BE8gZ2A)3i#ofQl#=^?3jAm|8&8Pk@+ka$Q zDA`Pe0*kz4ZdC*#u4}Xi!n9%(Y*>uS2hFf|Eq)J+1JCmnNs!L~RyJmk)Bo?XUtneJ ze+<6H-c|Tr*7Up_N;Mw}n@RP9uYuS8px`+f2xII@fobAaHetqza>Dg1G;;BYsKlOX zUiGGIj~UR9f0Z?cm90HA<3l;2{av>FQiB0|R~#mWvu1o&6!0$45)2@!ComQXPFQj# zAYLT=k|>WEbah_rU3yqqU1QGB2Y;7k2W1{@qc}leWsBJpIwQRD`58Fn-pJPjmcwEr zsBmJc53W+5VFx3erUq~6AHjYn@1#^!RYq#Dqf-^FhcQkHMsJb|arpxf`mp_E>Ze?2 zXaXzwuTOj!O!bT{7qFrLO8@KQ!~=+#oe``ELj>LZuMay3co<*Q-B8{NFkfwY3{d^9 zzL^@&>qePSPJ+7+)yakKbLYN&96oCQLZdh7=Jn=k6zy6UbZ{$bGcd09=38#XLdD0t zH+jo4T`K8wp_S}D5$8(bY?$*SYP2=PmRNxU{*ZUi@0 zJlE`NlRZCniiSlyIwjSt~{6x39O3rl0in+U%bQ+>aan(ZUz~;be@Sz$FKIp5V^Yuh4+z6 z4Y##!9cYlFw1eZY#e?L{+XjEDDaYEDCGL&K+}0y- zhO=%Y)H1locM;!9BT&KR1Vf0z%xX40;G^X>G7*_m`Mj?~HxcE6j}*~Wa2YjGi#;y# z-}ac6{*t{}$@$$`IrYcK3`u;>&4#NnjCwdseCSX1*uU|c+(M&)SHP@sJ3{nG;h)Ux z*IRf88LZvmd#G4ZL5umQu(yCF4T=J(eUtp2nu7wq8q4;R`rd4-*(TrCnLx5NWxJ2I zH2FoR6mi_}V3zTR_s-9^-{lS*#*Q#A&4G|CU=KT%_mnBm(nV)6md92}mw9rfu!HY6 zLN80CeAnLFc(Mp7ZS!hmS&V;OxHm|f zx)4q+voiNb)B(k?5BuSG0!TES&dzZonsx{a_&f`vVSxdSgbdICWdz;z!}_UN1$gKD z2NK`Bk$WZjQOS1XIfX(IGpvbQ4nub+&rAx`AIPscm8xM>3wUO08)H4Qx!YUou2o>B z@y#m*JrsBs#_u@SI>NT_BwG0$?Urn6{0vy;YYA6K>>?koTxr9DT?apbYdSsaY z5);+D5m2s4ccUk=LWQLb#LBZnBL%cx_g*>(c7~l(MXC>8>}A`xnz1MDb}eR|7|+_U zo4csI;%rPgVFLMuf*#yDHyB*ZQm^))oUHOdb@;&_G-wkZ>UaSG;q*$vVj5ULzgL)? zx|Bj-C+D@5X1eaoW44{NKfTcMDHm)jb$O3o9ub3#zf=Z5S*7QchDl*bN)_0S{gf#6 zagRdeSAb2|DRO14h{I6J3tjDn0-lB`o@~01#FLtkn8}9Sy&7bFS zEPp84K@(tGoxxBnvLt-9H4U$7J}NPX?<%zi+nEm93@9=tU30XyQ4SoQ;xDr0s7L12 zku!h@wlIPeHfUS2|9shB^LRb^|0}b@U;UyXhr`+xRx_v| zucPQWn1A0t531Lg0gp`uCwB3n7R*Jq?Q{i<3#(&;4mTK}iOihQlk*!EKrw68^i?@! z5We1-VCV>9vuO-F>aS>t&@W!-ORgh$TqEQWf6WKW3Aabqz~UEJ4$!4a!?bK$Z@bNg za^V41ExfA#wVZEc3gl|MTpi{-yTNL-2URqt{^if{=Q7);O0oWiU=u=cFSx@8IK{h< zHKs-TgF-4F>>TMJgatr~JYfB^zD{4cf<;`fE3y7#@Mkn7c7H4=_7ZB+_vSHKKv zbv?Vq)LjoVre~{;gA!<|p&vR*1_m$@<8IkqwVqZ?|)^Oo6&)LBuS ze1s2@Icln|tilC7R|k4N>zLy9){^caBILwOlzoI3AiA$ap0ilN9}ln#!9WHaBTaav zuuG-|!t*V}$SI*>eNjQ(ZF@S%v#@CQw~dWxRJtCDiC=1Oub zf8-_ZEI9pZ85_vhS@_&j3?=YbR|6Qo2f6R-c(@i|KvY)XKjAFC6)i@g)xgivFaTP8Fhh&oxVqorrf^ZK^v_zP1m5*mhFV&A}5lqm3|;oiuW11 zPbbV(TO$ht;lPB&0s<~^$x z<7O-+*mhbLJ?dymQVD&O{Sn53SiA(qZ7LzpWrouK@r-XKL6wLbiY7vx+B zk2z%!D+06z3b6(pm@0|bFC4{`(h1(A-Cp)GzuRN$0RVe$@m>H=Uh65=hqVy4roh#+ z)*oG1GS`a%Cc-|B@D$xITmKIW*ozDX_jwzd5cmyPRUs9voM@#%aeSl&=(Qr5W227<>iPkx-`!_Ms~T1OWxbQ zg_u%kC|YS?o?(A|+8^Y3q`_gNP>Fq;Uu((>Nvm+&fnue;{n4ISKU`8x!P1+#40e^G z8jTPE3imUKA?K`)s^}2HfoWfnT8QPN0{w}i6T_h7tuarrrAZPJsGvWgwVLPW=I3x_ zcP3`Vq_V0#1AX2%j-_|)E%%}xSeb}dW}s3OHfK}b&%P!KeYjT7E}$!PfBqbCb7}uu z-D8>y!X`t16SduaWg3m{R33HP`GVW$bq0I#3{}KZrvj^Cyf&Iz5(!->x5K zgA4safAT7jdXR#?()IMX_vw7-YZK05kQ5=Vv@eLzf}Ok68R)p;YHt0DMYD@az7;9m zQaAeSBaWprowTJ`EJABIwxfU%XOW?Pb4^Q`zPNf`++JroxJ8XdH!X?kV#@R`LZ3}b zy$fbk)8ZE;MzZJKtT-m~B#5%&n#x4b<7avf<1v3oRD?Yz!(i+w* z`Up37#F1&F3o*4`N9IIuE@x187j8R8v)oZu7>^ls*>$<5!8}fmwVfR=^hR+fEq51n z-KjRaI8TKfId+VFVLEZQaC`Fj0i9{Tb21w(F2bTsF(OUE7|mEu-^1Z1*9gPPO!Wl< zTHTdoMD{H82MGqt(`J@@=81)9@P>T@Z~@(+k|-Uhdn;0mi#xuzG_mz4LvNEm!nDWc zKA^*yq<#QJM0&{ju({z8DH>S(2BB$f_>nb4)>_U1E$4-=xhn<0QNDqKW=qT?8kYCS zuI(Z;6!b?47qz%XXwPg9d2%h@wJaAy7<0l51x=lI+DJcam6}j_Zc?^9diXDGcnfX9uT@CR6HIY7a4^ z#3N;5Q6l#>F!esRQ5yb~>#JJcmkHYW87jLzGwu~N_6iq6mDLSe;WoP&Xdvd|n&ml% zuU0T_E7~^x7R}kUa4Bpj@33n*C99e*Ztz1=ki4Gw;^a(4-HUej*zqmt@5%S8vx1hD zSlxx;M3Xx&u#xsiJx}Qtx(R+{#k|uAYm~@ja^YWKagFei3?@(>(p9+Leo>(F3~@0C z@{$MSI_gnZ@>q?4!|gFnj584}ko=LvQ@h5W#jF2b7-^B?=`^qP^jT1?aWoP#`wwuO zse2|hl6_*y0IUMfZFY?j;^hmF*%^i?^|u}{%qH5n((#WC_~IuL*z!zQA(sY2OmGHv z>~J(yxyYLz-qxRA3x#fcqQRpu7~FR=#CbfXgk|>3cC##uW9$`YVOpmXbh$hIb8+qK z;!w)iSEt6u?bPVN(0ZLnb79<)Va2UF?>X}|_nAse@Ckz>){@499+#{bluDW0{U6_J z6CcwQm9*EO%5M;PX&pNxHKZ@aC)^HDOWXZUa4ivC&|kxB5>1zu(T0yjFy875RlWNe z8*Vz>E6GjZSnflTS7o8GXt=-lat|u6lvb@?8xKRq9=S-Y-bYdf)8>LhyaNk!ulbpS zOXOEp&s)nY?>brN;@(Rhj!0kDHZ}j%X7j+spqF$N}+1J7&t56 zvcn&->p)dnW7nq_TsM1v>Daz0>1(9Snw~?5k>MSiBawc+L^DF4E-ZU;#={|9D#C9= z#vJlaR~`k)`;+lqf|2QflBBBcdGDW(1y7$kk*s4zRr0bR8O-sL?0gMwH-w-QT==2@ zelA1bx$)v|IIx*C777Iijt=0_@I#;Yb;J2eY{b<`wG~uWv55yj7Vzfhd39XE0@hv* zD@~QtiiZ1E1(ur5<5ZPO1f-8`BSfi*8y;SLn0X-TVPHV?4&r23grp}3v|_`3L0#M* zkz%E-si{c^V@yTrrQ0R`jvjLLs&tiCaYN;t8y#@bpDl1O{L(y6-uWvDNi0;3nv6J$ zcA(9P&-dM;a#?-n8vEfuK2+vO{nJ$pjP|Ul$Ku!6p~Gs6o-ZW(m?dU2dA2t&t8g+bjxS&MwdLb$>H7PZ(ML= z+4PAsxNxzzJ3*0^*lN#*wX&OzVnRlOVfW_j=_=rSeg%$hRJ&~5Q!I_3Xn^=0d&e!` zNrY?)<)l9$4ZoKR<1X55hpxg=_69z=^I3#dNHH_iisO|Fx&GqD zD)DpcdYLQ1D~b3Bih{i~%=}6nUfkb8a}^~c*|_@V5E^yy{7fKLMF$kup%R4;V-&1hgr?#aW{K?c>*xm5ac45t68|Y7HiMM?KTRd_IMOF2S;4 zrl~7-l@v5w)!=m2Vt6WrQ*xYoY3eW?zI4d(`bousD8Wc8m0_w|lV*S#>i750*i2H8 zi0)~S8x(OeW>QdSb|u_;Y=7rgrOZe6BAMcDC9`L`p%b^u+`=I?l{_-X$3l);5wwZ) zw2C6O#3xTbXg^{ig9OavIMlOqu#3xLLeY5aRaKX(vX|624=%Z-XI0Qp-iocsFw|PA~h1^ZR z^P^nDsdH3)n6daH#pI$vp-loUll7dMZ5yuN0}Ha$@}$&9;g{ZAoGH0fDUfI`ih|Nb zWQ~yd(dtU~zauzLgtDdy%ZoMQ$m@LD`+2q=xAyJIj(mY5cBD=;Ads#>dC3Je=u@Xd zCceS4Pa$5xX+M3(>4(jPChEzB<6NgZMSPYq220DH}q=i&CF&=IkoK6Vs^U@{ zm)oB^>M$9CR+RR(%V`*A3K!;{a>vv{QB;|$X5(#IN@bvL+fnz#jy68h>z**99?B4X zzz-PjWUqQ7<-vt*Ahv`GEUZ~yKfVGsc%FP*j``pj$P`H)hscR z*u07Wc%EfTjnXln)VGN3jJSSGix#ot;Vg|G`6$KTItut#bjp+0#zD5OOmXl!IB4qa zgZv&2@ja1UZd5n;sap~ZvH2R@+4;7Assu_?QD~jGjuel{V&do2DOOVE!wZ>qgFDrR z(ifzy&|=xMMMbOcjgcnu_)50kdmvwR@(=-xXez0{02_x);qDJqc3Z7$BvVkF*E*dg_&YrT6n2IWq);jBFA= z@*3^@XF`HH*0Ih|ZvBG}$M2us1RhhNh8Ntc3^h~fTRo&gjKc7&7HC7+vOf@@PjSGS zl5)}yG@83Q{fZjwCJ{K26$Y;wJh_U~%D}-iH9oG)fdPMBbMaka@T4LU-3atO%W7CJzA06u(KndsozrRd#dm0*hbKeb zPFbCDcU$j|(Y@BC*L+-i?wC?SQ%AyE>)JjlA!repN(LN>rm}lo2_}yL08VlJNDgte z$?_$JfSXc{{YU#nw%l4NYd*}VV1!y#*-hilBsr)XnJLDCgwEGz17?>msPG!^lQ$h4 z$79zqrJLg5xNuh>jaw>qK72!o!t$o(d`wINdy1nfRZ^$mWK9VDXO%kZDH6x3Rb>xL z_Y-lXJ+$%B?4-2HX~n?=3B{fRm<^mXxTa}CF*vMtrVhFg`M4CdJt!&47z&SE#)}_a45(MRsEzvo)lfXl4**h)E!rk zwvQjy`mxrUN(F=fYcGS0#g!8Jq3U?7W(WJ+mATgO`{E;n+7i7HIjX(VCrJgnN@H-* zy=Vb(k6NG7rNeC8(A!A;9;8H0W29+XZW{7&XP3`$9!8yMooR(So}t0J3hl7bVL#Q3 zQ*4L}vbipOzKbJ%I!J5b<#R=d$uRVIO>UzB%)-H}Mk(u4h}NsNOz(*2T3_k~<`5P_ z9|#srG>~=dbhs+@Qi2>QbTds?Y1C4s6j`p^Bvk-3vbX@e5G<79s|6C-xD4$E?vgS% zlVhBppKJ{vk+8@GhiJGVmx@@XVoou&8|l?}2Qv{<%#Vh6n;4-e4y+C%6Gkf&pN<-f zL|=SoXk-rEv3}1&^Z*)w)SGA}yH|hLcgV7me&0+{!T<`YVU+hECLlj<4f57)e-l?V z>C+_M5d;ZW~Bn(C$yv`Y0R+H>#pIs#gf9&cRdd}=z9|Sa=xi(_!%iYjXn$wmimQ%QS3LH3}zLlf=F!*68 zQaALI6ml9wNudP|WEl!Rt{eBEsV|C?Bj+wOMt3THqtxe|QTX_=2O(1F%7aXh zBxkjruE7zz52XY2KN7yigUm_4?CY(Gs}Ph@3Jb5WNDTX#k=(zq0I5&TC7Ptg(BlT$ zR>3?)&Uw`Y1XPozBj|(c{SMTyCylb?!$FXvj$$+Zy8H_FJ$*-+%d}v^L+L#B+wCbF zpR^-l>C)zN(D5!#!jS~-CoT&fE04XpDA#-H*BW#&ME?$xK~%9m{?;v{X@5O#ExRAy zVPOt;KNEgP>-vC9=*C_d2RXujDAH0^ARLzIeZU>@OP}5~vXgT=t@E zRi1m;XeCjx4z-xCR+S2W?Td~A6UX(KeN-lg_uZid(}rSAw6a4DE|X1ALXas|sLT~- z0wi^~%{ckuiDXGo&d1yuBKTL=G}!1pou~-c$-~RKnpi|jiA|YG4`gHZOxu?k1PnSX z6cUY|&xGSkEM1_JydmXD*N($hD8MXQTGY_2Rg0PuYL7L05RJ-`u2clEOl~9j{_$tKy?Qw^v-f1A z?RE6i8B5(gx7R;ysq5KUiPU8mRed-Wpg)ObGD7orT6iCZq*bs#bLDmm(#3M?X$}YZ zt25V}>p4QT0!%i;c4!E(1||-9?MngWcLsh=vDytJ=V7=nJB;fV>yFwFvmK235+_}0 z^LQ7+FK?~9{p5%z`h|Bg07_GP`7{7xAk(?o0>58TPhD5Y-t0PcU;H$(e_Nw;K4y6b zA4uz$4$OB^uUeZdV1G8Ao*CL&6;e8mcP(}hq{TbvuKh@Eh?Qskqf-$^((DTo zrl$7+#mQ|c>|c-~2@3okJ@9!IL^-*+W*m&)uGJr>xVaGfyfh+VSFPgiehgke>cUr6 z8z(PZB>(;jIS=54Se&EyzOS{_%l1T2?9~gb)TQFLuA{^B&|G@f>;@_8EllPDoyIaT ztDV920QvCyMiEqzzCTJHIn1vLJOZ$OUB=5))(j2jTNIgA+Xww4Dt0{BdsQ<=jS{0x z7D?K@sHO9GQ`s~JtTO4ch=~yE8RhdtJE3PSXv!3Onif%aOHS9U<*c``r9^4S&Rse0 zrF_FiK@d4vzsP+eIMq5#!H)Xuxn#I{BrP9nW1`0Gd5Et1mt*sN84GZmbjD6lrlQ|E z`MX_W7sp1XBN>rW5#pcRWrb2fSrjPM$Ie;>;P~{El<1KWMh$~J4JQ28cV~$wpBVVl zb@5w<>GPcS`0N$~@pfv&uD4YSbfghM4!UPQ!s!HLL<*3*Z@cXSkmYh9x-`T7#pdbw zaSSreR9P8(SMhi~A;@1XD)7uDho&G0*^RF(RO)sL-o|$llBHO6BW?W1cV!HPJ4&wc zi|yog6cOrXK~$Z2rA=a8Q+=H&k_ad$AUmS3skY~y?nlD_6!DZK^+?uK->#IPCfez< z#K|O{$~FExUj3Df+aZ=#(Fn-_U(1%Fncpa&m5KzQFA@^}43TOi5<7FRgIIkeGKx4U z{h3L=IPl?>Crp4M4>-j8QNsj)u2K$!Y*;-^$6iB)8e@Jv#|rG1$B&BAo={1W(T6AZ zMPZ}EWnL6Hur=<9Res_{SmX3G*4mj+PUI1$X?NF*5DP!AD6iwT#ZYkDgp^Z&3d@TK zy)WJxHC!ul(h$CoQ}wrevPb<9$YBbd|-1ryU|6 zo->P~4v9!g%Ge(y(+g4sIL0^$b~urzH24CGV{vE0IefAUZf{o3{0mg`dzXb&vohi{ z*Bz=PoS@S*%+098xuC5t`XIBNXh672AX)RsHa_u?W_HfFvqde4w^kaP#ZrQ}+#`{| zv6|%O>PrMg`qZ!PYQ(gsqQlBHxBOP^(6?gygol|IBY#s5RJxb8E|@CHOIrlAo-6_#=I!3A>=5H+GG&usHOy= ztbYbtV1XxCC`G$Jj=BawmQcq^jr;ypf(?}5_fJ9MgTbt6g^lq3Rl*94iIZ0FKuQG& zqDSIs*1!KxhkxOl+w;YTNofsKhq?G9z?iBL%)9fg{S#X$cGg4 zf?YxD+)xhCZJ1vs|6f(agOg4{vU5mo_d!dk*5KlVe~iNo(Nf?COKpvP}BF`c5U>532s38-^jwck|zKT?%uigf;0#I1+^Kf?H% zGFa9Ba}*2)>)U+_L*q2+8FoaVJ!@q-ro7wp#Bx&yF+u(G?f2mtNS85osgd%Uf)o?uZed0$Q(mrey)EEw|Sxv-azzLCz~ z1hB(!pYbJw8pglc8N*R;G@!;~(Fx-)?W16)M;*{%faZP-A_ho0lSA|M(MlXtyw5vl z(q)MMVZJREsDv5ei{)=D6pJ^=UTN6Mqg-ORMhX)9*FZ$KwI8g{xt9R^Cs`3c9xW6uZM)3qyDG=6B1&N00(iI^UUz%xNK08sxe zcitfYgs6^X9qN;9L0B1l{99hV`mqTf%^O>)Hz6gM>`9W~inWa42t9q{fB2k}fZyQO z?;3XRA$Z^BxEu7Hq;mijTk;k>fSjdTrUu>`3(sjm)cq?YdnNXKN9~8*k99~m^rLp_ zW8}gJvY<#Dm2hrEv>VhP26nDq?&B1?K>}wVdzS%JJUN67%K2BN89O}~CFhIV2O4d4 ze+dc(@^q5Jy;0?3LvsW6`;knp+!wWGg3Q`A0e$_4J{<;6@+71U02-h5qLQb?`3rZ> zD8I#9R>b#xs2ypKxb#3G2ABZM^C70VE!!OwA26aKt?iLyzMwHo|I(4&oP%!j22_8Vbe*=ZH)o<{$ zH_}%00AO|O&JGBdyK#a{6sn$VYuADekMWo2&NO)U2q8T|%c)WvJ%uES1{N2=IrO>f zN4EfgM65}$LFr)u`K*Txv5yY|+fj0LiTyWo%b7+r9-G;KJWQZ!E0|v1@KO(UwkTFJ zxQgaR4WRsHuPwVbI3vim<1t-ew zwb=P!ErD3jZ^SN4yZzBNmQ6j9oo3fb_-q%Sp-{W4O%q*5khvCWb9L!Qi+TcZJQ={mLIa-q{;4?3G{C z)Vm3o{Qn0u{STCYR}>0>9WkC3S?YW0OaAKvjZ7w&2QvH?f~mXc zpB#UL@XR@(18C{f=ZfU=Dg2@S=-*qOE0#LWc{&*$@Lk*N^krs#+z|}(q~VB?J@WO{ z8Knp~TVI0Wp_#{gBv(}tMO0BKc$kI@fr#S8P#7Rpa4L$4mLkN+ssxK7+ff)Nj1_$N8UbWpt zBEy-u2xq#_IS(~9F@CZ7h0uJoAdlOlrYF~aB-8NVFr`8Blau$}xz5w=u}KFaWCoGx zr|+M_!6O!s3#;qCrOiFl|2JXX> z=OwEB2v%1Iih2147yi2BJ*Ah zw9?B6T3Ovz;?qas0K@=_gaeSmFc?0TyNQVS3=Q31uiLrT^WzY@HiVkG!WMCV*zN%+ z^_&pC74=I6I|UC&@fB%hI(#z}m2?q5JK;}{@nrVLR;@hjty+UzGS@m`m5w80LVe{J z9)7qDhK@^k2^$atn)i?!?WQ(z+h_24=?WcMGxVp9jR?Hd?}QY2H2WS-Xm!#Ry}Tt>ZQH%NtS&aw z{1PAZj+`F}nW43uMMaehzBTA0?;hQ+E`Ow+|8m@uxD~^bqvHK6Rb|TAPp*vi@v>wv z_Z4MsF0}516*fRpV1Q(*xD2?()UntKO)_r===-c^%EJ# z!jbUPIil;M&6V&xF$2SyYWpNqnV$+;8>?YWqttT!-^vqB=rDIJMvH089Jj0QH+f9H zDEGn7A4Yrvm-IcZUTGdOSvmDIxO zN?K(FTKIrUhxA^7o8cCr$Gpq1n8fzIcAZpgBIA{D;`aTqQ{BSZCJlw6#E;^EaxnzS zX~pG^skU=eYC)(nXe&QslXrO%zh<|`87*=8_wO)VpO@;3?H-8N9o(~bwU5!*HT2^O z(@y=sA6_?f`6;P1sBA)2MHBNG1|Dp>gSpKPMtimOpQDXMZ_8P*zPC1o4*ym_lgQdt znXl}4J*6pZ^koxqX8G{s_fKMu`OeZZ#s^N)3`YfOIW%kEH*Z@l@4f$CnY?2@TUCv# z*bG!`xyy@nXZIN5sI+;yP@X0?y<%$}a?j>s!?vK!3`?d4yZ2Sb+e70RIqUnQGl#p> zj%KrAE1wDt6}UoWIOKKs>#Y+Sd#7I%B$Lib8rIDg_sR8fdCbL^tKo2e9coMxz?ytg z6$%?s#lMEpTn0#Sm5~Pi4QZo@KKf$MUL}$0T)QqAiG`2qgmiJvPF6k}=II~Yw4nV_ zf57q0`Rc2A%%r%D9#17d)d|08IJ1pM)2#ujZ>Xca46ES#3#9_b7hz3p^74K2O;c$paLQeEd<&@UPEeFk(% zbV7mZ7)(Hz`Yvw-L|!Yc%n|Aqb6)6QL|W_ySXyEr(MLLSenqgf7d~%D5$vKc+|tT; zsK+gWR+L(lZMWfakx94`?kQV;0B`I%eZS@+o!$@f<)~nIIg}%`;p-NcD#n4s*A1@ms`OExeBNwbFH{T3uPb!+zuKPuT5Kbm8Y8;Nsa(SzjpUW51*1+(iNQto zUH9E@-*H-ltP4M_ZxkI{46O=nH0Y2Kcw#gzbDSXbE4me$7s8c4geea%YDAzC2Rj31 z5cS2{t5QN?%4-#pE2WM#ZMB<>JH_FcZ<;9cjV3Q%_iBvd>)ch<#tW-&#qJniPu=o#C(b%w_(576)fqqO`x|VrB~-ukyXU&knQO?B)k(FHZA_`6 z_|CMx%ZKcj0u^BrR>v||8qJphO8wKws_u;}J-`C(mtNHsejiT@-7}E$Bu}H(EB=O- z$Pt2bucer{;kb2*E3J(ysERN2{+Ro5JdFqE@S{uH$FvO6^&RFjX@x1L9mtoAr*{pv zrr#mRU3KMaR@=Bs@~b1>EEC>6fao|h4xrQHYF$UdU=z|-+Bw-;MNFy&q9A zp(?6D{eGE1A3u8S2QCE!Docb0Xead=*GkUhZtmd9s`u)$=$F(DUAz%-8=W_m>Hv zQsw)5UHe5Ftmmxdpy%;*5?lW!7Y*=Ly(G^P<*zZlQ7Jb)|KIWh9!4%6E_p%8bL9!_ zU+n%}>rK!9xpw`hH5j@7M@ET|LM?d@$c&w*O}Nwlq^2gvpA6Bb#@sK-I9WQ} zCplv);FewChfZPjV{;4v;Q5_9!)DFDfM>5uZlkC5AsHZwoKzd>PWO(z^t zv;v(skFtaRn|xSTF|UC5EzD8U^Vb3zhQ0sUedOO$Cp3)-3o5EuK0*J_S4G2sc33`` z4C13MC^1l*!1?20J$Tq2?v)3R%pf}>81(0~q{j!dnf-r1Cv367BohB0tWaMt)BnFL z`u{J}Q6{)09UWin`c8v(UIruhA=xX$lRJM2DSX6IqrT^U=i~Y%gnnlCSaU*GMe`|j z=>5zgtJFNRg&&pn;qw`RnY08JYYNUEN4HyAe@Px95>U9r|5RJeA_!H>xV0X3e(yZb zeM|G}Cs^{HE zHj$w-1YI3@@6!n7<&JP7q3$cFAFm@-eZb8tko8_{mDVzT+0nCV-<|4O(>j-r4IN*^ zHVzIHw6|&|dy(1faig9@sKA0qA^AQJAIvuw{k80yvFSo1CW1N%PLQjuS<~wt^VHG` z>b_9Nnp%TW->sK9idI3Y*M$^nWckGA&?!dbRA~HMG5g6ektBaQT}wZ8NwyM3T9ixk zHTq&#y+hkJ(C)mf_;vT?#qj?2tcFKf>(9N%6T|nrC(<(!jE(8Fukay8vaxQuOZV51 zgwz&yC2(gxR_?iOtWCx6Y!Nyi@347b%u>EsUb5F2kt&;57C1@XzdFf~=J7DQxEwD5 zHy=xVA7**@@kxSxj{G~vuBPY%(5~+5Z;=)p7<4^|sz(R@J*j+Jem1?44Gw%+{RyUsC@JU`M!}&``7%7G zW#!Qh$A!;c1lnOp{KVUL`=rshY1!dWCsGwhU@UMzplFfxKwq%{K__eQBIJ}8J26f@?#Dm)NsEGoNouDaWqFYRt%aLUIpw00?W0I!awji2(9-?qtBMcIiBD^url6Tv+db|n%p)O$| zkIb-m=bqByyekCRqfIi6;`M9+zUn9Aax5q@5htQh%vq}?CF1GA_af=bnO93#@5o#o z)S8to85h@2grq$;R3Y_0s}QWK?ZT5vGgUsla6gs1-_)cud+R(#;6oD@6j8x~__Evc zCl{pgtZXlz>9IHpMi_Kjb?o!1ZI7OM+C>Nn(K|QL#<-fGJ(LPP0!zFQ7A>O!{BPtlPIF@9EY%mu5aYTgm0c|KQ};4z z8V08z$~bGI+3}}=;bCeQ-x{I?Tzq?~EZIi`>YTma`3rui4~gHe!(dn$wCmAhRq!jW ze?`@6=JZ~8L;{gCt%b#-7t`WkCJLun4NCL* z@qJbt&el>LYi#Ep^$cIHlTll_NoCU36wwp0jXshd!q+)_6Tau-^dRFN{pnZ^{ZyTr z-%C@%773LQ8KhwR&pUmp7jl=7nCOCn7I(;ne?;a^0dmUn!}B*C>XmHN*8~q`_bN`L6h{jjyr#Z-9fIYL%j_uu?pTC)eu0}H;`x@|ICg(d z*8YGxHIj#9lK>{u;F4tEz*JgFum%1v6ri6? z)GlLXK)*TS%06jAsarudx_l+l zW`B33Hs*MUGv?y*(y$TTsDaotg<7Ib{26)LB8D1+&G!%cjuP$KR}9qXM5-Q>ZLafL zD}p>NPEDe_U9(jU&q!0rO{ZQ$9)j?&;_exV)eU>%$4&a zqHZmhMTdJi#D$YRsfo&@9x5jhZFJ{^g&|S*8gHFvd0qQ*`8ae#Xs@f!aUbR{lM0ni z%~w{{AZ9iScSkxJ?!LLPF@+t#j_o5u=UST-P!>s_o-qBgA%2_I!U15ai6C~fA5s91 z9n0o4?jPoOt`1mSCl8GeCn3P)tD^_~)4===6i|llcH*ae?1^CfGrmvNpOEj{6Du=U?cO-KYz)eZwyeg>XmN{{#BD# zi4Wnwwzb|Rz~h3P|61|xpM3`v12@X8KV%x#7$y|q8tEJ2@zs(!k3AgtCnY8?`oFf$ HDWM4f5BtA) literal 0 HcmV?d00001 diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise.ipynb b/self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise.ipynb new file mode 100644 index 0000000000..285c908581 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise.ipynb @@ -0,0 +1,1062 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "81e68768", + "metadata": {}, + "source": [ + "# Vertex AI: Qwik Start" + ] + }, + { + "cell_type": "markdown", + "id": "8f3be9d1", + "metadata": {}, + "source": [ + "## Learning objectives\n", + "\n", + "* Train a TensorFlow model locally in a hosted [**Vertex Notebook**](https://cloud.google.com/vertex-ai/docs/general/notebooks?hl=sv).\n" + ] + }, + { + "cell_type": "markdown", + "id": "c7a746be", + "metadata": {}, + "source": [ + "## Introduction: customer lifetime value (CLV) prediction with BigQuery and TensorFlow on Vertex AI" + ] + }, + { + "cell_type": "markdown", + "id": "76bf82e0", + "metadata": {}, + "source": [ + "In this lab, you use [BigQuery](https://cloud.google.com/bigquery) for data processing and exploratory data analysis and the [Vertex AI](https://cloud.google.com/vertex-ai) platform to train and deploy a custom TensorFlow Regressor model to predict customer lifetime value (CLV). The goal of the lab is to introduce to Vertex AI through a high value real world use case - predictive CLV. You start with a local BigQuery and TensorFlow workflow that you may already be familiar with and progress toward training and deploying your model in the cloud with Vertex AI.\n", + "\n", + "![Vertex AI](./images/vertex-ai-overview.png \"Vertex AI Overview\")\n", + "\n", + "Vertex AI is Google Cloud's next generation, unified platform for machine learning development and the successor to AI Platform announced at Google I/O in May 2021. By developing machine learning solutions on Vertex AI, you can leverage the latest ML pre-built components and AutoML to significantly enhance development productivity, the ability to scale your workflow and decision making with your data, and accelerate time to value." + ] + }, + { + "cell_type": "markdown", + "id": "4fe3b8c6", + "metadata": {}, + "source": [ + "### Predictive CLV: how much monetary value existing customers will bring to the business in the future\n", + "\n", + "Predictive CLV is a high impact ML business use case. CLV is a customer's past value plus their predicted future value. The goal of predictive CLV is to predict how much monetary value a user will bring to the business in a defined future time range based on historical transactions.\n", + "\n", + "By knowing CLV, you can develop positive ROI strategies and make decisions about how much money to invest in acquiring new customers and retaining existing ones to grow revenue and profit.\n", + "\n", + "Once your ML model is a success, you can use the results to identify customers more likely to spend money than the others, and make them respond to your offers and discounts with a greater frequency. These customers, with higher lifetime value, are your main marketing target to increase revenue.\n", + "\n", + "By using the machine learning approach to predict your customers' value you will use in this lab, you can prioritize your next actions, such as the following:\n", + "\n", + "* Decide which customers to target with advertising to increase revenue.\n", + "* Identify which customer segments are most profitable and plan how to move customers from one segment to another.\n", + "\n", + "Your task is to predict the future value for existing customers based on their known transaction history. \n", + "\n", + "![CLV](./images/clv-rfm.svg \"Customer Lifetime Value\") \n", + "Source: [Cloud Architecture Center - Predicting Customer Lifetime Value with AI Platform: training the models](https://cloud.google.com/architecture/clv-prediction-with-offline-training-train)\n", + "\n", + "There is a strong positive correlation between the recency, frequency, and amount of money spent on each purchase each customer makes and their CLV. Consequently, you leverage these features to in your ML model. For this lab, they are defined as:\n", + "\n", + "* **Recency**: The time between the last purchase and today, represented by the distance between the rightmost circle and the vertical dotted line that's labeled \"Now\".\n", + "* **Frequency**: The time between purchases, represented by the distance between the circles on a single line.\n", + "* **Monetary**: The amount of money spent on each purchase, represented by the size of the circle. This amount could be the average order value or the quantity of products that the customer ordered." + ] + }, + { + "cell_type": "markdown", + "id": "d46a1982", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "id": "dc29eb23", + "metadata": {}, + "source": [ + "### Define constants" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd4c2e53", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Add installed library dependencies to Python PATH variable.\n", + "PATH=%env PATH\n", + "%env PATH={PATH}:/home/jupyter/.local/bin" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93ead7a0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Retrieve and set PROJECT_ID and REGION environment variables.\n", + "PROJECT_ID = !(gcloud config get-value core/project)\n", + "PROJECT_ID = PROJECT_ID[0]\n", + "# Replace the value below with your assigned lab region.\n", + "REGION = 'us-central1'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d6d4df6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create a globally unique Google Cloud Storage bucket for artifact storage.\n", + "GCS_BUCKET = f\"{PROJECT_ID}-bucket\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "883ab23c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!gsutil mb -l $REGION gs://$GCS_BUCKET" + ] + }, + { + "cell_type": "markdown", + "id": "8018cc87", + "metadata": {}, + "source": [ + "### Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "412ffc51", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import datetime\n", + "import numpy as np\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from google.cloud import aiplatform" + ] + }, + { + "cell_type": "markdown", + "id": "aecf21cb", + "metadata": {}, + "source": [ + "### Initialize the Vertex Python SDK client" + ] + }, + { + "cell_type": "markdown", + "id": "a301853d", + "metadata": {}, + "source": [ + "Import the Vertex SDK for Python into your Python environment and initialize it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae6029df", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=f\"gs://{GCS_BUCKET}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cf880707", + "metadata": {}, + "source": [ + "## Download and process the lab data into BigQuery" + ] + }, + { + "cell_type": "markdown", + "id": "742ceefd", + "metadata": {}, + "source": [ + "### Dataset\n", + "\n", + "In this lab, you use the publicly available [Online Retail data set](https://archive.ics.uci.edu/ml/datasets/online+retail) from the UCI Machine Learning Repository. This dataset contains 541,909 transnational customer transactions occuring between (YYYY-MM-DD) 2010-12-01 and 2011-12-09 for a UK-based and registered non-store retailer. The company primarily sells unique all-occasion gifts. Many of the company's customers are wholesalers.\n", + "\n", + "**Citation** \n", + "Dua, D. and Karra Taniskidou, E. (2017). UCI Machine Learning Repository http://archive.ics.uci.edu/ml. Irvine, CA: University of California, School of Information and Computer Science.\n", + "\n", + "This lab is also inspired by the Google Cloud Architect Guide Series [Predicting Customer Lifetime Value with AI Platform: introduction](https://cloud.google.com/architecture/clv-prediction-with-offline-training-intro)." + ] + }, + { + "cell_type": "markdown", + "id": "9c7d9d01", + "metadata": {}, + "source": [ + "### Data ingestion" + ] + }, + { + "cell_type": "markdown", + "id": "df4efbb9", + "metadata": {}, + "source": [ + "Execute the command below to ingest the lab data from the UCI Machine Learning repository into `Cloud Storage` and then upload to `BigQuery` for data processing. The data ingestion and processing scripts are available under the `utils` folder in the lab directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7720d05e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# BigQuery constants. Please leave these unchanged.\n", + "BQ_DATASET_NAME=\"online_retail\"\n", + "BQ_RAW_TABLE_NAME=\"online_retail_clv_raw\"\n", + "BQ_CLEAN_TABLE_NAME=\"online_retail_clv_clean\"\n", + "BQ_ML_TABLE_NAME=\"online_retail_clv_ml\"\n", + "BQ_URI=f\"bq://{PROJECT_ID}.{BQ_DATASET_NAME}.{BQ_ML_TABLE_NAME}\"" + ] + }, + { + "cell_type": "markdown", + "id": "557df7b2", + "metadata": {}, + "source": [ + "**Note**: This Python script will take about 2-3 min to download and process the lab data file. Follow along with logging output in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a42e87bc", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!python utils/data_download.py \\\n", + " --PROJECT_ID={PROJECT_ID} \\\n", + " --GCS_BUCKET={GCS_BUCKET} \\\n", + " --BQ_RAW_TABLE_NAME={BQ_RAW_TABLE_NAME} \\\n", + " --BQ_CLEAN_TABLE_NAME={BQ_CLEAN_TABLE_NAME} \\\n", + " --BQ_ML_TABLE_NAME={BQ_ML_TABLE_NAME} \\\n", + " --URL=\"https://archive.ics.uci.edu/ml/machine-learning-databases/00352/Online Retail.xlsx\"" + ] + }, + { + "cell_type": "markdown", + "id": "6ca57a9f", + "metadata": {}, + "source": [ + "### Data processing" + ] + }, + { + "cell_type": "markdown", + "id": "c7293fc2", + "metadata": {}, + "source": [ + "As is the case with many real-world datasets, the lab dataset required some cleanup for you to utilize this historical customer transaction data for predictive CLV.\n", + "\n", + "The following changes were applied:\n", + "\n", + "* Keep only records that have a Customer ID.\n", + "* Aggregate transactions by day from Invoices.\n", + "* Keep only records that have positive order quantities and monetary values.\n", + "* Aggregate transactions by Customer ID and compute recency, frequency, monetary features as well as the prediction target.\n", + "\n", + "**Features**:\n", + "- `customer_country` (CATEGORICAL): customer purchase country.\n", + "- `n_purchases` (NUMERIC): number of purchases made in feature window. (frequency)\n", + "- `avg_purchase_size` (NUMERIC): average unit purchase count in feature window. (monetary)\n", + "- `avg_purchase_revenue` (NUMERIC): average GBP purchase amount in in feature window. (monetary)\n", + "- `customer_age` (NUMERIC): days from first purchase in feature window.\n", + "- `days_since_last_purchase` (NUMERIC): days from the most recent purchase in the feature window. (recency) \n", + "\n", + "**Target**: \n", + "- `target_monetary_value_3M` (NUMERIC): customer revenue from the entire study window including feature and prediction windows.\n", + "\n", + "Note: This lab demonstrates a simple way to use a DNN predict customer 3-month ahead CLV monetary value based solely on the available dataset historical transaction history. Additional factors to consider in practice when using CLV to inform interventions include customer acquisition costs, profit margins, and discount rates to arrive at the present value of future customer cash flows. One of a DNN's benefits over traditional probabilistic modeling approaches is their ability to incorporate additional categorical and unstructured features; this is a great feature engineering opportunity to explore beyond this lab which just explores the RFM numeric features." + ] + }, + { + "cell_type": "markdown", + "id": "402abff6", + "metadata": {}, + "source": [ + "## Exploratory data analysis (EDA) in BigQuery" + ] + }, + { + "cell_type": "markdown", + "id": "f4fa4d6c", + "metadata": {}, + "source": [ + "Below you use BigQuery from this notebook to do exploratory data analysis to get to know this dataset and identify opportunities for data cleanup and feature engineering." + ] + }, + { + "cell_type": "markdown", + "id": "91c50cbe", + "metadata": {}, + "source": [ + "### Recency: how recently have customers purchased?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50110392", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery recency\n", + "\n", + "SELECT \n", + " days_since_last_purchase\n", + "FROM \n", + " `online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75edeba1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "recency.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89bc69b4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "recency.hist(bins=100);" + ] + }, + { + "cell_type": "markdown", + "id": "e857fb43", + "metadata": {}, + "source": [ + "From the chart, there are clearly a few different customer groups here such as loyal customers that have made purchases in the last few days as well as inactive customers that have not purchased in 250+ days. Using CLV predictions and insights, you can strategize on marketing and promotional interventions to improve customer purchase recency and re-active dormant customers." + ] + }, + { + "cell_type": "markdown", + "id": "1d4d8860", + "metadata": {}, + "source": [ + "### Frequency: how often are customers purchasing?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34402015", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery frequency\n", + "\n", + "SELECT\n", + " n_purchases\n", + "FROM\n", + " `online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc1fd5c2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "frequency.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cbeac7e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "frequency.hist(bins=100);" + ] + }, + { + "cell_type": "markdown", + "id": "00c933f5", + "metadata": {}, + "source": [ + "From the chart and quantiles, you can see that half of the customers have less than or equal to only 2 purchases. You can also tell from the average purchases > median purchases and max purchases of 81 that there are customers, likely wholesalers, who have made significantly more purchases. This should have you already thinking about feature engineering opportunities such as bucketizing purchases and removing or clipping outlier customers. You can also explore alternative modeling strategies for CLV on new customers who have only made 1 purchase as the approach demonstrated in this lab will perform better on customers with more relationship transactional history. " + ] + }, + { + "cell_type": "markdown", + "id": "00c0c043", + "metadata": {}, + "source": [ + "### Monetary: how much are customers spending?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b8d00ea", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery monetary\n", + "\n", + "SELECT\n", + " target_monetary_value_3M\n", + "FROM\n", + "`online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "636a5010", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "monetary.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08b651c5", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "monetary['target_monetary_value_3M'].plot(kind='box', title=\"Target Monetary Value 3M: wide range, long right tail distribution\", grid=True);" + ] + }, + { + "cell_type": "markdown", + "id": "7bc60b98", + "metadata": {}, + "source": [ + "From the chart and summary statistics, you can see there is a wide range in customer monetary value ranging from 2.90 to 268,478 GBP. Looking at the quantiles, it is clear there are a few outlier customers whose monetary value is greater than 3 standard deviations from the mean. With this small dataset, it is recommended to remove these outlier customer values to treat separately, change your model's loss function to be more resistant to outliers, log the target feature, or clip their values to a maximum threshold. You should also be revisiting your CLV business requirements to see if binning customer monetary value and reframing this as a ML classification problem would suit your needs." + ] + }, + { + "cell_type": "markdown", + "id": "02e553fd", + "metadata": {}, + "source": [ + "### Establish a simple model performance baseline" + ] + }, + { + "cell_type": "markdown", + "id": "08221502", + "metadata": {}, + "source": [ + "In order to evaluate the performance of your custom TensorFlow DNN Regressor model you will build in the next steps, it is a ML best practice to establish a simple performance baseline. Below is a simple SQL baseline that multiplies a customer's average purchase spent compounded by their daily purchase rate and computes standard regression metrics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf088864", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery\n", + "\n", + "WITH\n", + " day_intervals AS (\n", + " SELECT\n", + " customer_id,\n", + " DATE_DIFF(DATE('2011-12-01'), DATE('2011-09-01'), DAY) AS target_days,\n", + " DATE_DIFF(DATE('2011-09-01'), MIN(order_date), DAY) AS feature_days,\n", + " FROM\n", + " `online_retail.online_retail_clv_clean`\n", + " GROUP BY\n", + " customer_id\n", + " ),\n", + " \n", + " predicted_clv AS (\n", + " SELECT\n", + " customer_id,\n", + " AVG(avg_purchase_revenue) * (COUNT(n_purchases) * (1 + SAFE_DIVIDE(COUNT(target_days),COUNT(feature_days)))) AS predicted_monetary_value_3M,\n", + " SUM(target_monetary_value_3M) AS target_monetary_value_3M\n", + " FROM\n", + " `online_retail.online_retail_clv_ml`\n", + " LEFT JOIN day_intervals USING(customer_id)\n", + " GROUP BY\n", + " customer_id\n", + " )\n", + "\n", + "# Calculate overall baseline regression metrics.\n", + "SELECT\n", + " ROUND(AVG(ABS(predicted_monetary_value_3M - target_monetary_value_3M)), 2) AS MAE,\n", + " ROUND(AVG(POW(predicted_monetary_value_3M - target_monetary_value_3M, 2)), 2) AS MSE,\n", + " ROUND(SQRT(AVG(POW(predicted_monetary_value_3M - target_monetary_value_3M, 2))), 2) AS RMSE\n", + "FROM\n", + " predicted_clv" + ] + }, + { + "cell_type": "markdown", + "id": "956ac010", + "metadata": {}, + "source": [ + "These baseline results provide further support for the strong impact of outliers. The extremely high MSE comes from the exponential penalty applied to missed predictions and the magnitude of error on a few predictions.\n", + "\n", + "Next, you should look to plot the baseline results to get a sense of opportunity areas for you ML model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e14ff67", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery baseline\n", + "\n", + "WITH\n", + " day_intervals AS (\n", + " SELECT\n", + " customer_id,\n", + " DATE_DIFF(DATE('2011-12-01'), DATE('2011-09-01'), DAY) AS target_days,\n", + " DATE_DIFF(DATE('2011-09-01'), MIN(order_date), DAY) AS feature_days,\n", + " FROM\n", + " `online_retail.online_retail_clv_clean`\n", + " GROUP BY\n", + " customer_id\n", + " ),\n", + " \n", + " predicted_clv AS (\n", + " SELECT\n", + " customer_id,\n", + " AVG(avg_purchase_revenue) * (COUNT(n_purchases) * (1 + SAFE_DIVIDE(COUNT(target_days),COUNT(feature_days)))) AS predicted_monetary_value_3M,\n", + " SUM(target_monetary_value_3M) AS target_monetary_value_3M\n", + " FROM\n", + " `online_retail.online_retail_clv_ml`\n", + " INNER JOIN day_intervals USING(customer_id)\n", + " GROUP BY\n", + " customer_id\n", + " )\n", + "\n", + "SELECT\n", + " *\n", + "FROM\n", + " predicted_clv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afda73aa", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "baseline.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a543c10", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "ax = baseline.plot(kind='scatter',\n", + " x='predicted_monetary_value_3M', \n", + " y='target_monetary_value_3M',\n", + " title='Actual vs. Predicted customer 3-month monetary value',\n", + " figsize=(5,5),\n", + " grid=True)\n", + "\n", + "lims = [\n", + " np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes\n", + " np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "# now plot both limits against eachother\n", + "ax.plot(lims, lims, 'k-', alpha=0.5, zorder=0)\n", + "ax.set_aspect('equal')\n", + "ax.set_xlim(lims)\n", + "ax.set_ylim(lims);" + ] + }, + { + "cell_type": "markdown", + "id": "0d53ad3a", + "metadata": {}, + "source": [ + "## Train a TensorFlow model locally" + ] + }, + { + "cell_type": "markdown", + "id": "b3658b32", + "metadata": {}, + "source": [ + "Now that you have a simple baseline to benchmark your performance against, train a TensorFlow Regressor to predict CLV." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c45e2feb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery\n", + "\n", + "SELECT data_split, COUNT(*)\n", + "FROM `online_retail.online_retail_clv_ml`\n", + "GROUP BY data_split" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7e2994a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%bigquery clv\n", + "\n", + "SELECT *\n", + "FROM `online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80339852", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "clv_train = clv.loc[clv.data_split == 'TRAIN', :]\n", + "clv_dev = clv.loc[clv.data_split == 'VALIDATE', :]\n", + "clv_test = clv.loc[clv.data_split == 'TEST', :]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a15e9683", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Model training constants.\n", + "# Virtual epochs design pattern:\n", + "# https://medium.com/google-cloud/ml-design-pattern-3-virtual-epochs-f842296de730\n", + "N_TRAIN_EXAMPLES = 2638\n", + "STOP_POINT = 20.0\n", + "TOTAL_TRAIN_EXAMPLES = int(STOP_POINT * N_TRAIN_EXAMPLES)\n", + "BATCH_SIZE = 32\n", + "N_CHECKPOINTS = 10\n", + "STEPS_PER_EPOCH = (TOTAL_TRAIN_EXAMPLES // (BATCH_SIZE*N_CHECKPOINTS))\n", + "\n", + "NUMERIC_FEATURES = [\n", + " \"n_purchases\",\n", + " \"avg_purchase_size\",\n", + " \"avg_purchase_revenue\",\n", + " \"customer_age\",\n", + " \"days_since_last_purchase\",\n", + "]\n", + "\n", + "LABEL = \"target_monetary_value_3M\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "627cc31a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def df_dataset(df):\n", + " \"\"\"Transform Pandas Dataframe to TensorFlow Dataset.\"\"\"\n", + " return tf.data.Dataset.from_tensor_slices((df[NUMERIC_FEATURES].to_dict('list'), df[LABEL].values))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b0744b6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "trainds = df_dataset(clv_train).prefetch(1).batch(BATCH_SIZE).repeat()\n", + "devds = df_dataset(clv_dev).prefetch(1).batch(BATCH_SIZE)\n", + "testds = df_dataset(clv_test).prefetch(1).batch(BATCH_SIZE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9459079", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from keras.metrics import RootMeanSquaredError\n", + "\n", + "def rmse(y_true, y_pred):\n", + " \"\"\"Custom RMSE regression metric.\"\"\"\n", + " return tf.sqrt(tf.reduce_mean(tf.square(y_pred - y_true)))\n", + "\n", + "\n", + "def build_model():\n", + " \"\"\"Build and compile a TensorFlow Keras Regressor.\"\"\"\n", + "\n", + " # Define input feature tensors and input layers.\n", + " input_layers = {\n", + " feature: tf.keras.layers.Input(name=feature, shape=(1,), dtype=tf.float32) \n", + " for feature in NUMERIC_FEATURES\n", + " }\n", + "\n", + " inputs = tf.keras.layers.concatenate([\n", + " tf.keras.layers.Normalization(axis=-1)(input_layers[feature]) \n", + " for feature in NUMERIC_FEATURES\n", + " ])\n", + "\n", + " d1 = tf.keras.layers.Dense(256, activation=tf.nn.relu, name='d1')(inputs)\n", + " d2 = tf.keras.layers.Dropout(0.2, name='d2')(d1)\n", + "\n", + " # Note: the single neuron output for regression.\n", + " output = tf.keras.layers.Dense(1, name='output')(d2)\n", + "\n", + " model = tf.keras.Model(input_layers, output, name='online-retail-clv')\n", + "\n", + " optimizer = tf.keras.optimizers.Adam(0.001)\n", + "\n", + " # Note: MAE loss is more resistant to outliers than MSE.\n", + " model.compile(loss=tf.keras.losses.MAE,\n", + " optimizer=optimizer,\n", + " metrics=['mae', 'mse', RootMeanSquaredError()])\n", + "\n", + " return model\n", + "\n", + "model = build_model()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8601ff5f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(model, show_shapes=False, rankdir=\"LR\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "354206ee", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "tensorboard_callback = tf.keras.callbacks.TensorBoard(\n", + " log_dir='./local-training/tensorboard',\n", + " histogram_freq=1)\n", + "\n", + "earlystopping_callback = tf.keras.callbacks.EarlyStopping(patience=1)\n", + "\n", + "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n", + " filepath='./local-training/checkpoints/my_model_weights.weights.h5',\n", + " save_weights_only=True,\n", + " monitor='val_loss',\n", + " mode='min')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "730181fb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "history = model.fit(trainds,\n", + " validation_data=devds,\n", + " steps_per_epoch=STEPS_PER_EPOCH,\n", + " epochs=N_CHECKPOINTS,\n", + " callbacks=[tensorboard_callback,\n", + " earlystopping_callback,\n", + " checkpoint_callback])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2594d084", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "LOSS_COLS = [\"loss\", \"val_loss\"]\n", + "\n", + "pd.DataFrame(history.history)[LOSS_COLS].plot();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b71775db", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "train_pred = model.predict(df_dataset(clv_train).prefetch(1).batch(BATCH_SIZE))\n", + "dev_pred = model.predict(devds)\n", + "test_pred = model.predict(testds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b6eceb1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "train_results = pd.DataFrame({'actual': clv_train['target_monetary_value_3M'].to_numpy(), 'predicted': np.squeeze(train_pred)}, columns=['actual', 'predicted'])\n", + "dev_results = pd.DataFrame({'actual': clv_dev['target_monetary_value_3M'].to_numpy(), 'predicted': np.squeeze(dev_pred)}, columns=['actual', 'predicted'])\n", + "test_results = pd.DataFrame({'actual': clv_test['target_monetary_value_3M'].to_numpy(), 'predicted': np.squeeze(test_pred)}, columns=['actual', 'predicted'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4659dd09", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Model prediction calibration plots.\n", + "fig, (train_ax, dev_ax, test_ax) = plt.subplots(1, 3, figsize=(15,15))\n", + "\n", + "train_results.plot(kind='scatter',\n", + " x='predicted',\n", + " y='actual',\n", + " title='Train: act vs. pred customer 3M monetary value',\n", + " grid=True,\n", + " ax=train_ax)\n", + "\n", + "train_lims = [\n", + " np.min([train_ax.get_xlim(), train_ax.get_ylim()]), # min of both axes\n", + " np.max([train_ax.get_xlim(), train_ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "train_ax.plot(train_lims, train_lims, 'k-', alpha=0.5, zorder=0)\n", + "train_ax.set_aspect('equal')\n", + "train_ax.set_xlim(train_lims)\n", + "train_ax.set_ylim(train_lims)\n", + "\n", + "dev_results.plot(kind='scatter',\n", + " x='predicted',\n", + " y='actual',\n", + " title='Dev: act vs. pred customer 3M monetary value',\n", + " grid=True,\n", + " ax=dev_ax)\n", + "\n", + "dev_lims = [\n", + " np.min([dev_ax.get_xlim(), dev_ax.get_ylim()]), # min of both axes\n", + " np.max([dev_ax.get_xlim(), dev_ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "dev_ax.plot(dev_lims, dev_lims, 'k-', alpha=0.5, zorder=0)\n", + "dev_ax.set_aspect('equal')\n", + "dev_ax.set_xlim(dev_lims)\n", + "dev_ax.set_ylim(dev_lims)\n", + "\n", + "test_results.plot(kind='scatter',\n", + " x='predicted',\n", + " y='actual',\n", + " title='Test: act vs. pred customer 3M monetary value',\n", + " grid=True,\n", + " ax=test_ax)\n", + "\n", + "test_lims = [\n", + " np.min([test_ax.get_xlim(), test_ax.get_ylim()]), # min of both axes\n", + " np.max([test_ax.get_xlim(), test_ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "test_ax.plot(test_lims, test_lims, 'k-', alpha=0.5, zorder=0)\n", + "test_ax.set_aspect('equal')\n", + "test_ax.set_xlim(test_lims)\n", + "test_ax.set_ylim(test_lims);" + ] + }, + { + "cell_type": "markdown", + "id": "2a5f1582", + "metadata": {}, + "source": [ + "You have trained a model better than your baseline. As indicated in the charts above, there is still additional feature engineering and data cleaning opportunities to improve your model's performance on customers with CLV. Some options include handling these customers as a separate prediction task, applying a log transformation to your target, clipping their value or dropping these customers all together to improve model performance.\n" + ] + }, + { + "cell_type": "markdown", + "id": "2fc312cf", + "metadata": {}, + "source": [ + "## Next steps" + ] + }, + { + "cell_type": "markdown", + "id": "30ab0ae3", + "metadata": {}, + "source": [ + "Congratulations! In this lab, you walked through a machine learning experimentation workflow using Google Cloud's BigQuery for data storage and analysis and Vertex AI machine learning services to train and deploy a TensorFlow model to predict customer lifetime value" + ] + }, + { + "cell_type": "markdown", + "id": "0749f152", + "metadata": {}, + "source": [ + "## License" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d2cfd56", + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright 2021 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + } + ], + "metadata": { + "environment": { + "kernel": "conda-base-py", + "name": "workbench-notebooks.m125", + "type": "gcloud", + "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m125" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel) (Local)", + "language": "python", + "name": "conda-base-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise_long.ipynb b/self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise_long.ipynb new file mode 100644 index 0000000000..c31ce960b6 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/lab_exercise_long.ipynb @@ -0,0 +1,1931 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "81e68768", + "metadata": {}, + "source": [ + "# Vertex AI: Qwik Start" + ] + }, + { + "cell_type": "markdown", + "id": "8f3be9d1", + "metadata": {}, + "source": [ + "## Learning objectives\n", + "\n", + "* Train a TensorFlow model locally in a hosted [**Vertex Notebook**](https://cloud.google.com/vertex-ai/docs/general/notebooks?hl=sv).\n", + "* Create a [**managed Tabular dataset**](https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets?hl=sv) artifact for experiment tracking.\n", + "* Containerize your training code with [**Cloud Build**](https://cloud.google.com/build) and push it to [**Google Cloud Artifact Registry**](https://cloud.google.com/artifact-registry).\n", + "* Run a [**Vertex AI custom training job**](https://cloud.google.com/vertex-ai/docs/training/custom-training) with your custom model container.\n", + "* Use [**Vertex TensorBoard**](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview) to visualize model performance.\n", + "* Deploy your trained model to a [**Vertex Online Prediction Endpoint**](https://cloud.google.com/vertex-ai/docs/predictions/getting-predictions) for serving predictions.\n", + "* Request an online prediction and explanation and see the response." + ] + }, + { + "cell_type": "markdown", + "id": "c7a746be", + "metadata": {}, + "source": [ + "## Introduction: customer lifetime value (CLV) prediction with BigQuery and TensorFlow on Vertex AI" + ] + }, + { + "cell_type": "markdown", + "id": "76bf82e0", + "metadata": {}, + "source": [ + "In this lab, you use [BigQuery](https://cloud.google.com/bigquery) for data processing and exploratory data analysis and the [Vertex AI](https://cloud.google.com/vertex-ai) platform to train and deploy a custom TensorFlow Regressor model to predict customer lifetime value (CLV). The goal of the lab is to introduce to Vertex AI through a high value real world use case - predictive CLV. You start with a local BigQuery and TensorFlow workflow that you may already be familiar with and progress toward training and deploying your model in the cloud with Vertex AI.\n", + "\n", + "![Vertex AI](./images/vertex-ai-overview.png \"Vertex AI Overview\")\n", + "\n", + "Vertex AI is Google Cloud's next generation, unified platform for machine learning development and the successor to AI Platform announced at Google I/O in May 2021. By developing machine learning solutions on Vertex AI, you can leverage the latest ML pre-built components and AutoML to significantly enhance development productivity, the ability to scale your workflow and decision making with your data, and accelerate time to value." + ] + }, + { + "cell_type": "markdown", + "id": "4fe3b8c6", + "metadata": {}, + "source": [ + "### Predictive CLV: how much monetary value existing customers will bring to the business in the future\n", + "\n", + "Predictive CLV is a high impact ML business use case. CLV is a customer's past value plus their predicted future value. The goal of predictive CLV is to predict how much monetary value a user will bring to the business in a defined future time range based on historical transactions.\n", + "\n", + "By knowing CLV, you can develop positive ROI strategies and make decisions about how much money to invest in acquiring new customers and retaining existing ones to grow revenue and profit.\n", + "\n", + "Once your ML model is a success, you can use the results to identify customers more likely to spend money than the others, and make them respond to your offers and discounts with a greater frequency. These customers, with higher lifetime value, are your main marketing target to increase revenue.\n", + "\n", + "By using the machine learning approach to predict your customers' value you will use in this lab, you can prioritize your next actions, such as the following:\n", + "\n", + "* Decide which customers to target with advertising to increase revenue.\n", + "* Identify which customer segments are most profitable and plan how to move customers from one segment to another.\n", + "\n", + "Your task is to predict the future value for existing customers based on their known transaction history. \n", + "\n", + "![CLV](./images/clv-rfm.svg \"Customer Lifetime Value\") \n", + "Source: [Cloud Architecture Center - Predicting Customer Lifetime Value with AI Platform: training the models](https://cloud.google.com/architecture/clv-prediction-with-offline-training-train)\n", + "\n", + "There is a strong positive correlation between the recency, frequency, and amount of money spent on each purchase each customer makes and their CLV. Consequently, you leverage these features to in your ML model. For this lab, they are defined as:\n", + "\n", + "* **Recency**: The time between the last purchase and today, represented by the distance between the rightmost circle and the vertical dotted line that's labeled \"Now\".\n", + "* **Frequency**: The time between purchases, represented by the distance between the circles on a single line.\n", + "* **Monetary**: The amount of money spent on each purchase, represented by the size of the circle. This amount could be the average order value or the quantity of products that the customer ordered." + ] + }, + { + "cell_type": "markdown", + "id": "d46a1982", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "id": "dc29eb23", + "metadata": {}, + "source": [ + "### Define constants" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd4c2e53", + "metadata": {}, + "outputs": [], + "source": [ + "# Add installed library dependencies to Python PATH variable.\n", + "PATH=%env PATH\n", + "%env PATH={PATH}:/home/jupyter/.local/bin" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93ead7a0", + "metadata": {}, + "outputs": [], + "source": [ + "# Retrieve and set PROJECT_ID and REGION environment variables.\n", + "PROJECT_ID = !(gcloud config get-value core/project)\n", + "PROJECT_ID = PROJECT_ID[0]\n", + "REGION = 'us-central1' # Replace the region with the region mentioned in your lab manual.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d6d4df6", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a globally unique Google Cloud Storage bucket for artifact storage.\n", + "GCS_BUCKET = f\"{PROJECT_ID}-bucket\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "883ab23c", + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil mb -l $REGION gs://$GCS_BUCKET" + ] + }, + { + "cell_type": "markdown", + "id": "8018cc87", + "metadata": {}, + "source": [ + "### Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "412ffc51", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import datetime\n", + "import numpy as np\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from google.cloud import aiplatform" + ] + }, + { + "cell_type": "markdown", + "id": "aecf21cb", + "metadata": {}, + "source": [ + "### Initialize the Vertex Python SDK client" + ] + }, + { + "cell_type": "markdown", + "id": "a301853d", + "metadata": {}, + "source": [ + "Import the Vertex SDK for Python into your Python environment and initialize it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae6029df", + "metadata": {}, + "outputs": [], + "source": [ + "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=f\"gs://{GCS_BUCKET}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cf880707", + "metadata": {}, + "source": [ + "## Download and process the lab data into BigQuery" + ] + }, + { + "cell_type": "markdown", + "id": "742ceefd", + "metadata": {}, + "source": [ + "### Dataset\n", + "\n", + "In this lab, you use the publicly available [Online Retail data set](https://archive.ics.uci.edu/ml/datasets/online+retail) from the UCI Machine Learning Repository. This dataset contains 541,909 transnational customer transactions occuring between (YYYY-MM-DD) 2010-12-01 and 2011-12-09 for a UK-based and registered non-store retailer. The company primarily sells unique all-occasion gifts. Many of the company's customers are wholesalers.\n", + "\n", + "**Citation** \n", + "Dua, D. and Karra Taniskidou, E. (2017). UCI Machine Learning Repository http://archive.ics.uci.edu/ml. Irvine, CA: University of California, School of Information and Computer Science.\n", + "\n", + "This lab is also inspired by the Google Cloud Architect Guide Series [Predicting Customer Lifetime Value with AI Platform: introduction](https://cloud.google.com/architecture/clv-prediction-with-offline-training-intro)." + ] + }, + { + "cell_type": "markdown", + "id": "9c7d9d01", + "metadata": {}, + "source": [ + "### Data ingestion" + ] + }, + { + "cell_type": "markdown", + "id": "df4efbb9", + "metadata": {}, + "source": [ + "Execute the command below to ingest the lab data from the UCI Machine Learning repository into `Cloud Storage` and then upload to `BigQuery` for data processing. The data ingestion and processing scripts are available under the `utils` folder in the lab directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7720d05e", + "metadata": {}, + "outputs": [], + "source": [ + "# BigQuery constants. Please leave these unchanged.\n", + "BQ_DATASET_NAME=\"online_retail\"\n", + "BQ_RAW_TABLE_NAME=\"online_retail_clv_raw\"\n", + "BQ_CLEAN_TABLE_NAME=\"online_retail_clv_clean\"\n", + "BQ_ML_TABLE_NAME=\"online_retail_clv_ml\"\n", + "BQ_URI=f\"bq://{PROJECT_ID}.{BQ_DATASET_NAME}.{BQ_ML_TABLE_NAME}\"" + ] + }, + { + "cell_type": "markdown", + "id": "557df7b2", + "metadata": {}, + "source": [ + "**Note**: This Python script will take about 2-3 min to download and process the lab data file. Follow along with logging output in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a42e87bc", + "metadata": {}, + "outputs": [], + "source": [ + "!python utils/data_download.py \\\n", + " --PROJECT_ID={PROJECT_ID} \\\n", + " --GCS_BUCKET={GCS_BUCKET} \\\n", + " --BQ_RAW_TABLE_NAME={BQ_RAW_TABLE_NAME} \\\n", + " --BQ_CLEAN_TABLE_NAME={BQ_CLEAN_TABLE_NAME} \\\n", + " --BQ_ML_TABLE_NAME={BQ_ML_TABLE_NAME} \\\n", + " --URL=\"https://archive.ics.uci.edu/ml/machine-learning-databases/00352/Online Retail.xlsx\"" + ] + }, + { + "cell_type": "markdown", + "id": "6ca57a9f", + "metadata": {}, + "source": [ + "### Data processing" + ] + }, + { + "cell_type": "markdown", + "id": "c7293fc2", + "metadata": {}, + "source": [ + "As is the case with many real-world datasets, the lab dataset required some cleanup for you to utilize this historical customer transaction data for predictive CLV.\n", + "\n", + "The following changes were applied:\n", + "\n", + "* Keep only records that have a Customer ID.\n", + "* Aggregate transactions by day from Invoices.\n", + "* Keep only records that have positive order quantities and monetary values.\n", + "* Aggregate transactions by Customer ID and compute recency, frequency, monetary features as well as the prediction target.\n", + "\n", + "**Features**:\n", + "- `customer_country` (CATEGORICAL): customer purchase country.\n", + "- `n_purchases` (NUMERIC): number of purchases made in feature window. (frequency)\n", + "- `avg_purchase_size` (NUMERIC): average unit purchase count in feature window. (monetary)\n", + "- `avg_purchase_revenue` (NUMERIC): average GBP purchase amount in in feature window. (monetary)\n", + "- `customer_age` (NUMERIC): days from first purchase in feature window.\n", + "- `days_since_last_purchase` (NUMERIC): days from the most recent purchase in the feature window. (recency) \n", + "\n", + "**Target**: \n", + "- `target_monetary_value_3M` (NUMERIC): customer revenue from the entire study window including feature and prediction windows.\n", + "\n", + "Note: This lab demonstrates a simple way to use a DNN predict customer 3-month ahead CLV monetary value based solely on the available dataset historical transaction history. Additional factors to consider in practice when using CLV to inform interventions include customer acquisition costs, profit margins, and discount rates to arrive at the present value of future customer cash flows. One of a DNN's benefits over traditional probabilistic modeling approaches is their ability to incorporate additional categorical and unstructured features; this is a great feature engineering opportunity to explore beyond this lab which just explores the RFM numeric features." + ] + }, + { + "cell_type": "markdown", + "id": "402abff6", + "metadata": {}, + "source": [ + "## Exploratory data analysis (EDA) in BigQuery" + ] + }, + { + "cell_type": "markdown", + "id": "f4fa4d6c", + "metadata": {}, + "source": [ + "Below you use BigQuery from this notebook to do exploratory data analysis to get to know this dataset and identify opportunities for data cleanup and feature engineering." + ] + }, + { + "cell_type": "markdown", + "id": "91c50cbe", + "metadata": {}, + "source": [ + "### Recency: how recently have customers purchased?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50110392", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery recency\n", + "\n", + "SELECT \n", + " days_since_last_purchase\n", + "FROM \n", + " `online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75edeba1", + "metadata": {}, + "outputs": [], + "source": [ + "recency.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89bc69b4", + "metadata": {}, + "outputs": [], + "source": [ + "recency.hist(bins=100);" + ] + }, + { + "cell_type": "markdown", + "id": "e857fb43", + "metadata": {}, + "source": [ + "From the chart, there are clearly a few different customer groups here such as loyal customers that have made purchases in the last few days as well as inactive customers that have not purchased in 250+ days. Using CLV predictions and insights, you can strategize on marketing and promotional interventions to improve customer purchase recency and re-active dormant customers." + ] + }, + { + "cell_type": "markdown", + "id": "1d4d8860", + "metadata": {}, + "source": [ + "### Frequency: how often are customers purchasing?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34402015", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery frequency\n", + "\n", + "SELECT\n", + " n_purchases\n", + "FROM\n", + " `online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc1fd5c2", + "metadata": {}, + "outputs": [], + "source": [ + "frequency.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cbeac7e", + "metadata": {}, + "outputs": [], + "source": [ + "frequency.hist(bins=100);" + ] + }, + { + "cell_type": "markdown", + "id": "00c933f5", + "metadata": {}, + "source": [ + "From the chart and quantiles, you can see that half of the customers have less than or equal to only 2 purchases. You can also tell from the average purchases > median purchases and max purchases of 81 that there are customers, likely wholesalers, who have made significantly more purchases. This should have you already thinking about feature engineering opportunities such as bucketizing purchases and removing or clipping outlier customers. You can also explore alternative modeling strategies for CLV on new customers who have only made 1 purchase as the approach demonstrated in this lab will perform better on customers with more relationship transactional history. " + ] + }, + { + "cell_type": "markdown", + "id": "00c0c043", + "metadata": {}, + "source": [ + "### Monetary: how much are customers spending?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b8d00ea", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery monetary\n", + "\n", + "SELECT\n", + " target_monetary_value_3M\n", + "FROM\n", + "`online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "636a5010", + "metadata": {}, + "outputs": [], + "source": [ + "monetary.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08b651c5", + "metadata": {}, + "outputs": [], + "source": [ + "monetary['target_monetary_value_3M'].plot(kind='box', title=\"Target Monetary Value 3M: wide range, long right tail distribution\", grid=True);" + ] + }, + { + "cell_type": "markdown", + "id": "7bc60b98", + "metadata": {}, + "source": [ + "From the chart and summary statistics, you can see there is a wide range in customer monetary value ranging from 2.90 to 268,478 GBP. Looking at the quantiles, it is clear there are a few outlier customers whose monetary value is greater than 3 standard deviations from the mean. With this small dataset, it is recommended to remove these outlier customer values to treat separately, change your model's loss function to be more resistant to outliers, log the target feature, or clip their values to a maximum threshold. You should also be revisiting your CLV business requirements to see if binning customer monetary value and reframing this as a ML classification problem would suit your needs." + ] + }, + { + "cell_type": "markdown", + "id": "02e553fd", + "metadata": {}, + "source": [ + "### Establish a simple model performance baseline" + ] + }, + { + "cell_type": "markdown", + "id": "08221502", + "metadata": {}, + "source": [ + "In order to evaluate the performance of your custom TensorFlow DNN Regressor model you will build in the next steps, it is a ML best practice to establish a simple performance baseline. Below is a simple SQL baseline that multiplies a customer's average purchase spent compounded by their daily purchase rate and computes standard regression metrics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf088864", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "\n", + "WITH\n", + " day_intervals AS (\n", + " SELECT\n", + " customer_id,\n", + " DATE_DIFF(DATE('2011-12-01'), DATE('2011-09-01'), DAY) AS target_days,\n", + " DATE_DIFF(DATE('2011-09-01'), MIN(order_date), DAY) AS feature_days,\n", + " FROM\n", + " `online_retail.online_retail_clv_clean`\n", + " GROUP BY\n", + " customer_id\n", + " ),\n", + " \n", + " predicted_clv AS (\n", + " SELECT\n", + " customer_id,\n", + " AVG(avg_purchase_revenue) * (COUNT(n_purchases) * (1 + SAFE_DIVIDE(COUNT(target_days),COUNT(feature_days)))) AS predicted_monetary_value_3M,\n", + " SUM(target_monetary_value_3M) AS target_monetary_value_3M\n", + " FROM\n", + " `online_retail.online_retail_clv_ml`\n", + " LEFT JOIN day_intervals USING(customer_id)\n", + " GROUP BY\n", + " customer_id\n", + " )\n", + "\n", + "# Calculate overall baseline regression metrics.\n", + "SELECT\n", + " ROUND(AVG(ABS(predicted_monetary_value_3M - target_monetary_value_3M)), 2) AS MAE,\n", + " ROUND(AVG(POW(predicted_monetary_value_3M - target_monetary_value_3M, 2)), 2) AS MSE,\n", + " ROUND(SQRT(AVG(POW(predicted_monetary_value_3M - target_monetary_value_3M, 2))), 2) AS RMSE\n", + "FROM\n", + " predicted_clv" + ] + }, + { + "cell_type": "markdown", + "id": "956ac010", + "metadata": {}, + "source": [ + "These baseline results provide further support for the strong impact of outliers. The extremely high MSE comes from the exponential penalty applied to missed predictions and the magnitude of error on a few predictions.\n", + "\n", + "Next, you should look to plot the baseline results to get a sense of opportunity areas for you ML model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e14ff67", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery baseline\n", + "\n", + "WITH\n", + " day_intervals AS (\n", + " SELECT\n", + " customer_id,\n", + " DATE_DIFF(DATE('2011-12-01'), DATE('2011-09-01'), DAY) AS target_days,\n", + " DATE_DIFF(DATE('2011-09-01'), MIN(order_date), DAY) AS feature_days,\n", + " FROM\n", + " `online_retail.online_retail_clv_clean`\n", + " GROUP BY\n", + " customer_id\n", + " ),\n", + " \n", + " predicted_clv AS (\n", + " SELECT\n", + " customer_id,\n", + " AVG(avg_purchase_revenue) * (COUNT(n_purchases) * (1 + SAFE_DIVIDE(COUNT(target_days),COUNT(feature_days)))) AS predicted_monetary_value_3M,\n", + " SUM(target_monetary_value_3M) AS target_monetary_value_3M\n", + " FROM\n", + " `online_retail.online_retail_clv_ml`\n", + " INNER JOIN day_intervals USING(customer_id)\n", + " GROUP BY\n", + " customer_id\n", + " )\n", + "\n", + "SELECT\n", + " *\n", + "FROM\n", + " predicted_clv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afda73aa", + "metadata": {}, + "outputs": [], + "source": [ + "baseline.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a543c10", + "metadata": {}, + "outputs": [], + "source": [ + "ax = baseline.plot(kind='scatter',\n", + " x='predicted_monetary_value_3M', \n", + " y='target_monetary_value_3M',\n", + " title='Actual vs. Predicted customer 3-month monetary value',\n", + " figsize=(5,5),\n", + " grid=True)\n", + "\n", + "lims = [\n", + " np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes\n", + " np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "# now plot both limits against eachother\n", + "ax.plot(lims, lims, 'k-', alpha=0.5, zorder=0)\n", + "ax.set_aspect('equal')\n", + "ax.set_xlim(lims)\n", + "ax.set_ylim(lims);" + ] + }, + { + "cell_type": "markdown", + "id": "0d53ad3a", + "metadata": {}, + "source": [ + "## Train a TensorFlow model locally" + ] + }, + { + "cell_type": "markdown", + "id": "b3658b32", + "metadata": {}, + "source": [ + "Now that you have a simple baseline to benchmark your performance against, train a TensorFlow Regressor to predict CLV." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c45e2feb", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "\n", + "SELECT data_split, COUNT(*)\n", + "FROM `online_retail.online_retail_clv_ml`\n", + "GROUP BY data_split" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7e2994a", + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery clv\n", + "\n", + "SELECT *\n", + "FROM `online_retail.online_retail_clv_ml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80339852", + "metadata": {}, + "outputs": [], + "source": [ + "clv_train = clv.loc[clv.data_split == 'TRAIN', :]\n", + "clv_dev = clv.loc[clv.data_split == 'VALIDATE', :]\n", + "clv_test = clv.loc[clv.data_split == 'TEST', :]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a15e9683", + "metadata": {}, + "outputs": [], + "source": [ + "# Model training constants.\n", + "# Virtual epochs design pattern:\n", + "# https://medium.com/google-cloud/ml-design-pattern-3-virtual-epochs-f842296de730\n", + "N_TRAIN_EXAMPLES = 2638\n", + "STOP_POINT = 20.0\n", + "TOTAL_TRAIN_EXAMPLES = int(STOP_POINT * N_TRAIN_EXAMPLES)\n", + "BATCH_SIZE = 32\n", + "N_CHECKPOINTS = 10\n", + "STEPS_PER_EPOCH = (TOTAL_TRAIN_EXAMPLES // (BATCH_SIZE*N_CHECKPOINTS))\n", + "\n", + "NUMERIC_FEATURES = [\n", + " \"n_purchases\",\n", + " \"avg_purchase_size\",\n", + " \"avg_purchase_revenue\",\n", + " \"customer_age\",\n", + " \"days_since_last_purchase\",\n", + "]\n", + "\n", + "LABEL = \"target_monetary_value_3M\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "627cc31a", + "metadata": {}, + "outputs": [], + "source": [ + "def df_dataset(df):\n", + " \"\"\"Transform Pandas Dataframe to TensorFlow Dataset.\"\"\"\n", + " return tf.data.Dataset.from_tensor_slices((df[NUMERIC_FEATURES].to_dict('list'), df[LABEL].values))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b0744b6", + "metadata": {}, + "outputs": [], + "source": [ + "trainds = df_dataset(clv_train).prefetch(1).batch(BATCH_SIZE).repeat()\n", + "devds = df_dataset(clv_dev).prefetch(1).batch(BATCH_SIZE)\n", + "testds = df_dataset(clv_test).prefetch(1).batch(BATCH_SIZE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9459079", + "metadata": {}, + "outputs": [], + "source": [ + "def rmse(y_true, y_pred):\n", + " \"\"\"Custom RMSE regression metric.\"\"\"\n", + " return tf.sqrt(tf.reduce_mean(tf.square(y_pred - y_true)))\n", + "\n", + "\n", + "def build_model():\n", + " \"\"\"Build and compile a TensorFlow Keras Regressor.\"\"\"\n", + " # Define input feature tensors and input layers.\n", + " feature_columns = [\n", + " tf.feature_column.numeric_column(key=feature)\n", + " for feature in NUMERIC_FEATURES\n", + " ]\n", + " \n", + " input_layers = {\n", + " feature.key: tf.keras.layers.Input(name=feature.key, shape=(), dtype=tf.float32)\n", + " for feature in feature_columns\n", + " }\n", + " \n", + " # Keras Functional API: https://keras.io/guides/functional_api\n", + " inputs = tf.keras.layers.DenseFeatures(feature_columns, name='inputs')(input_layers)\n", + " d1 = tf.keras.layers.Dense(256, activation=tf.nn.relu, name='d1')(inputs)\n", + " d2 = tf.keras.layers.Dropout(0.2, name='d2')(d1) \n", + " # Note: the single neuron output for regression.\n", + " output = tf.keras.layers.Dense(1, name='output')(d2)\n", + " \n", + " model = tf.keras.Model(input_layers, output, name='online-retail-clv')\n", + " \n", + " optimizer = tf.keras.optimizers.Adam(0.001) \n", + " \n", + " # Note: MAE loss is more resistant to outliers than MSE.\n", + " model.compile(loss=tf.keras.losses.MAE,\n", + " optimizer=optimizer,\n", + " metrics=[['mae', 'mse', rmse]])\n", + " \n", + " return model\n", + "\n", + "model = build_model()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8601ff5f", + "metadata": {}, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(model, show_shapes=False, rankdir=\"LR\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "354206ee", + "metadata": {}, + "outputs": [], + "source": [ + "tensorboard_callback = tf.keras.callbacks.TensorBoard(\n", + " log_dir='./local-training/tensorboard',\n", + " histogram_freq=1)\n", + "\n", + "earlystopping_callback = tf.keras.callbacks.EarlyStopping(patience=1)\n", + "\n", + "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n", + " filepath='./local-training/checkpoints',\n", + " save_weights_only=True,\n", + " monitor='val_loss',\n", + " mode='min')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "730181fb", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.fit(trainds,\n", + " validation_data=devds,\n", + " steps_per_epoch=STEPS_PER_EPOCH,\n", + " epochs=N_CHECKPOINTS,\n", + " callbacks=[[tensorboard_callback,\n", + " earlystopping_callback,\n", + " checkpoint_callback]])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2594d084", + "metadata": {}, + "outputs": [], + "source": [ + "LOSS_COLS = [\"loss\", \"val_loss\"]\n", + "\n", + "pd.DataFrame(history.history)[LOSS_COLS].plot();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b71775db", + "metadata": {}, + "outputs": [], + "source": [ + "train_pred = model.predict(df_dataset(clv_train).prefetch(1).batch(BATCH_SIZE))\n", + "dev_pred = model.predict(devds)\n", + "test_pred = model.predict(testds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b6eceb1", + "metadata": {}, + "outputs": [], + "source": [ + "train_results = pd.DataFrame({'actual': clv_train['target_monetary_value_3M'].to_numpy(), 'predicted': np.squeeze(train_pred)}, columns=['actual', 'predicted'])\n", + "dev_results = pd.DataFrame({'actual': clv_dev['target_monetary_value_3M'].to_numpy(), 'predicted': np.squeeze(dev_pred)}, columns=['actual', 'predicted'])\n", + "test_results = pd.DataFrame({'actual': clv_test['target_monetary_value_3M'].to_numpy(), 'predicted': np.squeeze(test_pred)}, columns=['actual', 'predicted'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4659dd09", + "metadata": {}, + "outputs": [], + "source": [ + "# Model prediction calibration plots.\n", + "fig, (train_ax, dev_ax, test_ax) = plt.subplots(1, 3, figsize=(15,15))\n", + "\n", + "train_results.plot(kind='scatter',\n", + " x='predicted',\n", + " y='actual',\n", + " title='Train: act vs. pred customer 3M monetary value',\n", + " grid=True,\n", + " ax=train_ax)\n", + "\n", + "train_lims = [\n", + " np.min([train_ax.get_xlim(), train_ax.get_ylim()]), # min of both axes\n", + " np.max([train_ax.get_xlim(), train_ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "train_ax.plot(train_lims, train_lims, 'k-', alpha=0.5, zorder=0)\n", + "train_ax.set_aspect('equal')\n", + "train_ax.set_xlim(train_lims)\n", + "train_ax.set_ylim(train_lims)\n", + "\n", + "dev_results.plot(kind='scatter',\n", + " x='predicted',\n", + " y='actual',\n", + " title='Dev: act vs. pred customer 3M monetary value',\n", + " grid=True,\n", + " ax=dev_ax)\n", + "\n", + "dev_lims = [\n", + " np.min([dev_ax.get_xlim(), dev_ax.get_ylim()]), # min of both axes\n", + " np.max([dev_ax.get_xlim(), dev_ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "dev_ax.plot(dev_lims, dev_lims, 'k-', alpha=0.5, zorder=0)\n", + "dev_ax.set_aspect('equal')\n", + "dev_ax.set_xlim(dev_lims)\n", + "dev_ax.set_ylim(dev_lims)\n", + "\n", + "test_results.plot(kind='scatter',\n", + " x='predicted',\n", + " y='actual',\n", + " title='Test: act vs. pred customer 3M monetary value',\n", + " grid=True,\n", + " ax=test_ax)\n", + "\n", + "test_lims = [\n", + " np.min([test_ax.get_xlim(), test_ax.get_ylim()]), # min of both axes\n", + " np.max([test_ax.get_xlim(), test_ax.get_ylim()]), # max of both axes\n", + "]\n", + "\n", + "test_ax.plot(test_lims, test_lims, 'k-', alpha=0.5, zorder=0)\n", + "test_ax.set_aspect('equal')\n", + "test_ax.set_xlim(test_lims)\n", + "test_ax.set_ylim(test_lims);" + ] + }, + { + "cell_type": "markdown", + "id": "2a5f1582", + "metadata": {}, + "source": [ + "You have trained a model better than your baseline. As indicated in the charts above, there is still additional feature engineering and data cleaning opportunities to improve your model's performance on customers with CLV. Some options include handling these customers as a separate prediction task, applying a log transformation to your target, clipping their value or dropping these customers all together to improve model performance.\n", + "\n", + "Now, you work through taking this local TensorFlow workflow to the cloud with Vertex AI." + ] + }, + { + "cell_type": "markdown", + "id": "24bb7c43", + "metadata": {}, + "source": [ + "## Create a managed Tabular dataset from your BigQuery data source" + ] + }, + { + "cell_type": "markdown", + "id": "f8383baa", + "metadata": {}, + "source": [ + "[**Vertex AI managed datasets**](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular) can be used to train AutoML models or custom-trained models.\n", + "\n", + "You create a [**Tabular regression dataset**](https://cloud.google.com/vertex-ai/docs/datasets/bp-tabular) for managing the sharing and metadata for this lab's dataset stored in BigQuery. Managed datasets enable you to create a clear link between your data and custom-trained models, and provide descriptive statistics and automatic or manual splitting into train, test, and validation sets. \n", + "\n", + "In this lab, the data processing step already created a manual `data_split` column in your BQ ML table using [BigQuery's hashing functions](https://towardsdatascience.com/ml-design-pattern-5-repeatable-sampling-c0ccb2889f39) for repeatable sampling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "964c1eb3", + "metadata": {}, + "outputs": [], + "source": [ + "tabular_dataset = aiplatform.TabularDataset.create(display_name=\"online-retail-clv\", bq_source=f\"{BQ_URI}\")" + ] + }, + { + "cell_type": "markdown", + "id": "420b6fd9", + "metadata": {}, + "source": [ + "## Vertex AI custom ML model training workflow" + ] + }, + { + "cell_type": "markdown", + "id": "c3806a39", + "metadata": {}, + "source": [ + "There are two ways you can train a custom model on Vertex AI:\n", + "\n", + "Before you submit a custom training job, hyperparameter tuning job, or a training pipeline to Vertex AI, you need to create a Python training application or a custom container to define the training code and dependencies you want to run on Vertex AI.\n", + "\n", + "**1. Use a Google Cloud prebuilt container**: if you use a Vertex AI prebuilt container, you write a Python `task.py` script or Python package to install into the container image that defines your code for training a custom model. See [Creating a Python training application for a pre-built container](https://cloud.google.com/vertex-ai/docs/training/create-python-pre-built-container) for more details on how to structure you Python code. Choose this option if a prebuilt container already contains the model training libraries you need such as `tensorflow` or `xgboost` and you are just doing ML training and prediction quickly. You can also specific additional Python dependencies to install through the `CustomTrainingJob(requirements=...` argument.\n", + "\n", + "**2. Use your own custom container image**: If you want to use your own custom container, you write your Python training scripts and a Dockerfile that contains instructions on your ML model code, dependencies, and execution instructions. You will build your custom container with Cloud Build, whose instructions are specified in `cloudbuild.yaml` and publish your container to your Artifact Registry. Choose this option if you want to package your ML model code with dependencies together in a container to build toward running as part of a portable and scalable [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) workflow. " + ] + }, + { + "cell_type": "markdown", + "id": "2e42f26a", + "metadata": {}, + "source": [ + "### Containerize your model training code" + ] + }, + { + "cell_type": "markdown", + "id": "6b99d903", + "metadata": {}, + "source": [ + "In the next 5 steps, you proceed with **2. Use your own custom container image**. \n", + "\n", + "You build your custom model container on top of a [Google Cloud Deep Learning container](https://cloud.google.com/vertex-ai/docs/general/deep-learning) that contains tested and optimized versions of model code dependencies such as `tensorflow` and the `google-cloud-bigquery` SDK. This also gives you flexibility and enables to manage and share your model container image with others for reuse and reproducibility across environments while also enabling you to incorporate additional packages for your ML application. Lastly, by packaging your ML model code together with dependencies you also have a MLOps onboarding path to Vertex Pipelines.\n", + "\n", + "You walk through creating the following project structure for your ML mode code:\n", + "\n", + "```\n", + "|--/online-retail-clv-3M\n", + " |--/trainer\n", + " |--__init__.py\n", + " |--model.py\n", + " |--task.py\n", + " |--Dockerfile\n", + " |--cloudbuild.yaml\n", + " |--requirements.txt\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "2db0ba26", + "metadata": {}, + "source": [ + "#### 1. Write a `model.py` training script" + ] + }, + { + "cell_type": "markdown", + "id": "cb5a08e3", + "metadata": {}, + "source": [ + "First, you take tidy up your local TensorFlow model training code from above into a training script.\n", + "\n", + "The biggest change is you utilize the [TensorFlow IO](https://www.tensorflow.org/io/tutorials/bigquery) library to performantly read from BigQuery directly into your TensorFlow model graph during training. This improves your training performance rather than performing the intermediate step of reading from BigQuery into a Pandas Dataframe done for expediency above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0cae846", + "metadata": {}, + "outputs": [], + "source": [ + "# this is the name of your model subdirectory you will write your model code to. It is already created in your lab directory.\n", + "MODEL_NAME=\"online-retail-clv-3M\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbe19974", + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile {MODEL_NAME}/trainer/model.py\n", + "import os\n", + "import logging\n", + "import tempfile\n", + "import tensorflow as tf\n", + "from explainable_ai_sdk.metadata.tf.v2 import SavedModelMetadataBuilder\n", + "from tensorflow.python.framework import dtypes\n", + "from tensorflow_io.bigquery import BigQueryClient\n", + "from tensorflow_io.bigquery import BigQueryReadSession\n", + "\n", + "\n", + "# Model feature constants.\n", + "NUMERIC_FEATURES = [\n", + " \"n_purchases\",\n", + " \"avg_purchase_size\",\n", + " \"avg_purchase_revenue\",\n", + " \"customer_age\",\n", + " \"days_since_last_purchase\",\n", + "]\n", + "\n", + "CATEGORICAL_FEATURES = [\n", + " \"customer_country\"\n", + "]\n", + "\n", + "LABEL = \"target_monetary_value_3M\"\n", + "\n", + "\n", + "def caip_uri_to_fields(uri):\n", + " \"\"\"Helper function to parse BQ URI.\"\"\"\n", + " # Remove bq:// prefix.\n", + " uri = uri[5:]\n", + " project, dataset, table = uri.split('.')\n", + " return project, dataset, table\n", + "\n", + "\n", + "def features_and_labels(row_data):\n", + " \"\"\"Helper feature and label mapping function for tf.data.\"\"\"\n", + " label = row_data.pop(LABEL)\n", + " features = row_data\n", + " return features, label\n", + "\n", + "\n", + "def read_bigquery(project, dataset, table):\n", + " \"\"\"TensorFlow IO BigQuery Reader.\"\"\"\n", + " tensorflow_io_bigquery_client = BigQueryClient()\n", + " read_session = tensorflow_io_bigquery_client.read_session(\n", + " parent=\"projects/\" + project,\n", + " project_id=project, \n", + " dataset_id=dataset,\n", + " table_id=table,\n", + " # Pass list of features and label to be selected from BQ.\n", + " selected_fields=NUMERIC_FEATURES + [LABEL],\n", + " # Provide output TensorFlow data types for features and label.\n", + " output_types=[dtypes.int64, dtypes.float64, dtypes.float64, dtypes.int64, dtypes.int64] + [dtypes.float64],\n", + " requested_streams=2)\n", + " dataset = read_session.parallel_read_rows()\n", + " transformed_ds = dataset.map(features_and_labels)\n", + " return transformed_ds\n", + "\n", + "\n", + "def rmse(y_true, y_pred):\n", + " \"\"\"Custom RMSE regression metric.\"\"\"\n", + " return tf.sqrt(tf.reduce_mean(tf.square(y_pred - y_true)))\n", + "\n", + "\n", + "def build_model(hparams):\n", + " \"\"\"Build and compile a TensorFlow Keras DNN Regressor.\"\"\"\n", + "\n", + " feature_columns = [\n", + " tf.feature_column.numeric_column(key=feature)\n", + " for feature in NUMERIC_FEATURES\n", + " ]\n", + " \n", + " input_layers = {\n", + " feature.key: tf.keras.layers.Input(name=feature.key, shape=(), dtype=tf.float32)\n", + " for feature in feature_columns\n", + " }\n", + " # Keras Functional API: https://keras.io/guides/functional_api\n", + " inputs = tf.keras.layers.DenseFeatures(feature_columns, name='inputs')(input_layers)\n", + " d1 = tf.keras.layers.Dense(256, activation=tf.nn.relu, name='d1')(inputs)\n", + " d2 = tf.keras.layers.Dropout(hparams['dropout'], name='d2')(d1) \n", + " # Note: a single neuron scalar output for regression.\n", + " output = tf.keras.layers.Dense(1, name='output')(d2)\n", + " \n", + " model = tf.keras.Model(input_layers, output, name='online-retail-clv')\n", + " \n", + " optimizer = tf.keras.optimizers.Adam(hparams['learning-rate']) \n", + " \n", + " # Note: MAE loss is more resistant to outliers than MSE.\n", + " model.compile(loss=tf.keras.losses.MAE,\n", + " optimizer=optimizer,\n", + " metrics=[['mae', 'mse', rmse]])\n", + " \n", + " return model\n", + "\n", + "\n", + "def train_evaluate_explain_model(hparams):\n", + " \"\"\"Train, evaluate, explain TensorFlow Keras DNN Regressor.\n", + " Args:\n", + " hparams(dict): A dictionary containing model training arguments.\n", + " Returns:\n", + " history(tf.keras.callbacks.History): Keras callback that records training event history.\n", + " \"\"\"\n", + " training_ds = read_bigquery(*caip_uri_to_fields(hparams['training-data-uri'])).prefetch(1).shuffle(hparams['batch-size']*10).batch(hparams['batch-size']).repeat()\n", + " eval_ds = read_bigquery(*caip_uri_to_fields(hparams['validation-data-uri'])).prefetch(1).shuffle(hparams['batch-size']*10).batch(hparams['batch-size'])\n", + " test_ds = read_bigquery(*caip_uri_to_fields(hparams['test-data-uri'])).prefetch(1).shuffle(hparams['batch-size']*10).batch(hparams['batch-size'])\n", + " \n", + " model = build_model(hparams)\n", + " logging.info(model.summary())\n", + " \n", + " tensorboard_callback = tf.keras.callbacks.TensorBoard(\n", + " log_dir=hparams['tensorboard-dir'],\n", + " histogram_freq=1)\n", + " \n", + " # Reduce overfitting and shorten training times.\n", + " earlystopping_callback = tf.keras.callbacks.EarlyStopping(patience=2)\n", + " \n", + " # Ensure your training job's resilience to VM restarts.\n", + " checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n", + " filepath= hparams['checkpoint-dir'],\n", + " save_weights_only=True,\n", + " monitor='val_loss',\n", + " mode='min')\n", + " \n", + " # Virtual epochs design pattern:\n", + " # https://medium.com/google-cloud/ml-design-pattern-3-virtual-epochs-f842296de730\n", + " TOTAL_TRAIN_EXAMPLES = int(hparams['stop-point'] * hparams['n-train-examples'])\n", + " STEPS_PER_EPOCH = (TOTAL_TRAIN_EXAMPLES // (hparams['batch-size']*hparams['n-checkpoints'])) \n", + " \n", + " history = model.fit(training_ds,\n", + " validation_data=eval_ds,\n", + " steps_per_epoch=STEPS_PER_EPOCH,\n", + " epochs=hparams['n-checkpoints'],\n", + " callbacks=[[tensorboard_callback,\n", + " earlystopping_callback,\n", + " checkpoint_callback]])\n", + " \n", + " logging.info(model.evaluate(test_ds))\n", + " \n", + " # Create a temp directory to save intermediate TF SavedModel prior to Explainable metadata creation.\n", + " tmpdir = tempfile.mkdtemp()\n", + " \n", + " # Export Keras model in TensorFlow SavedModel format.\n", + " model.save(tmpdir)\n", + " \n", + " # Annotate and save TensorFlow SavedModel with Explainable metadata to GCS.\n", + " builder = SavedModelMetadataBuilder(tmpdir)\n", + " builder.save_model_with_metadata(hparams['model-dir'])\n", + " \n", + " return history" + ] + }, + { + "cell_type": "markdown", + "id": "c10121ec", + "metadata": {}, + "source": [ + "#### 2. Write a `task.py` file as an entrypoint to your custom ML model container" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d4d6add", + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile {MODEL_NAME}/trainer/task.py\n", + "import os\n", + "import argparse\n", + "\n", + "from trainer import model\n", + "\n", + "if __name__ == '__main__':\n", + " parser = argparse.ArgumentParser()\n", + " # Vertex custom container training args. These are set by Vertex AI during training but can also be overwritten.\n", + " parser.add_argument('--model-dir', dest='model-dir',\n", + " default=os.environ['AIP_MODEL_DIR'], type=str, help='Model dir.')\n", + " parser.add_argument('--checkpoint-dir', dest='checkpoint-dir',\n", + " default=os.environ['AIP_CHECKPOINT_DIR'], type=str, help='Checkpoint dir set during Vertex AI training.') \n", + " parser.add_argument('--tensorboard-dir', dest='tensorboard-dir',\n", + " default=os.environ['AIP_TENSORBOARD_LOG_DIR'], type=str, help='Tensorboard dir set during Vertex AI training.') \n", + " parser.add_argument('--data-format', dest='data-format',\n", + " default=os.environ['AIP_DATA_FORMAT'], type=str, help=\"Tabular data format set during Vertex AI training. E.g.'csv', 'bigquery'\")\n", + " parser.add_argument('--training-data-uri', dest='training-data-uri',\n", + " default=os.environ['AIP_TRAINING_DATA_URI'], type=str, help='Training data GCS or BQ URI set during Vertex AI training.')\n", + " parser.add_argument('--validation-data-uri', dest='validation-data-uri',\n", + " default=os.environ['AIP_VALIDATION_DATA_URI'], type=str, help='Validation data GCS or BQ URI set during Vertex AI training.')\n", + " parser.add_argument('--test-data-uri', dest='test-data-uri',\n", + " default=os.environ['AIP_TEST_DATA_URI'], type=str, help='Test data GCS or BQ URI set during Vertex AI training.')\n", + " # Model training args.\n", + " parser.add_argument('--learning-rate', dest='learning-rate', default=0.001, type=float, help='Learning rate for optimizer.')\n", + " parser.add_argument('--dropout', dest='dropout', default=0.2, type=float, help='Float percentage of DNN nodes [0,1] to drop for regularization.') \n", + " parser.add_argument('--batch-size', dest='batch-size', default=16, type=int, help='Number of examples during each training iteration.') \n", + " parser.add_argument('--n-train-examples', dest='n-train-examples', default=2638, type=int, help='Number of examples to train on.')\n", + " parser.add_argument('--stop-point', dest='stop-point', default=10, type=int, help='Number of passes through the dataset during training to achieve convergence.')\n", + " parser.add_argument('--n-checkpoints', dest='n-checkpoints', default=10, type=int, help='Number of model checkpoints to save during training.')\n", + " \n", + " args = parser.parse_args()\n", + " hparams = args.__dict__\n", + "\n", + " model.train_evaluate_explain_model(hparams)" + ] + }, + { + "cell_type": "markdown", + "id": "18058766", + "metadata": {}, + "source": [ + "#### 3. Write a `Dockerfile` for your custom ML model container" + ] + }, + { + "cell_type": "markdown", + "id": "987cc52a", + "metadata": {}, + "source": [ + "Third, you write a `Dockerfile` that contains your model code as well as specifies your model code's dependencies.\n", + "\n", + "Notice the base image below is a [Google Cloud Deep Learning container](https://cloud.google.com/vertex-ai/docs/general/deep-learning) that contains tested and optimized versions of model code dependencies such as `tensorflow` and the `google-cloud-bigquery` SDK." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28ea8f68", + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile {MODEL_NAME}/Dockerfile\n", + "# Specifies base image and tag.\n", + "# https://cloud.google.com/vertex-ai/docs/general/deep-learning\n", + "# https://cloud.google.com/deep-learning-containers/docs/choosing-container\n", + "FROM gcr.io/deeplearning-platform-release/tf2-cpu.2-3\n", + "\n", + "# Sets the container working directory.\n", + "WORKDIR /root\n", + "\n", + "# Copies the requirements.txt into the container to reduce network calls.\n", + "COPY requirements.txt .\n", + "# Installs additional packages.\n", + "RUN pip3 install -U -r requirements.txt\n", + "\n", + "# Copies the trainer code to the docker image.\n", + "COPY . /trainer\n", + "\n", + "# Sets the container working directory.\n", + "WORKDIR /trainer\n", + "\n", + "# Sets up the entry point to invoke the trainer.\n", + "ENTRYPOINT [\"python\", \"-m\", \"trainer.task\"]" + ] + }, + { + "cell_type": "markdown", + "id": "f2db8aea", + "metadata": {}, + "source": [ + "### 4. Write a `requirements.txt` file to specify additional ML code dependencies" + ] + }, + { + "cell_type": "markdown", + "id": "f13b99fb", + "metadata": {}, + "source": [ + "These are additional dependencies for your model code outside the deep learning containers needed for prediction explainability and the BigQuery TensorFlow IO reader." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06998a4e", + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile {MODEL_NAME}/requirements.txt\n", + "explainable-ai-sdk==1.3.0\n", + "tensorflow-io==0.15.0\n", + "pyarrow" + ] + }, + { + "cell_type": "markdown", + "id": "5214db92", + "metadata": {}, + "source": [ + "#### 5. Use Cloud Build to build and submit your container to Google Cloud Artifact Registry" + ] + }, + { + "cell_type": "markdown", + "id": "25ff06d2", + "metadata": {}, + "source": [ + "Next, you use [Cloud Build](https://cloud.google.com/build) to build and upload your custom TensorFlow model container to [Google Cloud Artifact Registry](https://cloud.google.com/artifact-registry).\n", + "\n", + "Cloud Build brings reusability and automation to your ML experimentation by enabling you to reliably build, test, and deploy your ML model code as part of a CI/CD workflow. Artifact Registry provides a centralized repository for you to store, manage, and secure your ML container images. This allows you to securely share your ML work with others and reproduce experiment results.\n", + "\n", + "**Note**: The initial build and submit step will take about 20 minutes but Cloud Build is able to take advantage of caching for subsequent builds." + ] + }, + { + "cell_type": "markdown", + "id": "65a8c7f1", + "metadata": {}, + "source": [ + "#### Create Artifact Repository for custom container images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8984969", + "metadata": {}, + "outputs": [], + "source": [ + "ARTIFACT_REPOSITORY=\"online-retail-clv\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff4c1484", + "metadata": {}, + "outputs": [], + "source": [ + "# Create an Artifact Repository using the gcloud CLI.\n", + "!gcloud artifacts repositories create $ARTIFACT_REPOSITORY \\\n", + "--repository-format=docker \\\n", + "--location=$REGION \\\n", + "--description=\"Artifact registry for ML custom training images for predictive CLV\"" + ] + }, + { + "cell_type": "markdown", + "id": "b8703d94", + "metadata": {}, + "source": [ + "#### Create `cloudbuild.yaml` instructions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efe17ff9", + "metadata": {}, + "outputs": [], + "source": [ + "IMAGE_NAME=\"dnn-regressor\"\n", + "IMAGE_TAG=\"latest\"\n", + "IMAGE_URI=f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{ARTIFACT_REPOSITORY}/{IMAGE_NAME}:{IMAGE_TAG}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c834b5a9", + "metadata": {}, + "outputs": [], + "source": [ + "cloudbuild_yaml = f\"\"\"steps:\n", + "- name: 'gcr.io/cloud-builders/docker'\n", + " args: [ 'build', '-t', '{IMAGE_URI}', '.' ]\n", + "images: \n", + "- '{IMAGE_URI}'\"\"\"\n", + "\n", + "with open(f\"{MODEL_NAME}/cloudbuild.yaml\", \"w\") as fp:\n", + " fp.write(cloudbuild_yaml)" + ] + }, + { + "cell_type": "markdown", + "id": "b590f66b", + "metadata": {}, + "source": [ + "#### Build and submit your container image to your Artifact Repository" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9361461", + "metadata": {}, + "outputs": [], + "source": [ + "!gcloud builds submit --timeout=20m --config {MODEL_NAME}/cloudbuild.yaml {MODEL_NAME}" + ] + }, + { + "cell_type": "markdown", + "id": "4efcc053", + "metadata": {}, + "source": [ + "Now that your custom container is built and stored in your Artifact Registry, its time to train your model in the cloud with Vertex AI." + ] + }, + { + "cell_type": "markdown", + "id": "ea2cdc6f", + "metadata": {}, + "source": [ + "## Run a custom training job on Vertex AI" + ] + }, + { + "cell_type": "markdown", + "id": "c77ba8b0", + "metadata": {}, + "source": [ + "### 1. Create a Vertex Tensorboard instance for tracking your model experiments" + ] + }, + { + "cell_type": "markdown", + "id": "f82f8bbb", + "metadata": {}, + "source": [ + "[**Vertex TensorBoard**](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview) is Google Cloud's managed version of open-source [**TensorBoard**](https://www.tensorflow.org/tensorboard) for ML experimental visualization. With Vertex TensorBoard you can track, visualize, and compare ML experiments and share them with your team. In addition to the powerful visualizations from open source TensorBoard, Vertex TensorBoard provides:\n", + "\n", + "* A persistent, shareable link to your experiment's dashboard.\n", + "* A searchable list of all experiments in a project.\n", + "* Integrations with Vertex AI services for model training evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec1755a1", + "metadata": {}, + "outputs": [], + "source": [ + "!gcloud beta ai tensorboards create \\\n", + "--display-name=$MODEL_NAME --region=$REGION" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aeac53ba", + "metadata": {}, + "outputs": [], + "source": [ + "TENSORBOARD_RESOURCE_NAME= !(gcloud beta ai tensorboards list --region=$REGION --format=\"value(name)\")\n", + "TENSORBOARD_RESOURCE_NAME= TENSORBOARD_RESOURCE_NAME[1]\n", + "TENSORBOARD_RESOURCE_NAME" + ] + }, + { + "cell_type": "markdown", + "id": "9ad5abad", + "metadata": {}, + "source": [ + "### 2. Run your custom container training job" + ] + }, + { + "cell_type": "markdown", + "id": "a92fe321", + "metadata": {}, + "source": [ + "Use the `CustomTrainingJob` class to define the job, which takes the following parameters specific to custom container training:\n", + "\n", + "* `display_name`: You user-defined name of this training pipeline.\n", + "* `container_uri`: The URI of your custom training container image.\n", + "* `model_serving_container_image_uri`: The URI of a container that can serve predictions for your model. You use a Vertex prebuilt container.\n", + "\n", + "Use the `run()` function to start training, which takes the following parameters:\n", + "\n", + "* `replica_count`: The number of worker replicas.\n", + "* `model_display_name`: The display name of the Model if the script produces a managed Model.\n", + "* `machine_type`: The type of machine to use for training.\n", + "* `bigquery_destination`: The BigQuery URI where your created Tabular dataset gets written to.\n", + "* `predefined_split_column_name`: Since this lab leveraged BigQuery for data processing and splitting, this column is specified to indicate data splits.\n", + "\n", + "The run function creates a training pipeline that trains and creates a Vertex `Model` object. After the training pipeline completes, the `run()` function returns the `Model` object.\n", + "\n", + "Note: This `CustomContainerTrainingJob` will take about 20 minutes to provision resources and train your model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e88b63a8", + "metadata": {}, + "outputs": [], + "source": [ + "# command line args for trainer.task defined above. Review the 'help' argument for a description.\n", + "# You will set the model training args below. Vertex AI will set the environment variables for training URIs.\n", + "CMD_ARGS= [\n", + " \"--learning-rate=\" + str(0.001),\n", + " \"--batch-size=\" + str(16),\n", + " \"--n-train-examples=\" + str(2638),\n", + " \"--stop-point=\" + str(10),\n", + " \"--n-checkpoints=\" + str(10),\n", + " \"--dropout=\" + str(0.2), \n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be63e362", + "metadata": {}, + "outputs": [], + "source": [ + "# By setting BASE_OUTPUT_DIR, Vertex AI will set the environment variables AIP_MODEL_DIR, AIP_CHECKPOINT_DIR, AIP_TENSORBOARD_LOG_DIR\n", + "# during training for your ML training code to write to.\n", + "TIMESTAMP=datetime.datetime.now().strftime('%Y%m%d%H%M%S')\n", + "BASE_OUTPUT_DIR= f\"gs://{GCS_BUCKET}/vertex-custom-training-{MODEL_NAME}-{TIMESTAMP}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0301c683", + "metadata": {}, + "outputs": [], + "source": [ + "job = aiplatform.CustomContainerTrainingJob(\n", + " display_name=\"online-retail-clv-3M-dnn-regressor\",\n", + " container_uri=IMAGE_URI,\n", + " # https://cloud.google.com/vertex-ai/docs/predictions/pre-built-containers\n", + " # gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-3:latest\n", + " model_serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-3:latest\",\n", + ")\n", + "\n", + "model = job.run(\n", + " dataset=tabular_dataset,\n", + " model_display_name=MODEL_NAME,\n", + " # GCS custom job output dir.\n", + " base_output_dir=BASE_OUTPUT_DIR,\n", + " # the BQ Tabular dataset splits will be written out to their own BQ dataset for reproducibility.\n", + " bigquery_destination=f\"bq://{PROJECT_ID}\",\n", + " # this corresponds to the BigQuery data split column.\n", + " predefined_split_column_name=\"data_split\",\n", + " # the model training command line arguments defined in trainer.task.\n", + " args=CMD_ARGS,\n", + " # Custom job WorkerPool arguments.\n", + " replica_count=1,\n", + " machine_type=\"e2-standard-4\",\n", + " # Provide your Tensorboard resource name to write Tensorboard logs during training.\n", + " tensorboard=TENSORBOARD_RESOURCE_NAME,\n", + " # Provide your Vertex custom training service account created during lab setup.\n", + " service_account=f\"vertex-custom-training-sa@{PROJECT_ID}.iam.gserviceaccount.com\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "932c4086", + "metadata": {}, + "source": [ + "### 3. Inspect model training performance with Vertex TensorBoard" + ] + }, + { + "cell_type": "markdown", + "id": "daa6b127", + "metadata": {}, + "source": [ + "You can view your model's logs on the Vertex AI [**Experiments tab**](https://console.cloud.google.com/vertex-ai/experiments) in the Cloud Console. Click the **Open Tensorboard** link. You will be asked to authenticate with your Qwiklabs Google account before a Vertex Tensorboard page opens in a browser tab. Once your model begins training, you will see your training evaluation metrics written to this dashboard that you can inspect during the training run as well as after the job completes.\n", + "\n", + "Note: Tensorboard provides a valuable debugging tool for inspecting your model's performance both during and after model training. This lab's model trains in less than a minute and sometimes completes before the logs finish appearing in Tensorboard. If that's the case, refresh the window when the training job completes to see your model's performance evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "28cfdf8e", + "metadata": {}, + "source": [ + "## Serve your model with Vertex AI Prediction: online model predictions and explanations" + ] + }, + { + "cell_type": "markdown", + "id": "0d343de7", + "metadata": {}, + "source": [ + "You have a trained model in GCS now, lets transition to serving your model with Vertex AI Prediction for online model predictions and explanations." + ] + }, + { + "cell_type": "markdown", + "id": "ce14ddf3", + "metadata": {}, + "source": [ + "### 1. Build the Explanation Metadata and Parameters" + ] + }, + { + "cell_type": "markdown", + "id": "02719fa3", + "metadata": {}, + "source": [ + "[**Vertex Explainable AI**](https://cloud.google.com/vertex-ai/docs/explainable-ai) integrates feature attributions into Vertex AI. Vertex Explainable AI helps you understand your model's outputs for classification and regression tasks. Vertex AI tells you how much each feature in the data contributed to the predicted result. You can then use this information to verify that the model is behaving as expected, identify and mitigate biases in your models, and get ideas for ways to improve your model and your training data.\n", + "\n", + "You retrieve these feature attributions to gain insight into your model's CLV predictions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba8decb7", + "metadata": {}, + "outputs": [], + "source": [ + "DEPLOYED_MODEL_DIR = os.path.join(BASE_OUTPUT_DIR, 'model')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48faadfe", + "metadata": {}, + "outputs": [], + "source": [ + "loaded = tf.keras.models.load_model(DEPLOYED_MODEL_DIR)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f10451af", + "metadata": {}, + "outputs": [], + "source": [ + "serving_input = list(\n", + " loaded.signatures[\"serving_default\"].structured_input_signature[1].keys())[0]\n", + "\n", + "serving_output = list(loaded.signatures[\"serving_default\"].structured_outputs.keys())[0]\n", + "\n", + "feature_names = [\n", + " \"n_purchases\",\n", + " \"avg_purchase_size\",\n", + " \"avg_purchase_revenue\",\n", + " \"customer_age\",\n", + " \"days_since_last_purchase\"\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba63105f", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify sampled Shapley feature attribution method with path_count parameter \n", + "# controlling the number of feature permutations to consider when approximating the Shapley values.\n", + "\n", + "explain_params = aiplatform.explain.ExplanationParameters(\n", + " {\"sampled_shapley_attribution\": {\"path_count\": 10}}\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a1cec81", + "metadata": {}, + "outputs": [], + "source": [ + "# https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/ExplanationSpec\n", + "input_metadata = {\n", + " \"input_tensor_name\": serving_input,\n", + " \"encoding\": \"BAG_OF_FEATURES\",\n", + " \"modality\": \"numeric\",\n", + " \"index_feature_mapping\": feature_names,\n", + "}\n", + "\n", + "output_metadata = {\"output_tensor_name\": serving_output}\n", + "\n", + "input_metadata = aiplatform.explain.ExplanationMetadata.InputMetadata(input_metadata)\n", + "output_metadata = aiplatform.explain.ExplanationMetadata.OutputMetadata(output_metadata)\n", + "\n", + "explain_metadata = aiplatform.explain.ExplanationMetadata(\n", + " inputs={\"features\": input_metadata}, outputs={\"medv\": output_metadata}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8692547b", + "metadata": {}, + "source": [ + "## Deploy a Vertex `Endpoint` for online predictions" + ] + }, + { + "cell_type": "markdown", + "id": "2ba9cd05", + "metadata": {}, + "source": [ + "Before you use your model to make predictions, you need to deploy it to an `Endpoint` object. When you deploy a model to an `Endpoint`, you associate physical (machine) resources with that model to enable it to serve online predictions. Online predictions have low latency requirements; providing resources to the model in advance reduces latency. You can do this by calling the deploy function on the `Model` resource. This will do two things:\n", + "\n", + "1. Create an `Endpoint` resource for deploying the `Model` resource to.\n", + "2. Deploy the `Model` resource to the `Endpoint` resource.\n", + "\n", + "The `deploy()` function takes the following parameters:\n", + "\n", + "* `deployed_model_display_name`: A human readable name for the deployed model.\n", + "* `traffic_split`: Percent of traffic at the endpoint that goes to this model, which is specified as a dictionary of one or more key/value pairs. If only one model, then specify as { \"0\": 100 }, where \"0\" refers to this model being uploaded and 100 means 100% of the traffic.\n", + "* `machine_type`: The type of machine to use for training.\n", + "* `accelerator_type`: The hardware accelerator type.\n", + "* `accelerator_count`: The number of accelerators to attach to a worker replica.\n", + "* `starting_replica_count`: The number of compute instances to initially provision.\n", + "* `max_replica_count`: The maximum number of compute instances to scale to. In this lab, only one instance is provisioned.\n", + "* `explanation_parameters`: Metadata to configure the Explainable AI learning method.\n", + "* `explanation_metadata`: Metadata that describes your TensorFlow model for Explainable AI such as features, input and output tensors.\n", + "\n", + "Note: This can take about 15 minutes to provision prediction resources for your model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "726c0e82", + "metadata": {}, + "outputs": [], + "source": [ + "endpoint = model.deploy(\n", + " traffic_split={\"0\": 100},\n", + " machine_type=\"e2-standard-2\",\n", + " explanation_parameters=explain_params,\n", + " explanation_metadata=explain_metadata\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "9bc4f1c7", + "metadata": {}, + "source": [ + "## Get an online prediction and explanation from deployed model" + ] + }, + { + "cell_type": "markdown", + "id": "36aaa774", + "metadata": {}, + "source": [ + "Finally, you use your `Endpoint` to retrieve predictions and feature attributions. This is a customer instance retrieved from the test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "875bab00", + "metadata": {}, + "outputs": [], + "source": [ + "# actual: 3181.04\n", + "test_instance_dict = {\n", + " \"n_purchases\": 2,\n", + " \"avg_purchase_size\": 536.5,\n", + " \"avg_purchase_revenue\": 1132.7,\n", + " \"customer_age\": 123,\n", + " \"days_since_last_purchase\": 32,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "d0946246", + "metadata": {}, + "source": [ + "To request predictions, you call the `predict()` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b9f446c", + "metadata": {}, + "outputs": [], + "source": [ + "endpoint.predict([test_instance_dict])" + ] + }, + { + "cell_type": "markdown", + "id": "4ba59e1d", + "metadata": {}, + "source": [ + "To retrieve explanations (predictions + feature attributions), call the `explain()` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c78e91f", + "metadata": {}, + "outputs": [], + "source": [ + "explanations = endpoint.explain([test_instance_dict])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "999cda11", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame.from_dict(explanations.explanations[0].attributions[0].feature_attributions, orient='index').plot(kind='barh');" + ] + }, + { + "cell_type": "markdown", + "id": "195e9dcc", + "metadata": {}, + "source": [ + "Based on the feature attributions for this prediction, your model has learned that average purchase revenue and customer age had the largest marginal contribution in predicting this customer's monetary value over the 3-month test period. It also identified the relatively lengthy days since last purchase as negatively impacting the prediction. Using these insights, you can plan for an experiment to evaluate targeted marketing interventions for this repeat customer, such as volume discounts, to encourage this customer to purchase more frequently in order to drive additional revenue." + ] + }, + { + "cell_type": "markdown", + "id": "2fc312cf", + "metadata": {}, + "source": [ + "## Next steps" + ] + }, + { + "cell_type": "markdown", + "id": "30ab0ae3", + "metadata": {}, + "source": [ + "Congratulations! In this lab, you walked through a machine learning experimentation workflow using Google Cloud's BigQuery for data storage and analysis and Vertex AI machine learning services to train and deploy a TensorFlow model to predict customer lifetime value. You progressed from training a TensorFlow model locally to training on the cloud with Vertex AI and leveraged several new unified platform capabilities such as Vertex TensorBoard and Explainable AI prediction feature attributions." + ] + }, + { + "cell_type": "markdown", + "id": "0749f152", + "metadata": {}, + "source": [ + "## License" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d2cfd56", + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright 2021 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + } + ], + "metadata": { + "environment": { + "name": "tf2-gpu.2-3.m75", + "type": "gcloud", + "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-3:m75" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/Dockerfile b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/Dockerfile new file mode 100644 index 0000000000..7315c375e4 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/Dockerfile @@ -0,0 +1,21 @@ +# Specifies base image and tag. +# https://cloud.google.com/vertex-ai/docs/training/pre-built-containers +# us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-3:latest +FROM gcr.io/deeplearning-platform-release/tf2-cpu.2-3 + +# Sets the container working directory. +WORKDIR /root + +# Copies the requirements.txt into the container to reduce network calls. +COPY requirements.txt . +# Installs additional packages. +RUN pip3 install -U -r requirements.txt + +# Copies the trainer code to the docker image. +COPY . /trainer + +# Sets the container working directory. +WORKDIR /trainer + +# Sets up the entry point to invoke the trainer. +ENTRYPOINT ["python", "-m", "trainer.task"] diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/cloudbuild.yaml b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/cloudbuild.yaml new file mode 100644 index 0000000000..ab3fbcc27f --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/cloudbuild.yaml @@ -0,0 +1,5 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'us-central1-docker.pkg.dev/dougkelly-vertex-demos/online-retail-clv/dnn-regressor:latest', '.' ] +images: +- 'us-central1-docker.pkg.dev/dougkelly-vertex-demos/online-retail-clv/dnn-regressor:latest' \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/requirements.txt b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/requirements.txt new file mode 100644 index 0000000000..af3b7f30a2 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/requirements.txt @@ -0,0 +1,3 @@ +explainable-ai-sdk==1.3.0 +tensorflow-io==0.16.0 +pyarrow diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/__init__.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/model.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/model.py new file mode 100644 index 0000000000..f2fda59021 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/model.py @@ -0,0 +1,149 @@ +import os +import logging +import tempfile +import tensorflow as tf +from explainable_ai_sdk.metadata.tf.v2 import SavedModelMetadataBuilder +from tensorflow.python.framework import dtypes +from tensorflow_io.bigquery import BigQueryClient +from tensorflow_io.bigquery import BigQueryReadSession + + +# Model feature constants. +NUMERIC_FEATURES = [ + "n_purchases", + "avg_purchase_size", + "avg_purchase_revenue", + "customer_age", + "days_since_last_purchase", +] + +CATEGORICAL_FEATURES = [ + "customer_country" +] + +LABEL = "target_monetary_value_3M" + + +def caip_uri_to_fields(uri): + """Helper function to parse BQ URI.""" + # Remove bq:// prefix. + uri = uri[5:] + project, dataset, table = uri.split('.') + return project, dataset, table + + +def features_and_labels(row_data): + """Helper feature and label mapping function for tf.data.""" + label = row_data.pop(LABEL) + features = row_data + return features, label + + +def read_bigquery(project, dataset, table): + """TensorFlow IO BigQuery Reader.""" + tensorflow_io_bigquery_client = BigQueryClient() + read_session = tensorflow_io_bigquery_client.read_session( + parent="projects/" + project, + project_id=project, + dataset_id=dataset, + table_id=table, + # Pass list of features and label to be selected from BQ. + selected_fields=NUMERIC_FEATURES + [LABEL], + # Provide output TensorFlow data types for features and label. + output_types=[dtypes.int64, dtypes.float64, dtypes.float64, dtypes.int64, dtypes.int64] + [dtypes.float64], + requested_streams=2) + dataset = read_session.parallel_read_rows() + transformed_ds = dataset.map(features_and_labels) + return transformed_ds + + +def rmse(y_true, y_pred): + """Custom RMSE regression metric.""" + return tf.sqrt(tf.reduce_mean(tf.square(y_pred - y_true))) + + +def build_model(hparams): + """Build and compile a TensorFlow Keras DNN Regressor.""" + + feature_columns = [ + tf.feature_column.numeric_column(key=feature) + for feature in NUMERIC_FEATURES + ] + + input_layers = { + feature.key: tf.keras.layers.Input(name=feature.key, shape=(), dtype=tf.float32) + for feature in feature_columns + } + # Keras Functional API: https://keras.io/guides/functional_api + inputs = tf.keras.layers.DenseFeatures(feature_columns, name='inputs')(input_layers) + d1 = tf.keras.layers.Dense(256, activation=tf.nn.relu, name='d1')(inputs) + d2 = tf.keras.layers.Dropout(hparams['dropout'], name='d2')(d1) + # Note: a single neuron scalar output for regression. + output = tf.keras.layers.Dense(1, name='output')(d2) + + model = tf.keras.Model(input_layers, output, name='online-retail-clv') + + optimizer = tf.keras.optimizers.Adam(hparams['learning-rate']) + + # Note: MAE loss is more resistant to outliers than MSE. + model.compile(loss=tf.keras.losses.MAE, + optimizer=optimizer, + metrics=[['mae', 'mse', rmse]]) + + return model + + +def train_evaluate_explain_model(hparams): + """Train, evaluate, explain TensorFlow Keras DNN Regressor. + Args: + hparams(dict): A dictionary containing model training arguments. + Returns: + history(tf.keras.callbacks.History): Keras callback that records training event history. + """ + training_ds = read_bigquery(*caip_uri_to_fields(hparams['training-data-uri'])).prefetch(1).shuffle(hparams['batch-size']*10).batch(hparams['batch-size']).repeat() + eval_ds = read_bigquery(*caip_uri_to_fields(hparams['validation-data-uri'])).prefetch(1).shuffle(hparams['batch-size']*10).batch(hparams['batch-size']) + test_ds = read_bigquery(*caip_uri_to_fields(hparams['test-data-uri'])).prefetch(1).shuffle(hparams['batch-size']*10).batch(hparams['batch-size']) + + model = build_model(hparams) + logging.info(model.summary()) + + tensorboard_callback = tf.keras.callbacks.TensorBoard( + log_dir=hparams['tensorboard-dir'], + histogram_freq=1) + + # Reduce overfitting and shorten training times. + earlystopping_callback = tf.keras.callbacks.EarlyStopping(patience=2) + + # Ensure your training job's resilience to VM restarts. + checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( + filepath= hparams['checkpoint-dir'], + save_weights_only=True, + monitor='val_loss', + mode='min') + + # Virtual epochs design pattern: + # https://medium.com/google-cloud/ml-design-pattern-3-virtual-epochs-f842296de730 + TOTAL_TRAIN_EXAMPLES = int(hparams['stop-point'] * hparams['n-train-examples']) + STEPS_PER_EPOCH = (TOTAL_TRAIN_EXAMPLES // (hparams['batch-size']*hparams['n-checkpoints'])) + + history = model.fit(training_ds, + validation_data=eval_ds, + steps_per_epoch=STEPS_PER_EPOCH, + epochs=hparams['n-checkpoints'], + callbacks=[[tensorboard_callback, + earlystopping_callback, + checkpoint_callback]]) + + logging.info(model.evaluate(test_ds)) + + # Create a temp directory to save intermediate TF SavedModel prior to Explainable metadata creation. + tmpdir = tempfile.mkdtemp() + + # Export Keras model in TensorFlow SavedModel format. + model.save(tmpdir) + + # Annotate and save TensorFlow SavedModel with Explainable metadata to GCS. + builder = SavedModelMetadataBuilder(tmpdir) + builder.save_model_with_metadata(hparams['model-dir']) + + return history diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/task.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/task.py new file mode 100644 index 0000000000..0c03902fa6 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/online-retail-clv-3M/trainer/task.py @@ -0,0 +1,34 @@ +import os +import argparse + +from trainer import model + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + # Vertex custom container training args. These are set by Vertex AI during training can be overwritten. + parser.add_argument('--model-dir', dest='model-dir', + default=os.environ['AIP_MODEL_DIR'], type=str, help='Model dir.') + parser.add_argument('--checkpoint-dir', dest='checkpoint-dir', + default=os.environ['AIP_CHECKPOINT_DIR'], type=str, help='Checkpoint dir set during Vertex AI training.') + parser.add_argument('--tensorboard-dir', dest='tensorboard-dir', + default=os.environ['AIP_TENSORBOARD_LOG_DIR'], type=str, help='Tensorboard dir set during Vertex AI training.') + parser.add_argument('--data-format', dest='data-format', + default=os.environ['AIP_DATA_FORMAT'], type=str, help="Tabular data format set during Vertex AI training. E.g.'csv', 'bigquery'") + parser.add_argument('--training-data-uri', dest='training-data-uri', + default=os.environ['AIP_TRAINING_DATA_URI'], type=str, help='Training data GCS or BQ URI set during Vertex AI training.') + parser.add_argument('--validation-data-uri', dest='validation-data-uri', + default=os.environ['AIP_VALIDATION_DATA_URI'], type=str, help='Validation data GCS or BQ URI set during Vertex AI training.') + parser.add_argument('--test-data-uri', dest='test-data-uri', + default=os.environ['AIP_TEST_DATA_URI'], type=str, help='Test data GCS or BQ URI set during Vertex AI training.') + # Model training args. + parser.add_argument('--learning-rate', dest='learning-rate', default=0.001, type=float, help='Learning rate for optimizer.') + parser.add_argument('--dropout', dest='dropout', default=0.2, type=float, help='Float percentage of DNN nodes [0,1] to drop for regularization.') + parser.add_argument('--batch-size', dest='batch-size', default=16, type=int, help='Number of examples during each training iteration.') + parser.add_argument('--n-train-examples', dest='n-train-examples', default=2638, type=int, help='Number of examples to train on.') + parser.add_argument('--stop-point', dest='stop-point', default=10, type=int, help='Number of passes through the dataset during training to achieve convergence.') + parser.add_argument('--n-checkpoints', dest='n-checkpoints', default=10, type=int, help='Number of model checkpoints to save during training.') + + args = parser.parse_args() + hparams = args.__dict__ + + model.train_evaluate_explain_model(hparams) diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/requirements.txt b/self-paced-labs/vertex-ai/train-deploy-tf-model/requirements.txt new file mode 100644 index 0000000000..cfb9a83554 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/requirements.txt @@ -0,0 +1,12 @@ +tensorflow==2.15.0 +pyarrow==10.0.1 +httplib2>=0.20.4 +grpcio-status>=1.38.1 +google-api-python-client>=1.8.0 +apache-beam>=2.28.0 +google-cloud-aiplatform[tensorboard]>=1.8.0 +six==1.16.0 +wget==3.2 +xlrd==2.0.1 +openpyxl==3.0.10 +pandas >= 1.5 \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/data_download.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/data_download.py new file mode 100644 index 0000000000..1525169733 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/data_download.py @@ -0,0 +1,188 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import logging +import shutil +import wget +import argparse +import pandas as pd +from google.cloud import storage +from google.cloud import bigquery +from google.cloud.exceptions import NotFound, Conflict + +from dataset_schema import table_schema +from dataset_clean import dataset_clean_query +from dataset_ml import dataset_ml_query + +LOCAL_PATH ="./data" +FILENAME = "online_retail" + + +def download_url2gcs(args): + """ + args: + """ + + #set GCS client. + client = storage.Client() + + # Retrieve GCS bucket. + bucket = client.get_bucket(args.GCS_BUCKET) + blob = bucket.blob("data/online_retail.csv") + + #See if file already exists. + if blob.exists() == False: + try: + os.mkdir(LOCAL_PATH) + logging.info('Downloading xlsx file...') + local_xlsx = wget.download(args.URL, out=f"{LOCAL_PATH}/{FILENAME}.xlsx") + logging.info('Converting xlsx -> csv...') + df = pd.read_excel(local_xlsx) + df.to_csv(f"{LOCAL_PATH}/{FILENAME}.csv", index=False) + logging.info('Uploading local csv file to GCS...') + blob.upload_from_filename(f"{LOCAL_PATH}/{FILENAME}.csv") + logging.info('Copied local csv file to GCS.') + # Delete all contents of a directory using shutil.rmtree() and handle exceptions. + try: + shutil.rmtree(LOCAL_PATH) + logging.info('Cleaning up local tmp data directory...') + except: + logging.error('Error while deleting local tmp data directory.') + + #print error if file doesn't exist. + except BaseException as error: + logging.error('An exception occurred: {}'.format(error)) + + #print error if file already exists in GCS. + else: + logging.warning('File already exists in GCS.') + + +def upload_gcs2bq(args, schema): + """ + args: + schema: + """ + # Construct a BigQuery client object. + client = bigquery.Client() + + # Construct a full Dataset object to send to the API. + logging.info('Initializing BigQuery dataset.') + dataset = bigquery.Dataset(f"{args.PROJECT_ID}.{args.BQ_DATASET_NAME}") + + try: + # Send the dataset to the API for creation, with an explicit timeout. + # Raises google.api_core.exceptions.Conflict if the Dataset already + # exists within the project. + dataset = client.create_dataset(dataset, timeout=30) # Make an API request. + # Specify the geographic location where the dataset should reside. + dataset.location = args.BQ_LOCATION + except Conflict: + logging.warning('Dataset %s already exists, not creating.', dataset.dataset_id) + else: + logging.info("Created dataset %s.%s", client.project, dataset.dataset_id) + + try: + URI = f"gs://{args.GCS_BUCKET}/data/{FILENAME}.csv" + RAW_TABLE_ID = f"{args.PROJECT_ID}.{args.BQ_DATASET_NAME}.{args.BQ_RAW_TABLE_NAME}" + + # Load job. + job_config = bigquery.LoadJobConfig( + schema=schema, + skip_leading_rows=1, + allow_jagged_rows=True, + write_disposition="WRITE_TRUNCATE", + source_format=bigquery.SourceFormat.CSV) + load_job = client.load_table_from_uri(source_uris=URI, destination=RAW_TABLE_ID, job_config=job_config) + logging.info('BQ raw dataset load job starting...') + load_job.result() # Waits for the job to complete. + logging.info('BQ raw dataset load job complete.') + except BaseException as error: + logging.error('An exception occurred: {}'.format(error)) + + destination_table = client.get_table(RAW_TABLE_ID) # Make an API request. + logging.info("Loaded %s rows into %s.",destination_table.num_rows, RAW_TABLE_ID) + + +def make_dataset_clean_bq(args, query: str): + """ + args: + query: + """ + client = bigquery.Client() + CLEAN_TABLE_ID = f"{args.PROJECT_ID}.{args.BQ_DATASET_NAME}.{args.BQ_CLEAN_TABLE_NAME}" + RAW_TABLE_ID = f"{args.PROJECT_ID}.{args.BQ_DATASET_NAME}.{args.BQ_RAW_TABLE_NAME}" + + clean_query = query.replace("@CLEAN_TABLE_ID", CLEAN_TABLE_ID).replace("@RAW_TABLE_ID", RAW_TABLE_ID) + + logging.info('BQ make clean dataset starting...') + try: + job = client.query(clean_query) + _ = job.result() + logging.info('BQ make clean dataset complete') + except BaseException as error: + logging.error('An exception occurred: {}'.format(error)) + + destination_table = client.get_table(CLEAN_TABLE_ID) # Make an API request. + logging.info("Loaded %s rows into %s.",destination_table.num_rows, CLEAN_TABLE_ID) + + +def make_dataset_ml_bq(args, query: str): + """ + args: + query: + """ + client = bigquery.Client() + ML_TABLE_ID = f"{args.PROJECT_ID}.{args.BQ_DATASET_NAME}.{args.BQ_ML_TABLE_NAME}" + CLEAN_TABLE_ID = f"{args.PROJECT_ID}.{args.BQ_DATASET_NAME}.{args.BQ_CLEAN_TABLE_NAME}" + + ml_query = query.replace("@ML_TABLE_ID", ML_TABLE_ID).replace("@CLEAN_TABLE_ID", CLEAN_TABLE_ID) + + logging.info('BQ make ML dataset starting...') + try: + job = client.query(ml_query) + _ = job.result() + logging.info('BQ make ML dataset complete') + except BaseException as error: + logging.error('An exception occurred: {}'.format(error)) + + destination_table = client.get_table(ML_TABLE_ID) # Make an API request. + logging.info("Loaded %s rows into %s.",destination_table.num_rows, ML_TABLE_ID) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--PROJECT_ID", dest="PROJECT_ID", type=str, required=True) + parser.add_argument("--GCS_BUCKET", dest="GCS_BUCKET", type=str, required=True) + parser.add_argument("--URL", dest="URL", type=str, required=True) + parser.add_argument("--BQ_DATASET_NAME", dest="BQ_DATASET_NAME", type=str, default="online_retail") + parser.add_argument("--BQ_LOCATION", dest="BQ_LOCATION", type=str, default="US") + parser.add_argument("--BQ_RAW_TABLE_NAME", dest="BQ_RAW_TABLE_NAME", type=str, default="online_retail_clv_raw") + parser.add_argument("--BQ_CLEAN_TABLE_NAME", dest="BQ_CLEAN_TABLE_NAME", type=str, default="online_retail_clv_clean") + parser.add_argument("--BQ_ML_TABLE_NAME", dest="BQ_ML_TABLE_NAME", type=str, default="online_retail_clv_ml") + + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="\n %(asctime)s [%(levelname)s] %(message)s", + handlers=[logging.StreamHandler()] + ) + + download_url2gcs(args) + upload_gcs2bq(args, table_schema) + make_dataset_clean_bq(args, dataset_clean_query) + make_dataset_ml_bq(args, dataset_ml_query) \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_clean.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_clean.py new file mode 100644 index 0000000000..b99c6acf9f --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_clean.py @@ -0,0 +1,49 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TODO.""" + +dataset_clean_query = """ +CREATE OR REPLACE TABLE `@CLEAN_TABLE_ID` +AS ( +WITH + customer_daily_sales AS ( + SELECT + CustomerID AS customer_id, + Country AS customer_country, + EXTRACT(DATE FROM InvoiceDate) AS order_date, + COUNT(DISTINCT InvoiceNo) AS n_purchases, + SUM(Quantity) AS order_qty, + ROUND(SUM(UnitPrice * Quantity), 2) AS revenue + FROM + `@RAW_TABLE_ID` + WHERE + CustomerID IS NOT NULL + AND Quantity > 0 + GROUP BY + customer_id, + customer_country, + order_date) + +SELECT + customer_id, + customer_country, + order_date, + n_purchases, + order_qty, + revenue +FROM + customer_daily_sales +) + +""" \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_ml.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_ml.py new file mode 100644 index 0000000000..84d4e3e30d --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_ml.py @@ -0,0 +1,72 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dataset_ml_query = """ +CREATE OR REPLACE TABLE `@ML_TABLE_ID` +AS ( +WITH +-- Calculate features before CUTOFF_DATE date. + features AS ( + SELECT + customer_id, + customer_country, + COUNT(n_purchases) AS n_purchases, + AVG(order_qty) AS avg_purchase_size, + AVG(revenue) AS avg_purchase_revenue, + DATE_DIFF(MAX(order_date), MIN(order_date), DAY) AS customer_age, + DATE_DIFF(DATE('2011-09-01'), MAX(order_date), DAY) AS days_since_last_purchase + FROM + `@CLEAN_TABLE_ID` + WHERE + order_date <= DATE('2011-09-01') + GROUP BY + customer_id, + customer_country), + + -- Calculate customer target monetary value over historical period + 3M future period. + label AS ( + SELECT + customer_id, + SUM(revenue) AS target_monetary_value_3M + FROM + `@CLEAN_TABLE_ID` + WHERE + order_date < DATE('2011-12-01') + GROUP BY + customer_id + ) + +SELECT + features.customer_id, + features.customer_country, + features.n_purchases, -- frequency + features.avg_purchase_size, --monetary + features.avg_purchase_revenue, --monetary + features.customer_age, + features.days_since_last_purchase, --recency + label.target_monetary_value_3M, --target + CASE + WHEN MOD(ABS(FARM_FINGERPRINT(CAST(features.customer_id AS STRING))), 10) < 8 + THEN 'TRAIN' + WHEN MOD(ABS(FARM_FINGERPRINT(CAST(features.customer_id AS STRING))), 10) = 9 + THEN 'VALIDATE' + ELSE + 'TEST' END AS data_split +FROM + features +INNER JOIN label + ON features.customer_id = label.customer_id +); +""" \ No newline at end of file diff --git a/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_schema.py b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_schema.py new file mode 100644 index 0000000000..f12cccb9f2 --- /dev/null +++ b/self-paced-labs/vertex-ai/train-deploy-tf-model/utils/dataset_schema.py @@ -0,0 +1,27 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery + + +table_schema = [ + bigquery.SchemaField("InvoiceNo", "STRING"), + bigquery.SchemaField("StockCode", "STRING"), + bigquery.SchemaField("Description", "STRING", mode="NULLABLE"), + bigquery.SchemaField("Quantity", "INTEGER"), + bigquery.SchemaField("InvoiceDate", "TIMESTAMP"), + bigquery.SchemaField("UnitPrice", "FLOAT"), + bigquery.SchemaField("CustomerID", "STRING", mode="NULLABLE"), + bigquery.SchemaField("Country", "STRING"), +] \ No newline at end of file