From 691fc82681b0a5d6bd304e1bbd541d979ab4b326 Mon Sep 17 00:00:00 2001 From: ddeleo Date: Tue, 12 Oct 2021 16:55:37 -0400 Subject: [PATCH] Add missing phone_assertions.js file Javscript and Markdown formatting Upgrade dataform cli version in package.json to 1.20.0 --- .../dataform_assertion_unit_test/README.md | 55 +++++++-- .../dataform.json | 2 +- .../definitions/tests/test_date_assertions.js | 36 +++--- .../tests/test_email_assertions.js | 26 ++-- .../tests/test_gender_assertions.js | 22 ++-- .../tests/test_marital_status_assertions.js | 22 ++-- .../tests/test_personal_info_assertions.js | 28 ++--- .../tests/test_telephone_number_assertions.js | 40 +++---- .../includes/date_assertions.js | 60 +++++----- .../includes/personal_info_assertions.js | 112 +++++++++--------- .../includes/phone_assertions.js | 62 ++++++++++ .../includes/unit_test_utils.js | 38 +++--- .../dataform_assertion_unit_test/package.json | 6 +- 13 files changed, 299 insertions(+), 210 deletions(-) create mode 100644 dataform/examples/dataform_assertion_unit_test/includes/phone_assertions.js diff --git a/dataform/examples/dataform_assertion_unit_test/README.md b/dataform/examples/dataform_assertion_unit_test/README.md index f9a600ae1..dd4b8fe5c 100644 --- a/dataform/examples/dataform_assertion_unit_test/README.md +++ b/dataform/examples/dataform_assertion_unit_test/README.md @@ -1,12 +1,28 @@ # Dataform Custom Assertions -Dataform is a platform to manage data in Big Query and other data warehouses and dataform perform the **T (Transformation)** in the **ELT** Pipeline. But before any transformaiton can be done, we need to make sure our input data is valid. Dataform has many built in data assertions such as uniqueKey, nonNull, and etc. But you can also customize your dataform assertions to meet individual's needs. This directory gives you a set of custom assertions that you can use in testing your data quailty in your project. -## Requirement -In order to test and run the custom dataform assertions, the person has to have have: + +Dataform is a platform to manage data in Big Query and other data warehouses and +dataform perform the **T (Transformation)** in the **ELT** Pipeline. But before +any transformaiton can be done, we need to make sure our input data is valid. +Dataform has many built in data assertions such as uniqueKey, nonNull, and etc. +But you can also customize your dataform assertions to meet individual's needs. +This directory gives you a set of custom assertions that you can use in testing +your data quailty in your project. + +## Requirement + +In order to test and run the custom dataform assertions, the person has to have +have: + - A dataform project - Credentials granting access to bigquery warehouse .df-credentials ## Usage -The custom assertions are in javascript files in the includes folder. To test them, simply refer to the function in your transformation query. For example, if a person wants to use the ```test_telephone_number_digits(colName)``` assertions in the ```phone_assertions.js``` file, the person only needs to refer in the config section of the transformation query like: + +The custom assertions are in javascript files in the includes folder. To test +them, simply refer to the function in your transformation query. For example, if +a person wants to use the ```test_telephone_number_digits(colName)``` assertions +in the ```phone_assertions.js``` file, the person only needs to refer in the +config section of the transformation query like: ``` type: ...., @@ -20,13 +36,23 @@ The custom assertions are in javascript files in the includes folder. To test th ] } ``` + ## Unit Testing Custom Assertions -Unit testing your custom assertions is important because it helps you safeguard your ELT pipeline. In this project, we want to demonstrate an easy way for you to unit test your custom assertions. The workflow is simple and as listed below: -* Create a ```test_[NAME_YOUR_TEST]_assertions.js``` file in the ```definitions/tests/``` folder if your custom row assertions are not included in the existing template. -* In ```test_[NAME_YOUR_TEST]_assertions.js``` change the code snippets ```const {[YOUR_CUSTOM_ASSERTION]} = [CUSTOMER_ASSERTION_FILE_NAME];``` and change the test name ```const test_name = "[YOUR_TEST_NAME]";```. -* Add the testing data in the ```test_cases``` block with the following format ```"[INPUT]" : "[EXPECTED_OUTPUT]"``` -* Finally supply your custom function name in the ```generatetest(...)``` function. +Unit testing your custom assertions is important because it helps you safeguard +your ELT pipeline. In this project, we want to demonstrate an easy way for you +to unit test your custom assertions. The workflow is simple and as listed below: + +* Create a ```test_[NAME_YOUR_TEST]_assertions.js``` file in + the ```definitions/tests/``` folder if your custom row assertions are not + included in the existing template. +* In ```test_[NAME_YOUR_TEST]_assertions.js``` change the code + snippets ```const {[YOUR_CUSTOM_ASSERTION]} = [CUSTOMER_ASSERTION_FILE_NAME];``` + and change the test name ```const test_name = "[YOUR_TEST_NAME]";```. +* Add the testing data in the ```test_cases``` block with the following + format ```"[INPUT]" : "[EXPECTED_OUTPUT]"``` +* Finally supply your custom function name in the ```generatetest(...)``` + function. Below is an example of the ```test_[NAME_YOUR_TEST]_assertions.js``` file: @@ -54,7 +80,12 @@ generate_test(test_name, [YOUT_CUSTOM_ASSERTIONS]); ``` -* Afterwards you can perform the unit test by running ```dataform test``` command. - +* Afterwards you can perform the unit test by running ```dataform test``` + command. + ## Liscense -All solutions within this repository are provided under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. Please see the [LICENSE](https://www.apache.org/licenses/LICENSE-2.0) file for more detailed terms and conditions. \ No newline at end of file + +All solutions within this repository are provided under +the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. Please +see the [LICENSE](https://www.apache.org/licenses/LICENSE-2.0) file for more +detailed terms and conditions. \ No newline at end of file diff --git a/dataform/examples/dataform_assertion_unit_test/dataform.json b/dataform/examples/dataform_assertion_unit_test/dataform.json index 7b1a49913..f8548688d 100644 --- a/dataform/examples/dataform_assertion_unit_test/dataform.json +++ b/dataform/examples/dataform_assertion_unit_test/dataform.json @@ -3,5 +3,5 @@ "defaultSchema": "dataform", "assertionSchema": "dataform_assertions", "defaultDatabase": "YOUR_PROJECT_ID", - "useRunCache": false + "concurrentQueryLimit": 100 } diff --git a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_date_assertions.js b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_date_assertions.js index 6db077bde..69e47e77d 100644 --- a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_date_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_date_assertions.js @@ -16,25 +16,23 @@ const {generate_test} = unit_test_utils; const {test_date} = date_assertions; const test_name = "test_date_assertions"; const test_cases = { - /* - Provide your own testing data following the structure - : "" - For example, if a testing data has the to be TRUE, - then the program will expect the custom data quality rules to also produce TRUE. - Otherwise it will show that the custom data quality rules failed. - */ - - "1997/11/03" : "TRUE", - "2008/08/08" : "TRUE", - "1996/11/03" : "TRUE", - "2005/04/13" : "TRUE", - "1998/11/03" : "TRUE", - "2006/07/29" : "TRUE", - "2025/03/24" : "FALSE", - "1769/03/24" : "FALSE" + /* + Provide your own testing data following the structure + : "" + For example, if a testing data has the to be TRUE, + then the program will expect the custom data quality rules to also produce TRUE. + Otherwise it will show that the custom data quality rules failed. + */ + + "1997/11/03": "TRUE", + "2008/08/08": "TRUE", + "1996/11/03": "TRUE", + "2005/04/13": "TRUE", + "1998/11/03": "TRUE", + "2006/07/29": "TRUE", + "2025/03/24": "FALSE", + "1769/03/24": "FALSE" }; // The function below will generate the necessary SQL to run unit tests. -generate_test(test_name, - test_cases, - test_date); +generate_test(test_name, test_cases, test_date); \ No newline at end of file diff --git a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_email_assertions.js b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_email_assertions.js index 717f862e8..957969f11 100644 --- a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_email_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_email_assertions.js @@ -16,20 +16,18 @@ const {generate_test} = unit_test_utils; const {test_email_validity} = personal_info_assertions; const test_name = "test_email_assertion_test"; const test_cases = { - /* - Provide your own testing data following the structure - : "" - For example, if a testing data has the to be TRUE, - then the program will expect the custom data quality rules to also produce TRUE. - Otherwise it will show that the custom data quality rules failed. - */ - - "ruinanliu@google.com" : "TRUE", - "among.us@amongus.net" : "TRUE", - "1736#$%.com" : "FALSE" + /* + Provide your own testing data following the structure + : "" + For example, if a testing data has the to be TRUE, + then the program will expect the custom data quality rules to also produce TRUE. + Otherwise it will show that the custom data quality rules failed. + */ + + "someone@google.com": "TRUE", + "among.us@amongus.net": "TRUE", + "1736#$%.com": "FALSE" }; // The function below will generate the necessary SQL to run unit tests. -generate_test(test_name, - test_cases, - test_email_validity); +generate_test(test_name, test_cases, test_email_validity); \ No newline at end of file diff --git a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_gender_assertions.js b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_gender_assertions.js index e2014be26..25ce90cc9 100644 --- a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_gender_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_gender_assertions.js @@ -16,17 +16,17 @@ const {generate_test} = unit_test_utils; const {test_gender_status} = personal_info_assertions; const test_name = "test_gemder_assertions"; const test_cases = { - /* - Provide your own testing data following the structure - : "" - For example, if a testing data has the to be TRUE, - then the program will expect the custom data quality rules to also produce TRUE. - Otherwise it will show that the custom data quality rules failed. - */ - - "Female" : "TRUE", - "Male" : "TRUE", - "one" : "FALSE" + /* + Provide your own testing data following the structure + : "" + For example, if a testing data has the to be TRUE, + then the program will expect the custom data quality rules to also produce TRUE. + Otherwise it will show that the custom data quality rules failed. + */ + + "Female": "TRUE", + "Male": "TRUE", + "one": "FALSE" }; // The function below will generate the necessary SQL to run unit tests. generate_test(test_name, diff --git a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_marital_status_assertions.js b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_marital_status_assertions.js index 4f6d43fc9..394ddc9bb 100644 --- a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_marital_status_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_marital_status_assertions.js @@ -16,17 +16,17 @@ const {generate_test} = unit_test_utils; const {test_marital_status} = personal_info_assertions; const test_name = "test_marital_status_assertions"; const test_cases = { - /* - Provide your own testing data following the structure - : "" - For example, if a testing data has the to be TRUE, - then the program will expect the custom data quality rules to also produce TRUE. - Otherwise it will show that the custom data quality rules failed. - */ - - "Married" : "TRUE", - "Divorced" : "TRUE", - "Happy" : "FALSE" + /* + Provide your own testing data following the structure + : "" + For example, if a testing data has the to be TRUE, + then the program will expect the custom data quality rules to also produce TRUE. + Otherwise it will show that the custom data quality rules failed. + */ + + "Married": "TRUE", + "Divorced": "TRUE", + "Happy": "FALSE" }; // The function below will generate the necessary SQL to run unit tests. generate_test(test_name, diff --git a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_personal_info_assertions.js b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_personal_info_assertions.js index 67b984eae..58d3973a8 100644 --- a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_personal_info_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_personal_info_assertions.js @@ -16,20 +16,20 @@ const {generate_test} = unit_test_utils; const {test_name} = personal_info_assertions; const test_file_name = "test_personal_info_assertions"; const test_cases = { - /* - Provide your own testing data following the structure - : "" - For example, if a testing data has the to be TRUE, - then the program will expect the custom data quality rules to also produce TRUE. - Otherwise it will show that the custom data quality rules failed. - */ - - "Alan" : "TRUE", - "Bob" : "TRUE", - "Jack" : "TRUE", - "John" : "TRUE", - "y*(*&^^%$" : "FALSE", - "Alannnn" : "FALSE" + /* + Provide your own testing data following the structure + : "" + For example, if a testing data has the to be TRUE, + then the program will expect the custom data quality rules to also produce TRUE. + Otherwise it will show that the custom data quality rules failed. + */ + + "Alan": "TRUE", + "Bob": "TRUE", + "Jack": "TRUE", + "John": "TRUE", + "y*(*&^^%$": "FALSE", + "Alannnn": "FALSE" }; // The function below will generate the necessary SQL to run unit tests. generate_test(test_file_name, diff --git a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_telephone_number_assertions.js b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_telephone_number_assertions.js index e702e2046..ce69ff9d8 100644 --- a/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_telephone_number_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/definitions/tests/test_telephone_number_assertions.js @@ -16,26 +16,26 @@ const {generate_test} = unit_test_utils; const {test_phone_number} = phone_assertions; const test_name = "test_telephone_number_assertions"; const test_cases = { - /* - Provide your own testing data following the structure - : "" - For example, if a testing data has the to be TRUE, - then the program will expect the custom data quality rules to also produce TRUE. - Otherwise it will show that the custom data quality rules failed. - */ - - "8123456789" : "TRUE", - "1234567899" : "TRUE", - "5123456789" : "TRUE", - "4576839485" : "TRUE", - "2938475638" : "TRUE", - "7928374657" : "TRUE", - "7847563738" : "TRUE", - "6768907654" : "TRUE", - "1234567" : "FALSE", - "0123456789" : "FALSE", - "1111111111" : "FALSE", - "374657389a" : "FALSE" + /* + Provide your own testing data following the structure + : "" + For example, if a testing data has the to be TRUE, + then the program will expect the custom data quality rules to also produce TRUE. + Otherwise it will show that the custom data quality rules failed. + */ + + "8123456789": "TRUE", + "1234567899": "TRUE", + "5123456789": "TRUE", + "4576839485": "TRUE", + "2938475638": "TRUE", + "7928374657": "TRUE", + "7847563738": "TRUE", + "6768907654": "TRUE", + "1234567": "FALSE", + "0123456789": "FALSE", + "1111111111": "FALSE", + "374657389a": "FALSE" }; // The function below will generate the necessary SQL to run unit tests. generate_test(test_name, diff --git a/dataform/examples/dataform_assertion_unit_test/includes/date_assertions.js b/dataform/examples/dataform_assertion_unit_test/includes/date_assertions.js index 9ae90547e..9d905b677 100644 --- a/dataform/examples/dataform_assertion_unit_test/includes/date_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/includes/date_assertions.js @@ -15,53 +15,53 @@ /* This assertion checks whether input date is future */ -function test_future_date(colName){ - var result_query = `PARSE_DATE('%Y/%m/%d', ${colName}) < CURRENT_DATE()` - return result_query +function test_future_date(colName) { + var result_query = `PARSE_DATE('%Y/%m/%d', ${colName}) < CURRENT_DATE()` + return result_query } /* This assertion checks whether the input birthdate is less than 100 yrs old */ -function test_valid_years(colName){ - var result_query = `DATE_DIFF(CURRENT_DATE(), PARSE_DATE('%Y/%m/%d', ${colName}), YEAR) < 100` - return result_query +function test_valid_years(colName) { + var result_query = `DATE_DIFF(CURRENT_DATE(), PARSE_DATE('%Y/%m/%d', ${colName}), YEAR) < 100` + return result_query } /* This function checks whether the format of the date is correct */ -function test_date_format(colName, date_format){ - if(date_format == "yyyy/mm/dd"){ - var result_query = `REGEXP_CONTAINS(${colName}, r'^[0-9]{4}[/][0-9]{2}[/][0-9]{2}$')` - return result_query - } else if (date_format == "yyyymmdd"){ - var result_query = `REGEXP_CONTAINS(${colName}, r'^[0-9]{4}[0-9]{2}[0-9]{2}$')` - return result_query - }else{ - return `FALSE` - } +function test_date_format(colName, date_format) { + if (date_format == "yyyy/mm/dd") { + var result_query = `REGEXP_CONTAINS(${colName}, r'^[0-9]{4}[/][0-9]{2}[/][0-9]{2}$')` + return result_query + } else if (date_format == "yyyymmdd") { + var result_query = `REGEXP_CONTAINS(${colName}, r'^[0-9]{4}[0-9]{2}[0-9]{2}$')` + return result_query + } else { + return `FALSE` + } } /* This assertions combines custom assertions for testing future date and valid years */ -function test_date(colName){ - var result_query = - `IF(${colName} IS NOT NULL AND ${colName} <> "",` + - `IF(${test_date_format(colName, "yyyy/mm/dd")}, ` + - `IF(${test_future_date(colName)}, ` + - `${test_valid_years(colName, 100)}` + - `, FALSE),` + - `IF(${test_date_format(colName, "yyyymmdd")}, ` + - `TRUE, FALSE)), FALSE)` - return result_query +function test_date(colName) { + var result_query = + `IF(${colName} IS NOT NULL AND ${colName} <> "",` + + `IF(${test_date_format(colName, "yyyy/mm/dd")}, ` + + `IF(${test_future_date(colName)}, ` + + `${test_valid_years(colName, 100)}` + + `, FALSE),` + + `IF(${test_date_format(colName, "yyyymmdd")}, ` + + `TRUE, FALSE)), FALSE)` + return result_query } module.exports = { - test_future_date, - test_valid_years, - test_date_format, - test_date + test_future_date, + test_valid_years, + test_date_format, + test_date } diff --git a/dataform/examples/dataform_assertion_unit_test/includes/personal_info_assertions.js b/dataform/examples/dataform_assertion_unit_test/includes/personal_info_assertions.js index 71d063623..2a64fee92 100644 --- a/dataform/examples/dataform_assertion_unit_test/includes/personal_info_assertions.js +++ b/dataform/examples/dataform_assertion_unit_test/includes/personal_info_assertions.js @@ -15,93 +15,93 @@ /* This assertion checks whether the input email format is valid */ -function test_email_validity(colName){ - var result_query = `REGEXP_CONTAINS(${colName}, r'^[\\w.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$')` - return result_query +function test_email_validity(colName) { + var result_query = `REGEXP_CONTAINS(${colName}, r'^[\\w.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$')` + return result_query } /* This assertion checks whether the input marital status is within an acceptable list */ -function test_marital_status(colName){ - var marital_list = "'Married', 'Single', 'Divorced', 'Widowed'" - var result_query = `${colName} IN(${marital_list})` - return result_query +function test_marital_status(colName) { + var marital_list = "'Married', 'Single', 'Divorced', 'Widowed'" + var result_query = `${colName} IN(${marital_list})` + return result_query } /* This assertion checks whether the input gender status is within an acceptable list */ -function test_gender_status(colName){ - var gender_list = "'Female','Male','Transgender Female','Transgender Male','Gender Variant','Prefer Not to Say'" - var result_query = `${colName} IN (${gender_list})` - return result_query +function test_gender_status(colName) { + var gender_list = "'Female','Male','Transgender Female','Transgender Male','Gender Variant','Prefer Not to Say'" + var result_query = `${colName} IN (${gender_list})` + return result_query } /* This assertion checks whether the name is valid and only contain characters and numbers */ -function test_name_validity(colName){ - var result_query = `REGEXP_CONTAINS(${colName}, r'^[a-zA-Z]+$')` - return result_query +function test_name_validity(colName) { + var result_query = `REGEXP_CONTAINS(${colName}, r'^[a-zA-Z]+$')` + return result_query } /* This assertions compares with other input column to check whether the last name is unique */ -function test_last_name_unique(colName1, colName2){ - var result_query = `${colName1} != ${colName2}` - return result_query +function test_last_name_unique(colName1, colName2) { + var result_query = `${colName1} != ${colName2}` + return result_query } /* The assertion checks that no name contain more than n repeated characters */ -function test_same_character_not_more_than_n_times(colName, n_times){ - var regex = `(A{${n_times + 1},})+|` + - `(B{${n_times + 1},})+|` + - `(C{${n_times + 1},})+|` + - `(D{${n_times + 1},})+|` + - `(E{${n_times + 1},})+|` + - `(F{${n_times + 1},})+|` + - `(G{${n_times + 1},})+|` + - `(H{${n_times + 1},})+|` + - `(I{${n_times + 1},})+|` + - `(J{${n_times + 1},})+|` + - `(K{${n_times + 1},})+|` + - `(L{${n_times + 1},})+|` + - `(M{${n_times + 1},})+|` + - `(N{${n_times + 1},})+|` + - `(O{${n_times + 1},})+|` + - `(P{${n_times + 1},})+|` + - `(Q{${n_times + 1},})+|` + - `(R{${n_times + 1},})+|` + - `(S{${n_times + 1},})+|` + - `(T{${n_times + 1},})+|` + - `(U{${n_times + 1},})+|` + - `(V{${n_times + 1},})+|` + - `(W{${n_times + 1},})+|` + - `(X{${n_times + 1},})+|` + - `(Y{${n_times + 1},})+|` + - `(Z{${n_times + 1},})+` - var query_result = `NOT REGEXP_CONTAINS(UPPER(${colName}), r'${regex}')` - return query_result +function test_same_character_not_more_than_n_times(colName, n_times) { + var regex = `(A{${n_times + 1},})+|` + + `(B{${n_times + 1},})+|` + + `(C{${n_times + 1},})+|` + + `(D{${n_times + 1},})+|` + + `(E{${n_times + 1},})+|` + + `(F{${n_times + 1},})+|` + + `(G{${n_times + 1},})+|` + + `(H{${n_times + 1},})+|` + + `(I{${n_times + 1},})+|` + + `(J{${n_times + 1},})+|` + + `(K{${n_times + 1},})+|` + + `(L{${n_times + 1},})+|` + + `(M{${n_times + 1},})+|` + + `(N{${n_times + 1},})+|` + + `(O{${n_times + 1},})+|` + + `(P{${n_times + 1},})+|` + + `(Q{${n_times + 1},})+|` + + `(R{${n_times + 1},})+|` + + `(S{${n_times + 1},})+|` + + `(T{${n_times + 1},})+|` + + `(U{${n_times + 1},})+|` + + `(V{${n_times + 1},})+|` + + `(W{${n_times + 1},})+|` + + `(X{${n_times + 1},})+|` + + `(Y{${n_times + 1},})+|` + + `(Z{${n_times + 1},})+` + var query_result = `NOT REGEXP_CONTAINS(UPPER(${colName}), r'${regex}')` + return query_result } /* This assertions combines custom assertions for names */ -function test_name(colName, n_times){ - var result_query = `${test_name_validity(colName)} AND ${test_same_character_not_more_than_n_times(colName, 3)}` - return result_query +function test_name(colName, n_times) { + var result_query = `${test_name_validity(colName)} AND ${test_same_character_not_more_than_n_times(colName, 3)}` + return result_query } module.exports = { - test_email_validity, - test_marital_status, - test_gender_status, - test_name_validity, - test_last_name_unique, - test_same_character_not_more_than_n_times, - test_name + test_email_validity, + test_marital_status, + test_gender_status, + test_name_validity, + test_last_name_unique, + test_same_character_not_more_than_n_times, + test_name } diff --git a/dataform/examples/dataform_assertion_unit_test/includes/phone_assertions.js b/dataform/examples/dataform_assertion_unit_test/includes/phone_assertions.js new file mode 100644 index 000000000..6f1b90641 --- /dev/null +++ b/dataform/examples/dataform_assertion_unit_test/includes/phone_assertions.js @@ -0,0 +1,62 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + This assertion checks that the the input telephone number does not begin with 0 +*/ +function test_telephone_number_start_with_zero(colName) { + var result_query = `SUBSTRING(${colName},1,1) != '0'` + return result_query +} + +/* + This assertion checks that the input telephone number does not contain more than all repeated digits +*/ +function test_repeated_phone_number(colName) { + var result_query = `TRIM(${colName}, '0') != "" AND ` + + `TRIM(${colName}, '1') != "" AND ` + + `TRIM(${colName}, '2') != "" AND ` + + `TRIM(${colName}, '3') != "" AND ` + + `TRIM(${colName}, '4') != "" AND ` + + `TRIM(${colName}, '5') != "" AND ` + + `TRIM(${colName}, '6') != "" AND ` + + `TRIM(${colName}, '7') != "" AND ` + + `TRIM(${colName}, '8') != "" AND ` + + `TRIM(${colName}, '9') != ""` + return result_query +} + +/* + This assertion checks that the telephone only contains 10 digits +*/ +function test_only_contain_ten_digit(colName) { + var result_query = `REGEXP_CONTAINS(${colName}, r'\\d{10}')` + return result_query +} + +/* + This assertions combines the above assertions +*/ +function test_phone_number(colName) { + var result_query = `${test_telephone_number_start_with_zero(colName)} AND ` + + `${test_repeated_phone_number(colName)} AND ${test_only_contain_ten_digit(colName)}` + return result_query +} + +module.exports = { + test_telephone_number_start_with_zero, + test_repeated_phone_number, + test_only_contain_ten_digit, + test_phone_number, +} diff --git a/dataform/examples/dataform_assertion_unit_test/includes/unit_test_utils.js b/dataform/examples/dataform_assertion_unit_test/includes/unit_test_utils.js index 686b5b725..bd2ee816f 100644 --- a/dataform/examples/dataform_assertion_unit_test/includes/unit_test_utils.js +++ b/dataform/examples/dataform_assertion_unit_test/includes/unit_test_utils.js @@ -12,28 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -function generate_test(test_name, test_cases, data_quality_function){ - publish(test_name) - .type("view") - .query(ctx => ` - SELECT - ${data_quality_function("test_input")} AS is_valid - FROM ${ctx.resolve("test_inputs")} - `); +function generate_test(test_name, test_cases, data_quality_function) { + publish(test_name) + .type("view") + .query(ctx => ` + SELECT ${data_quality_function("test_input")} AS is_valid + FROM ${ctx.resolve("test_inputs")} + `); - let expected_output_select_statements = []; - let test_input_select_statements = []; - for(var test_case in test_cases) { - test_input_select_statements.push(`SELECT "${test_case}" AS test_input`); - expected_output_select_statements.push(`SELECT ${test_cases[test_case]} AS is_valid`); - }; + let expected_output_select_statements = []; + let test_input_select_statements = []; + for (var test_case in test_cases) { + test_input_select_statements.push(`SELECT "${test_case}" AS test_input`); + expected_output_select_statements.push(`SELECT ${test_cases[test_case]} AS is_valid`); + } - test(test_name) - .dataset(test_name) - .input("test_inputs", `${test_input_select_statements.join(' UNION ALL\n')}`) - .expect(`${expected_output_select_statements.join(' UNION ALL\n')}`); + + test(test_name) + .dataset(test_name) + .input("test_inputs", `${test_input_select_statements.join(' UNION ALL\n')}`) + .expect(`${expected_output_select_statements.join(' UNION ALL\n')}`); } module.exports = { - generate_test, + generate_test, } diff --git a/dataform/examples/dataform_assertion_unit_test/package.json b/dataform/examples/dataform_assertion_unit_test/package.json index df648e973..7d6a2304c 100644 --- a/dataform/examples/dataform_assertion_unit_test/package.json +++ b/dataform/examples/dataform_assertion_unit_test/package.json @@ -1,5 +1,5 @@ { - "dependencies": { - "@dataform/core": "1.18.0" - } + "dependencies": { + "@dataform/core": "1.20.0" + } }