From 93fe69a8c564317234ebae1a7092d437c5824c7b Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Mon, 20 Dec 2021 19:36:49 +0530 Subject: [PATCH 1/7] PMM-7 Adding playbooks --- playbooks/mysql.yml | 139 ++++++++++++++++++++++++++++ playbooks/queries/mysql/mariadb.sql | 2 + playbooks/queries/mysql/mysql.sql | 2 + playbooks/queries/mysql/percona.sql | 5 + playbooks/queries/mysql/users.sql | 6 ++ 5 files changed, 154 insertions(+) create mode 100644 playbooks/mysql.yml create mode 100644 playbooks/queries/mysql/mariadb.sql create mode 100644 playbooks/queries/mysql/mysql.sql create mode 100644 playbooks/queries/mysql/percona.sql create mode 100644 playbooks/queries/mysql/users.sql diff --git a/playbooks/mysql.yml b/playbooks/mysql.yml new file mode 100644 index 00000000..ec084a4b --- /dev/null +++ b/playbooks/mysql.yml @@ -0,0 +1,139 @@ +--- +- name: Running MYSQL Database as Docker Container using Ansible. + hosts: localhost + vars: + mysqldockerdb: "{{ lookup('vars', 'extra_mysqldockerdb', default=lookup('env','MYSQL_CONTAINER') | default('mysql57-db', true) ) }}" + mysqlport: "{{ lookup('vars', 'extra_mysqlport', default=lookup('env','MYSQL_PORT') | default('43306', true) ) }}" + mysqlversion: "{{ lookup('vars', 'extra_mysqlversion', default=lookup('env','MYSQL_VERSION') | default('5.7', true) ) }}" + mysqldistribution: "{{ lookup('vars', 'extra_mysqldistribution', default=lookup('env','MYSQL_DISTRIBUTION') | default('mysql', true) ) }}" + mysqlhost: 127.0.0.1 + tasks: + - name: Check for mysql container already running, kill it + shell: > + docker ps -q --filter "name={{ mysqldockerdb }}" | grep -q . && docker stop {{ mysqldockerdb }} && docker rm -fv {{ mysqldockerdb }} + ignore_errors: true + + - name: Check for mysql container created previously, kill it + shell: > + docker rm {{ mysqldockerdb }} + ignore_errors: true + + - name: delete network if exist + shell: docker network rm "{{ mysqldockerdb }}_network" + ignore_errors: true + + - name: Create a network + shell: docker network create "{{ mysqldockerdb }}_network" + + - name: Delete slowlog file if exist from previous runs + file: + path: "/tmp/mysql/{{ mysqldockerdb }}/mysql_log.log" + state: absent + become: yes + tags: + - slowlog-setup + + - name: ensure slowlog directory exist + command: mkdir -p /tmp/mysql/{{ mysqldockerdb }}/ + tags: + - slowlog-setup + + - name: ensure slowlog file is writable by Docker container + command: chmod 777 -R /tmp/mysql/{{ mysqldockerdb }}/ + become: yes + tags: + - slowlog-setup + + - name: Setting Source for Container based on Distribution requested by user [mysql] + set_fact: db="mysql" + when: mysqldistribution == "mysql" + + - name: Setting Source for Container based on Distribution requested by user [percona] + set_fact: db="percona" + when: mysqldistribution == "percona" + + - name: Setting Source for Container based on Distribution requested by user [mariadb] + set_fact: db="mariadb" + when: mysqldistribution == "mariadb" + + - name: Launch MySQL Database Container + shell: > + docker run --name {{ mysqldockerdb }} -e MYSQL_ROOT_PASSWORD=password + -p {{ mysqlport }}:3306 + -v /tmp/mysql/{{ mysqldockerdb }}:/tmp/mysql/{{ mysqldockerdb }} + -v "/$(pwd)/queries/mysql/users.sql:/docker-entrypoint-initdb.d/1.sql" + -v "/$(pwd)/queries/mysql/{{ db }}.sql:/docker-entrypoint-initdb.d/2.sql" + --network "{{ mysqldockerdb }}_network" + -d {{ db }}:{{ mysqlversion }} + --slow-query-log-file=/tmp/mysql/{{ mysqldockerdb }}/mysql_log.log + --net_read_timeout=100 + --slow-query-log='ON' + --long-query-time=0 + --log-slow-admin-statements='ON' + --log-slow-slave-statements='ON' + tags: + - container-mysql + + - name: Wait for Container to be ready + shell: docker logs {{ mysqldockerdb }} 2>&1 | grep "ready for connections." + register: result + until: result.rc == 0 + retries: 10 + delay: 5 + + - name: Pause for 30 seconds for container buildup + pause: + seconds: 30 + + - name: Check for mysql slow log query + shell: docker exec {{ mysqldockerdb }} mysql -u root -ppassword -e "SET GLOBAL slow_query_log='ON';" + tags: + - slowlog-setup + + - name: Enable Query Response time plugin for Percona Server 5.x + shell: docker exec {{ mysqldockerdb }} mysql -u root -ppassword -e "{{ item }}" + with_items: + - "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" + - "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" + - "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" + - "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" + - "SET GLOBAL query_response_time_stats=ON;" + when: mysqlversion is search("5") and db == "percona" + ignore_errors: true + tags: + - slowlog-setup + + - name: setup sysbench for Mysql DB deployment + shell: > + docker run --rm=true --name=sb-prepare --network "{{ mysqldockerdb }}_network" severalnines/sysbench sysbench + --db-driver=mysql + --oltp-table-size=100000 + --oltp-tables-count=24 + --threads=1 + --mysql-host={{ mysqldockerdb }} + --mysql-port=3306 + --mysql-user=sbtest + --mysql-password=sbtestsecret + /usr/share/sysbench/tests/include/oltp_legacy/parallel_prepare.lua run + tags: + - sysbench-mysql + + - name: Check for sysbench container for this DB already running, kill it + shell: > + docker stop sb{{ mysqldockerdb }}-run && docker rm -fv sb{{ mysqldockerdb }}-run + ignore_errors: true + + - name: run sysbench Read & Write Load for mysql DB + shell: > + docker run --name="sb{{ mysqldockerdb }}-run" -d --network "{{ mysqldockerdb }}_network" severalnines/sysbench + sysbench --db-driver=mysql --report-interval=2 --mysql-table-engine=innodb --oltp-table-size=100000 + --oltp-tables-count=24 + --threads=64 + --time=99999 + --mysql-host={{ mysqldockerdb }} + --mysql-port=3306 + --mysql-user=sbtest + --mysql-password=sbtestsecret + /usr/share/sysbench/tests/include/oltp_legacy/oltp.lua run + tags: + - sysbench-mysql \ No newline at end of file diff --git a/playbooks/queries/mysql/mariadb.sql b/playbooks/queries/mysql/mariadb.sql new file mode 100644 index 00000000..38fc0071 --- /dev/null +++ b/playbooks/queries/mysql/mariadb.sql @@ -0,0 +1,2 @@ +UPDATE performance_schema.setup_instruments SET ENABLED = 'YES', TIMED = 'YES' WHERE NAME LIKE 'statement/%'; +UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%'; \ No newline at end of file diff --git a/playbooks/queries/mysql/mysql.sql b/playbooks/queries/mysql/mysql.sql new file mode 100644 index 00000000..e3235335 --- /dev/null +++ b/playbooks/queries/mysql/mysql.sql @@ -0,0 +1,2 @@ +ALTER USER 'pmm'@'%' IDENTIFIED WITH mysql_native_password BY 'secret'; +ALTER USER 'sbtest'@'%' IDENTIFIED WITH mysql_native_password BY 'sbtestsecret'; \ No newline at end of file diff --git a/playbooks/queries/mysql/percona.sql b/playbooks/queries/mysql/percona.sql new file mode 100644 index 00000000..49c6c5c8 --- /dev/null +++ b/playbooks/queries/mysql/percona.sql @@ -0,0 +1,5 @@ +ALTER USER 'pmm'@'%' IDENTIFIED WITH mysql_native_password BY 'secret'; +ALTER USER 'sbtest'@'%' IDENTIFIED WITH mysql_native_password BY 'sbtestsecret'; +SET GLOBAL userstat=1; +SET GLOBAL innodb_monitor_enable=all; +SET GLOBAL log_slow_rate_limit=1; \ No newline at end of file diff --git a/playbooks/queries/mysql/users.sql b/playbooks/queries/mysql/users.sql new file mode 100644 index 00000000..9d189aaf --- /dev/null +++ b/playbooks/queries/mysql/users.sql @@ -0,0 +1,6 @@ +DROP USER IF EXISTS 'pmm'@'%'; +CREATE USER 'pmm'@'%' IDENTIFIED BY 'secret'; +GRANT SELECT, PROCESS, REPLICATION CLIENT, RELOAD ON *.* TO 'pmm'@'%'; +CREATE SCHEMA sbtest; +CREATE USER 'sbtest'@'%' IDENTIFIED BY 'sbtestsecret'; +GRANT ALL PRIVILEGES ON sbtest.* to 'sbtest'@'%'; From fe21a400dfbd4e270fccb96beed0f4f8d1951164 Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Mon, 20 Dec 2021 19:39:05 +0530 Subject: [PATCH 2/7] PMM-7 Adding extra line --- playbooks/mysql.yml | 2 +- playbooks/queries/mysql/mariadb.sql | 2 +- playbooks/queries/mysql/mysql.sql | 2 +- playbooks/queries/mysql/percona.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/mysql.yml b/playbooks/mysql.yml index ec084a4b..2a068b65 100644 --- a/playbooks/mysql.yml +++ b/playbooks/mysql.yml @@ -136,4 +136,4 @@ --mysql-password=sbtestsecret /usr/share/sysbench/tests/include/oltp_legacy/oltp.lua run tags: - - sysbench-mysql \ No newline at end of file + - sysbench-mysql diff --git a/playbooks/queries/mysql/mariadb.sql b/playbooks/queries/mysql/mariadb.sql index 38fc0071..eadf6596 100644 --- a/playbooks/queries/mysql/mariadb.sql +++ b/playbooks/queries/mysql/mariadb.sql @@ -1,2 +1,2 @@ UPDATE performance_schema.setup_instruments SET ENABLED = 'YES', TIMED = 'YES' WHERE NAME LIKE 'statement/%'; -UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%'; \ No newline at end of file +UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%'; diff --git a/playbooks/queries/mysql/mysql.sql b/playbooks/queries/mysql/mysql.sql index e3235335..09510037 100644 --- a/playbooks/queries/mysql/mysql.sql +++ b/playbooks/queries/mysql/mysql.sql @@ -1,2 +1,2 @@ ALTER USER 'pmm'@'%' IDENTIFIED WITH mysql_native_password BY 'secret'; -ALTER USER 'sbtest'@'%' IDENTIFIED WITH mysql_native_password BY 'sbtestsecret'; \ No newline at end of file +ALTER USER 'sbtest'@'%' IDENTIFIED WITH mysql_native_password BY 'sbtestsecret'; diff --git a/playbooks/queries/mysql/percona.sql b/playbooks/queries/mysql/percona.sql index 49c6c5c8..f9de9f25 100644 --- a/playbooks/queries/mysql/percona.sql +++ b/playbooks/queries/mysql/percona.sql @@ -2,4 +2,4 @@ ALTER USER 'pmm'@'%' IDENTIFIED WITH mysql_native_password BY 'secret'; ALTER USER 'sbtest'@'%' IDENTIFIED WITH mysql_native_password BY 'sbtestsecret'; SET GLOBAL userstat=1; SET GLOBAL innodb_monitor_enable=all; -SET GLOBAL log_slow_rate_limit=1; \ No newline at end of file +SET GLOBAL log_slow_rate_limit=1; From 62907dcc416bcf99b8213870e99fa4b37f5661b7 Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Mon, 20 Dec 2021 21:20:36 +0530 Subject: [PATCH 3/7] PMM-7 Refactor mysql-setup --- playbooks/mysql.yml | 26 ++++++++++++++++++++++++++ pmm-tests/Dockerfile | 1 + 2 files changed, 27 insertions(+) diff --git a/playbooks/mysql.yml b/playbooks/mysql.yml index 2a068b65..48c3c3ca 100644 --- a/playbooks/mysql.yml +++ b/playbooks/mysql.yml @@ -6,6 +6,7 @@ mysqlport: "{{ lookup('vars', 'extra_mysqlport', default=lookup('env','MYSQL_PORT') | default('43306', true) ) }}" mysqlversion: "{{ lookup('vars', 'extra_mysqlversion', default=lookup('env','MYSQL_VERSION') | default('5.7', true) ) }}" mysqldistribution: "{{ lookup('vars', 'extra_mysqldistribution', default=lookup('env','MYSQL_DISTRIBUTION') | default('mysql', true) ) }}" + mysqldataload: "{{ lookup('vars', 'extra_mysqldataload', default=lookup('env','MYSQL_DATALOAD') | default('pmmworkload', true) ) }}" mysqlhost: 127.0.0.1 tasks: - name: Check for mysql container already running, kill it @@ -115,13 +116,17 @@ --mysql-user=sbtest --mysql-password=sbtestsecret /usr/share/sysbench/tests/include/oltp_legacy/parallel_prepare.lua run + when: mysqldataload == "sysbench" tags: - sysbench-mysql - name: Check for sysbench container for this DB already running, kill it shell: > docker stop sb{{ mysqldockerdb }}-run && docker rm -fv sb{{ mysqldockerdb }}-run + when: mysqldataload == "sysbench" ignore_errors: true + tags: + - sysbench-mysql - name: run sysbench Read & Write Load for mysql DB shell: > @@ -135,5 +140,26 @@ --mysql-user=sbtest --mysql-password=sbtestsecret /usr/share/sysbench/tests/include/oltp_legacy/oltp.lua run + when: mysqldataload == "sysbench" tags: - sysbench-mysql + + - name: run QAN Workload Script for High Bucket size + shell: > + docker run --rm --name="pmm{{ mysqldockerdb }}-run" -d + --network "{{ mysqldockerdb }}_network" + -e MYSQL_HOST={{ mysqldockerdb }} + -e MYSQL_PORT=3306 + -e MYSQL_USER=root + -e MYSQL_PASSWORD=password + -e TEST_TARGET_QPS=1000 + -e TEST_QUERIES=100 + -e TEST_TABLES=50 + -e TEST_TABLES=10 + -v /srv/pmm-qa/pmm-tests:/usr/src/myapp + -w /usr/src/myapp + php-db + php schema_table_query.php + when: mysqldataload == "pmmworkload" + tags: + - pmm-workload diff --git a/pmm-tests/Dockerfile b/pmm-tests/Dockerfile index dca92c76..dcdf66c3 100644 --- a/pmm-tests/Dockerfile +++ b/pmm-tests/Dockerfile @@ -3,6 +3,7 @@ FROM php:7.4-alpine RUN set -ex \ && apk --no-cache add postgresql-libs postgresql-dev \ && docker-php-ext-install pgsql pdo_pgsql \ + && docker-php-ext-install mysqli && docker-php-ext-enable mysqli \ && apk del postgresql-dev RUN apk --update add --virtual build-dependencies build-base openssl-dev autoconf \ From c592195ba9c006ee187370d891bf0ac863471b71 Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Tue, 21 Dec 2021 15:14:47 +0530 Subject: [PATCH 4/7] PMM-7 Adding postgresql and moving workload scripts as part of playbooks --- playbooks/mysql.yml | 12 +- playbooks/postgresql.yml | 146 ++++++++++++++++++ playbooks/queries/postgresql/initialize.sql | 16 ++ {pmm-tests => playbooks/workload}/Dockerfile | 0 .../workload}/mongodb_query.php | 0 .../workload}/pgsql_schema_table_query.php | 0 .../workload}/schema_table_query.php | 0 7 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 playbooks/postgresql.yml create mode 100644 playbooks/queries/postgresql/initialize.sql rename {pmm-tests => playbooks/workload}/Dockerfile (100%) rename {pmm-tests => playbooks/workload}/mongodb_query.php (100%) rename {pmm-tests => playbooks/workload}/pgsql_schema_table_query.php (100%) rename {pmm-tests => playbooks/workload}/schema_table_query.php (100%) diff --git a/playbooks/mysql.yml b/playbooks/mysql.yml index 48c3c3ca..5e909c9a 100644 --- a/playbooks/mysql.yml +++ b/playbooks/mysql.yml @@ -5,7 +5,7 @@ mysqldockerdb: "{{ lookup('vars', 'extra_mysqldockerdb', default=lookup('env','MYSQL_CONTAINER') | default('mysql57-db', true) ) }}" mysqlport: "{{ lookup('vars', 'extra_mysqlport', default=lookup('env','MYSQL_PORT') | default('43306', true) ) }}" mysqlversion: "{{ lookup('vars', 'extra_mysqlversion', default=lookup('env','MYSQL_VERSION') | default('5.7', true) ) }}" - mysqldistribution: "{{ lookup('vars', 'extra_mysqldistribution', default=lookup('env','MYSQL_DISTRIBUTION') | default('mysql', true) ) }}" + mysqldistribution: "{{ lookup('vars', 'extra_mysqldistribution', default=lookup('env','MYSQL_DISTRIBUTION') | default('percona', true) ) }}" mysqldataload: "{{ lookup('vars', 'extra_mysqldataload', default=lookup('env','MYSQL_DATALOAD') | default('pmmworkload', true) ) }}" mysqlhost: 127.0.0.1 tasks: @@ -144,6 +144,14 @@ tags: - sysbench-mysql + - name: Check if PMM Workload container for this Deployment already running, kill it + shell: > + docker stop pmm{{ mysqldockerdb }}-run && docker rm -fv pmm{{ mysqldockerdb }}-run + when: mysqldataload == "pmmworkload" + ignore_errors: true + tags: + - pmm-workload + - name: run QAN Workload Script for High Bucket size shell: > docker run --rm --name="pmm{{ mysqldockerdb }}-run" -d @@ -155,7 +163,7 @@ -e TEST_TARGET_QPS=1000 -e TEST_QUERIES=100 -e TEST_TABLES=50 - -e TEST_TABLES=10 + -e TEST_SCHEMAS=10 -v /srv/pmm-qa/pmm-tests:/usr/src/myapp -w /usr/src/myapp php-db diff --git a/playbooks/postgresql.yml b/playbooks/postgresql.yml new file mode 100644 index 00000000..52d765f2 --- /dev/null +++ b/playbooks/postgresql.yml @@ -0,0 +1,146 @@ +--- +- name: Running Postgresql Database as Docker Container using Ansible. + hosts: localhost + vars: + pgsqldockerdb: "{{ lookup('vars', 'extra_pgsqldockerdb', default=lookup('env','PGSQL_CONTAINER') | default('pgsql14db', true) ) }}" + pgsqlport: "{{ lookup('vars', 'extra_pgsqlport', default=lookup('env','PGSQL_PORT') | default('45432', true) ) }}" + pgsqlversion: "{{ lookup('vars', 'extra_pgsqlversion', default=lookup('env','PGSQL_VERSION') | default('14.1', true) ) }}" + pgsqldistribution: "{{ lookup('vars', 'extra_pgsqldistribution', default=lookup('env','PGSQL_DISTRIBUTION') | default('pdpgsql', true) ) }}" + pgsqldataload: "{{ lookup('vars', 'extra_pgsqldataload', default=lookup('env','PGSQL_DATALOAD') | default('pmmworkload', true) ) }}" + pgsqlhost: 127.0.0.1 + tasks: + - name: Check for pgsql container already running, kill it + shell: > + docker ps -q --filter "name={{ pgsqldockerdb }}" | grep -q . && docker stop {{ pgsqldockerdb }} && docker rm -fv {{ pgsqldockerdb }} + ignore_errors: true + + - name: Check for pgsql container created previously, kill it + shell: > + docker rm {{ pgsqldockerdb }} + ignore_errors: true + + - name: delete network if exist + shell: docker network rm "{{ pgsqldockerdb }}_network" + ignore_errors: true + + - name: Create a network + docker_network: + name: "{{ pgsqldockerdb }}_network" + + - name: Launch Official Postgresql database container + shell: > + docker run --name {{ pgsqldockerdb }} + -p {{ pgsqlport }}:5432 + -e POSTGRES_PASSWORD=password + -v "/$(pwd)/queries/postgresql:/docker-entrypoint-initdb.d/:rw" + --network "{{ pgsqldockerdb }}_network" + -d postgres:{{ pgsqlversion }} + -c shared_preload_libraries=pg_stat_statements + -c track_activity_query_size=2048 + -c pg_stat_statements.max=10000 + -c pg_stat_statements.track=all + -c pg_stat_statements.save=off + -c track_io_timing=on + -c password_encryption=md5 + when: pgsqldistribution == "pgsql" + + - name: Launch Percona Postgresql database container + shell: > + docker run --name {{ pgsqldockerdb }} + -p {{ pgsqlport }}:5432 + -e POSTGRES_PASSWORD=password + -v "/$(pwd)/queries/postgresql:/docker-entrypoint-initdb.d/:rw" + --network "{{ pgsqldockerdb }}_network" + -d perconalab/percona-distribution-postgresql:{{ pgsqlversion }} + -c shared_preload_libraries=pg_stat_monitor,pg_stat_statements + -c track_activity_query_size=2048 + -c pg_stat_statements.max=10000 + -c pg_stat_monitor.pgsm_normalized_query=0 + -c pg_stat_monitor.pgsm_query_max_len=10000 + -c pg_stat_monitor.pgsm_enable_query_plan=1 + -c pg_stat_statements.track=all + -c pg_stat_statements.save=off + -c track_io_timing=on + -c password_encryption=md5 + when: pgsqldistribution == "pdpgsql" + + - name: Wait for Container to be ready + shell: docker logs {{ pgsqldockerdb }} 2>&1 | grep "PostgreSQL init process complete; ready for start up." + register: result + until: result.rc == 0 + retries: 10 + delay: 5 + + - name: Pause for 30 seconds for container buildup + pause: + seconds: 30 + + - name: setup sysbench for Postgres DB deployment + shell: > + docker run --rm=true --name=sb-prepare --network "{{ pgsqldockerdb }}_network" severalnines/sysbench sysbench + --db-driver=pgsql + --oltp-table-size=100000 + --oltp-tables-count=24 + --threads=1 + --pgsql-host={{ pgsqldockerdb }} + --pgsql-port=5432 + --pgsql-user=sbtest + --pgsql-password=sbtestsecret + /usr/share/sysbench/tests/include/oltp_legacy/parallel_prepare.lua run + when: pgsqldataload == "sysbench" + ignore_errors: true + tags: + - sysbench-pgsql + + - name: Check for sysbench container for this DB already running, kill it + shell: > + docker stop sb{{ pgsqldockerdb }}-run && docker rm -fv sb{{ pgsqldockerdb }}-run + when: pgsqldataload == "sysbench" + ignore_errors: true + tags: + - sysbench-pgsql + + - name: run sysbench Read & Write Load for postgresql DB + shell: > + docker run --name="sb{{ pgsqldockerdb }}-run" -d --network "{{ pgsqldockerdb }}_network" severalnines/sysbench + sysbench --db-driver=pgsql --report-interval=2 --oltp-table-size=100000 + --oltp-tables-count=24 + --threads=64 + --time=99999 + --pgsql-host={{ pgsqldockerdb }} + --pgsql-port=5432 + --pgsql-user=sbtest + --pgsql-password=sbtestsecret + /usr/share/sysbench/tests/include/oltp_legacy/oltp.lua run + when: pgsqldataload == "sysbench" + ignore_errors: true + tags: + - sysbench-pgsql + + - name: Check if PMM Workload container for this PGSQL Deployment already running, kill it + shell: > + docker stop pmm{{ pgsqldockerdb }}-run && docker rm -fv pmm{{ pgsqldockerdb }}-run + when: pgsqldataload == "pmmworkload" + ignore_errors: true + tags: + - pmm-workload + + - name: run QAN Workload Script for Postgresql + shell: > + docker run --rm --name="pmm{{ pgsqldockerdb }}-run" -d + --network "{{ pgsqldockerdb }}_network" + -e PGSQL_HOST={{ pgsqldockerdb }} + -e PGSQL_PORT=5432 + -e PGSQL_USER=root + -e PGSQL_PASSWORD=password + -e TEST_TARGET_QPS=1000 + -e TEST_QUERIES=100 + -e TEST_TABLES=50 + -e TEST_SCHEMAS=10 + -v /srv/pmm-qa/pmm-tests:/usr/src/myapp + -w /usr/src/myapp + php-db + php pgsql_schema_table_query.php + when: pgsqldataload == "pmmworkload" + tags: + - pmm-workload \ No newline at end of file diff --git a/playbooks/queries/postgresql/initialize.sql b/playbooks/queries/postgresql/initialize.sql new file mode 100644 index 00000000..5a912bb0 --- /dev/null +++ b/playbooks/queries/postgresql/initialize.sql @@ -0,0 +1,16 @@ +create extension pg_stat_statements; +ALTER SYSTEM SET track_io_timing=ON; +SELECT pg_reload_conf(); +CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD 'secret'; +GRANT pg_monitor to pmm; +CREATE DATABASE pmm; +SELECT pg_reload_conf(); +create table dummy_table(name varchar(20),address text,age int); +insert into dummy_table values('XYZ','location-A',25); +insert into dummy_table values('ABC','location-B',35); +insert into dummy_table values('DEF','location-C',40); +insert into dummy_table values('PQR','location-D',54); +select * from dummy_table; +update dummy_table set age=50 where name='PQR'; +select * from dummy_table; +delete from dummy_table where age=50; diff --git a/pmm-tests/Dockerfile b/playbooks/workload/Dockerfile similarity index 100% rename from pmm-tests/Dockerfile rename to playbooks/workload/Dockerfile diff --git a/pmm-tests/mongodb_query.php b/playbooks/workload/mongodb_query.php similarity index 100% rename from pmm-tests/mongodb_query.php rename to playbooks/workload/mongodb_query.php diff --git a/pmm-tests/pgsql_schema_table_query.php b/playbooks/workload/pgsql_schema_table_query.php similarity index 100% rename from pmm-tests/pgsql_schema_table_query.php rename to playbooks/workload/pgsql_schema_table_query.php diff --git a/pmm-tests/schema_table_query.php b/playbooks/workload/schema_table_query.php similarity index 100% rename from pmm-tests/schema_table_query.php rename to playbooks/workload/schema_table_query.php From b86ec326be3651567228e626cdb82f0d55fa56cb Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Wed, 22 Dec 2021 16:46:48 +0530 Subject: [PATCH 5/7] PMM-7 fix bug with workload script --- playbooks/mysql.yml | 11 ++++++++++- playbooks/postgresql.yml | 13 +++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/playbooks/mysql.yml b/playbooks/mysql.yml index 5e909c9a..cacbc8b3 100644 --- a/playbooks/mysql.yml +++ b/playbooks/mysql.yml @@ -7,6 +7,7 @@ mysqlversion: "{{ lookup('vars', 'extra_mysqlversion', default=lookup('env','MYSQL_VERSION') | default('5.7', true) ) }}" mysqldistribution: "{{ lookup('vars', 'extra_mysqldistribution', default=lookup('env','MYSQL_DISTRIBUTION') | default('percona', true) ) }}" mysqldataload: "{{ lookup('vars', 'extra_mysqldataload', default=lookup('env','MYSQL_DATALOAD') | default('pmmworkload', true) ) }}" + workloadscripts: "{{ lookup('vars', 'extra_workloadscripts', default=lookup('env','WORKLOAD_SCRIPTS') | default('/$(pwd)/workload/', true) ) }}" mysqlhost: 127.0.0.1 tasks: - name: Check for mysql container already running, kill it @@ -152,6 +153,14 @@ tags: - pmm-workload + - name: Build the Workload container if not already build + shell: > + touch docker-build.log | docker build --tag php-db {{ workloadscripts }} > docker-build.log 2>&1 + when: mysqldataload == "pmmworkload" + ignore_errors: true + tags: + - pmm-workload + - name: run QAN Workload Script for High Bucket size shell: > docker run --rm --name="pmm{{ mysqldockerdb }}-run" -d @@ -164,7 +173,7 @@ -e TEST_QUERIES=100 -e TEST_TABLES=50 -e TEST_SCHEMAS=10 - -v /srv/pmm-qa/pmm-tests:/usr/src/myapp + -v "{{ workloadscripts }}:/usr/src/myapp" -w /usr/src/myapp php-db php schema_table_query.php diff --git a/playbooks/postgresql.yml b/playbooks/postgresql.yml index 52d765f2..098828bd 100644 --- a/playbooks/postgresql.yml +++ b/playbooks/postgresql.yml @@ -7,6 +7,7 @@ pgsqlversion: "{{ lookup('vars', 'extra_pgsqlversion', default=lookup('env','PGSQL_VERSION') | default('14.1', true) ) }}" pgsqldistribution: "{{ lookup('vars', 'extra_pgsqldistribution', default=lookup('env','PGSQL_DISTRIBUTION') | default('pdpgsql', true) ) }}" pgsqldataload: "{{ lookup('vars', 'extra_pgsqldataload', default=lookup('env','PGSQL_DATALOAD') | default('pmmworkload', true) ) }}" + workloadscripts: "{{ lookup('vars', 'extra_workloadscripts', default=lookup('env','WORKLOAD_SCRIPTS') | default('/$(pwd)/workload/', true) ) }}" pgsqlhost: 127.0.0.1 tasks: - name: Check for pgsql container already running, kill it @@ -125,19 +126,27 @@ tags: - pmm-workload + - name: Build the Workload container if not already build + shell: > + touch docker-build.log | docker build --tag php-db {{ workloadscripts }} > docker-build.log 2>&1 + when: pgsqldataload == "pmmworkload" + ignore_errors: true + tags: + - pmm-workload + - name: run QAN Workload Script for Postgresql shell: > docker run --rm --name="pmm{{ pgsqldockerdb }}-run" -d --network "{{ pgsqldockerdb }}_network" -e PGSQL_HOST={{ pgsqldockerdb }} -e PGSQL_PORT=5432 - -e PGSQL_USER=root + -e PGSQL_USER=postgres -e PGSQL_PASSWORD=password -e TEST_TARGET_QPS=1000 -e TEST_QUERIES=100 -e TEST_TABLES=50 -e TEST_SCHEMAS=10 - -v /srv/pmm-qa/pmm-tests:/usr/src/myapp + -v "{{ workloadscripts }}:/usr/src/myapp" -w /usr/src/myapp php-db php pgsql_schema_table_query.php From dba280b9f89a46a797655d48969cf1e3a386095d Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Tue, 28 Dec 2021 12:32:44 +0530 Subject: [PATCH 6/7] PMM-7 Refactor setup scripts --- playbooks/postgresql.yml | 3 +- .../queries/mongodb}/mongodb_user_setup.js | 1 + .../setup_scripts/mongodb/mongodb_setup.sh | 46 +++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) rename {pmm-tests => playbooks/queries/mongodb}/mongodb_user_setup.js (96%) create mode 100755 playbooks/setup_scripts/mongodb/mongodb_setup.sh diff --git a/playbooks/postgresql.yml b/playbooks/postgresql.yml index 098828bd..8a6586dc 100644 --- a/playbooks/postgresql.yml +++ b/playbooks/postgresql.yml @@ -25,8 +25,7 @@ ignore_errors: true - name: Create a network - docker_network: - name: "{{ pgsqldockerdb }}_network" + shell: docker network create "{{ pgsqldockerdb }}_network" - name: Launch Official Postgresql database container shell: > diff --git a/pmm-tests/mongodb_user_setup.js b/playbooks/queries/mongodb/mongodb_user_setup.js similarity index 96% rename from pmm-tests/mongodb_user_setup.js rename to playbooks/queries/mongodb/mongodb_user_setup.js index 7ed6044a..0e5dd255 100644 --- a/pmm-tests/mongodb_user_setup.js +++ b/playbooks/queries/mongodb/mongodb_user_setup.js @@ -1,3 +1,4 @@ +db.setProfilingLevel(2) db.getSiblingDB("admin").createRole({ role: "explainRole", privileges: [{ diff --git a/playbooks/setup_scripts/mongodb/mongodb_setup.sh b/playbooks/setup_scripts/mongodb/mongodb_setup.sh new file mode 100755 index 00000000..b7ed4e81 --- /dev/null +++ b/playbooks/setup_scripts/mongodb/mongodb_setup.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export MONGODB_VERSION=$1 +export TYPE_OF_DEPLOYMENT=$2 + +apt-get update +apt-get install wget tar git -y +wget https://github.com/feliixx/mgodatagen/releases/download/v0.9.4/mgodatagen_0.9.4_Linux_x86_64.tar.gz +tar -xvf mgodatagen_0.9.4_Linux_x86_64.tar.gz +export MONGODB_BINARY=$(wget -qO- https://www.percona.com/downloads/percona-server-mongodb-${MONGODB_VERSION}/LATEST/binary/ | grep -Eo "(http|https)://[a-zA-Z0-9./?=_%:-]*-minimal.tar.gz" | sort -u) +wget ${MONGODB_BINARY} + +wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh + +tar -xvf percona-server-mongodb-${MONGODB_VERSION}*.tar.gz +rm percona-server-mongodb-${MONGODB_VERSION}*.tar.gz +mv percona-server-mongodb-${MONGODB_VERSION}* psmdb_${MONGODB_VERSION} + +if [[ "$TYPE_OF_DEPLOYMENT" == "sharding" ]]; then + echo "Setting up Sharded Cluster"; + bash ./mongo_startup.sh -s -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=$(pwd)/psmdb_${MONGODB_VERSION}/bin +elif [[ "$TYPE_OF_DEPLOYMENT" == "replicaset" ]]; then + echo "Setting up replicaset"; + bash ./mongo_startup.sh -r -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=$(pwd)/psmdb_${MONGODB_VERSION}/bin +else + echo "Setting up Regular Deployment"; + bash ./mongo_startup.sh -m -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=$(pwd)/psmdb_${MONGODB_VERSION}/bin +fi + +wget https://raw.githubusercontent.com/feliixx/mgodatagen/master/datagen/generators/testdata/full-bson.json +sleep 10 +./nodes/cl_primary.sh /tmp/mongodb/mongodb_user_setup.js +echo "MongoDB setup Ready for Connection, waiting for connections on port 27017"; + +#for j in {1..2} +# do +# for i in {1..10} +# do +# cp full-bson.json full-bson-$i.json +# sed "s/test_bson/test_bson_${i}/g" -i full-bson-$i.json +# sed "s/mgodatagen_test/mgodatagen_test_${j}/g" -i full-bson-$i.json +# ./mgodatagen -f full-bson-$i.json +# rm -f full-bson-$i.json +# sleep 20 +# done +#done From 9c6c1c2a66c2299e3c0e1dc7cbea03b6adb5f87f Mon Sep 17 00:00:00 2001 From: puneet0191 Date: Tue, 28 Dec 2021 12:34:37 +0530 Subject: [PATCH 7/7] PMM-7 adding mongodb setup playbook --- playbooks/mongodb.yml | 170 ++++++++++++++++++++++ playbooks/workload/mongodb_query.php | 0 playbooks/workload/schema_table_query.php | 0 3 files changed, 170 insertions(+) create mode 100644 playbooks/mongodb.yml mode change 100644 => 100755 playbooks/workload/mongodb_query.php mode change 100644 => 100755 playbooks/workload/schema_table_query.php diff --git a/playbooks/mongodb.yml b/playbooks/mongodb.yml new file mode 100644 index 00000000..19acaa9c --- /dev/null +++ b/playbooks/mongodb.yml @@ -0,0 +1,170 @@ +--- +- name: Running Mongodb Database as Container using Ansible. + hosts: localhost + vars: + mongodbdockerdb: "{{ lookup('vars', 'extra_mongodbdockerdb', default=lookup('env','MONGODB_CONTAINER') | default('mongo44db', true) ) }}" + mongodbportrange: "{{ lookup('vars', 'extra_mongodbportrange', default=lookup('env','MONGODB_PORTRANGE') | default('27017-27019', true) ) }}" + mongodbversion: "{{ lookup('vars', 'extra_mongodbversion', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}" + mongodbsetup: "{{ lookup('vars', 'extra_mongodbsetup', default=lookup('env','MONGODB_SETUP') | default('replicaset', true) ) }}" + mongodbdistribution: "{{ lookup('vars', 'extra_mongodbdistribution', default=lookup('env','MONGODB_DISTRIBUTION') | default('psmdb', true) ) }}" + mongodbdataload: "{{ lookup('vars', 'extra_mongodbdataload', default=lookup('env','MONGODB_DATALOAD') | default('pmmworkload', true) ) }}" + workloadscripts: "{{ lookup('vars', 'extra_workloadscripts', default=lookup('env','WORKLOAD_SCRIPTS') | default('/$(pwd)/workload/', true) ) }}" + mongodbhost: 127.0.0.1 + tasks: + - name: Check for mongodb container already running, kill it + shell: > + docker ps -q --filter "name={{ mongodbdockerdb }}" | grep -q . && docker stop {{ mongodbdockerdb }} && docker rm -fv {{ mongodbdockerdb }} + ignore_errors: true + tags: + - cleanup + + - name: Check for mongodb container created previously, kill it + shell: > + docker rm {{ mongodbdockerdb }} + ignore_errors: true + tags: + - cleanup + + - name: delete network if exist + shell: docker network rm "{{ mongodbdockerdb }}_network" + ignore_errors: true + tags: + - cleanup + + - name: Create a network + shell: docker network create "{{ mongodbdockerdb }}_network" + + - name: Prepare volume for official mongodb container + shell: "{{ item }}" + with_items: + - mkdir -p /tmp/modb_{{ mongodbdockerdb }} + - chown -R $USER:$USER /tmp/modb_{{ mongodbdockerdb }} > /dev/null 2>&1 + - chmod -R go+w /tmp/modb_{{ mongodbdockerdb }} > /dev/null 2>&1 + when: mongodbdistribution == "modb" + become: yes + + - name: Cleanup the tmp on local + shell: rm -r /tmp/modb_{{ mongodbdockerdb }} + tags: ['never', 'cleanup' ] + become: yes + ignore_errors: true + + - name: Launch Official Mongodb database container + shell: > + docker run -d + -p {{ mongodbportrange }}:27017-27019 + --network "{{ mongodbdockerdb }}_network" + -v "/tmp/modb_{{ mongodbdockerdb }}/:/tmp/" + -v "/$(pwd)/queries/mongodb:/tmp/mongodb/" + -e MONGO_INITDB_ROOT_USERNAME=mongoadmin + -e MONGO_INITDB_ROOT_PASSWORD=secret + -e UMASK=0777 + --name {{ mongodbdockerdb }} + mongo:{{ mongodbversion }} + when: mongodbdistribution == "modb" + + - name: Prepare Container for Mongodb + shell: > + docker run -d --name={{ mongodbdockerdb }} + -p {{ mongodbportrange }}:27017-27019 + --network "{{ mongodbdockerdb }}_network" + -v "/$(pwd)/setup_scripts/mongodb:/tmp/setup/" + -v "/$(pwd)/queries/mongodb:/tmp/mongodb/" + phusion/baseimage:focal-1.1.0 + when: mongodbdistribution == "psmdb" + + - name: Setup Percona Server Mongodb using setup script + shell: docker exec {{ mongodbdockerdb }} nohup bash -x /tmp/setup/mongodb_setup.sh {{ mongodbversion }} {{ mongodbsetup }} > /$(pwd)/setup_scripts/mongodb/mongodb_setup_log 2>&1 + when: mongodbdistribution == "psmdb" + + - name: Wait for Container to be ready + shell: docker logs {{ mongodbdockerdb }} 2>&1 | grep "waiting for connections on port 27017" + register: result + until: result.rc == 0 + retries: 10 + delay: 10 + when: mongodbdistribution == "modb" + + - name: Wait for Container to be ready + shell: cat /$(pwd)/setup_scripts/mongodb/mongodb_setup_log | grep "waiting for connections on port 27017" + register: result + until: result.rc == 0 + retries: 10 + delay: 10 + when: mongodbdistribution == "psmdb" + + - name: Get the Port for Primary in Mongodb Replica Set and Sharded Cluster Setups + shell: docker exec {{ mongodbdockerdb }} ./nodes/cl_primary.sh --eval 'db.runCommand("ismaster")' | grep "primary" | awk '{print $3}' | awk -F':' '{print $2}' | awk -F"\"" '{print $1}' + when: mongodbdistribution == "psmdb" and (mongodbsetup == "replicaset" or mongodbsetup == "sharding" ) + register: primaryport + + - name: Enable MongoDb Profiler on Docker official Mongodb + shell: docker exec {{ mongodbdockerdb }} mongo -u mongoadmin -p 'secret' "{{ item }}" + with_items: + - /tmp/mongodb/mongodb_user_setup.js + when: mongodbdistribution == "modb" + + - name: Check if PMM Workload container for this Mongodb Deployment already running, kill it + shell: > + docker stop pmm{{ mongodbdockerdb }}-run && docker rm -fv pmm{{ mongodbdockerdb }}-run + when: mongodbdataload == "pmmworkload" + ignore_errors: true + tags: + - pmm-workload + - cleanup + + - name: Build the Workload container if not already build + shell: > + touch docker-build.log | docker build --tag php-db {{ workloadscripts }} > docker-build.log 2>&1 + when: mongodbdataload == "pmmworkload" + ignore_errors: true + tags: + - pmm-workload + + - name: setup QAN Workload container with composer requirement for mongodb driver + shell: > + docker run --rm --name="pmm{{ mongodbdockerdb }}-run" + --network "{{ mongodbdockerdb }}_network" + -v "{{ workloadscripts }}:/usr/src/myapp" + -w /usr/src/myapp + php-db + composer require mongodb/mongodb + when: mongodbdataload == "pmmworkload" + tags: + - pmm-workload + + - name: run QAN Workload Script for Official Mongodb + shell: > + docker run --rm --name="pmm{{ mongodbdockerdb }}-run" -d + --network "{{ mongodbdockerdb }}_network" + -e MONGODB_HOST={{ mongodbdockerdb }} + -e MONGODB_PORT=27017 + -e MONGODB_USER=mongoadmin + -e MONGODB_PASSWORD=secret + -e TEST_TARGET_QPS=1000 + -e TEST_COLLECTION=50 + -e TEST_DB=10 + -v "{{ workloadscripts }}:/usr/src/myapp" + -w /usr/src/myapp + php-db + php mongodb_query.php + when: mongodbdataload == "pmmworkload" and mongodbdistribution == "modb" + tags: + - pmm-workload + + - name: run QAN Workload Script for Official Mongodb + shell: > + docker run --rm --name="pmm{{ mongodbdockerdb }}-run" -d + --network "{{ mongodbdockerdb }}_network" + -e MONGODB_HOST={{ mongodbdockerdb }} + -e MONGODB_PORT={{ primaryport.stdout }} + -e TEST_TARGET_QPS=1000 + -e TEST_COLLECTION=50 + -e TEST_DB=10 + -v "{{ workloadscripts }}:/usr/src/myapp" + -w /usr/src/myapp + php-db + php mongodb_query.php + when: mongodbdataload == "pmmworkload" and mongodbdistribution == "psmdb" + tags: + - pmm-workload \ No newline at end of file diff --git a/playbooks/workload/mongodb_query.php b/playbooks/workload/mongodb_query.php old mode 100644 new mode 100755 diff --git a/playbooks/workload/schema_table_query.php b/playbooks/workload/schema_table_query.php old mode 100644 new mode 100755