PMM-7 Refactor framework, use playbooks

playbooks/mongodb.yml

@@ -0,0 +1,170 @@
+---
+- name: Running Mongodb Database as Container using Ansible.
+  hosts: localhost
+  vars:
+    mongodbdockerdb: "{{ lookup('vars', 'extra_mongodbdockerdb', default=lookup('env','MONGODB_CONTAINER') | default('mongo44db', true) ) }}"
+    mongodbportrange: "{{ lookup('vars', 'extra_mongodbportrange', default=lookup('env','MONGODB_PORTRANGE') | default('27017-27019', true) ) }}"
+    mongodbversion: "{{ lookup('vars', 'extra_mongodbversion', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}"
+    mongodbsetup: "{{ lookup('vars', 'extra_mongodbsetup', default=lookup('env','MONGODB_SETUP') | default('replicaset', true) ) }}"
+    mongodbdistribution: "{{ lookup('vars', 'extra_mongodbdistribution', default=lookup('env','MONGODB_DISTRIBUTION') | default('psmdb', true) ) }}"
+    mongodbdataload: "{{ lookup('vars', 'extra_mongodbdataload', default=lookup('env','MONGODB_DATALOAD') | default('pmmworkload', true) ) }}"
+    workloadscripts: "{{ lookup('vars', 'extra_workloadscripts', default=lookup('env','WORKLOAD_SCRIPTS') | default('/$(pwd)/workload/', true) ) }}"
+    mongodbhost: 127.0.0.1
+  tasks:
+    - name: Check for mongodb container already running, kill it
+      shell: >
+        docker ps -q --filter "name={{ mongodbdockerdb }}" | grep -q . && docker stop {{ mongodbdockerdb }} && docker rm -fv {{ mongodbdockerdb }}
+      ignore_errors: true
+      tags:
+        - cleanup
+
+    - name: Check for mongodb container created previously, kill it
+      shell: >
+        docker rm {{ mongodbdockerdb }}
+      ignore_errors: true
+      tags:
+        - cleanup
+
+    - name: delete network if exist
+      shell: docker network rm "{{ mongodbdockerdb }}_network"
+      ignore_errors: true
+      tags:
+        - cleanup
+
+    - name: Create a network
+      shell: docker network create "{{ mongodbdockerdb }}_network"
+
+    - name: Prepare volume for official mongodb container
+      shell: "{{ item }}"
+      with_items:
+        - mkdir -p /tmp/modb_{{ mongodbdockerdb }}
+        - chown -R $USER:$USER /tmp/modb_{{ mongodbdockerdb }} > /dev/null 2>&1
+        - chmod -R go+w /tmp/modb_{{ mongodbdockerdb }} > /dev/null 2>&1
+      when: mongodbdistribution == "modb"
+      become: yes
+
+    - name: Cleanup the tmp on local
+      shell: rm -r /tmp/modb_{{ mongodbdockerdb }}
+      tags: ['never', 'cleanup' ]
+      become: yes
+      ignore_errors: true
+
+    - name: Launch Official Mongodb database container
+      shell: >
+        docker run -d
+        -p {{ mongodbportrange }}:27017-27019
+        --network "{{ mongodbdockerdb }}_network"
+        -v "/tmp/modb_{{ mongodbdockerdb }}/:/tmp/"
+        -v "/$(pwd)/queries/mongodb:/tmp/mongodb/"
+        -e MONGO_INITDB_ROOT_USERNAME=mongoadmin
+        -e MONGO_INITDB_ROOT_PASSWORD=secret
+        -e UMASK=0777
+        --name {{ mongodbdockerdb }}
+        mongo:{{ mongodbversion }}
+      when: mongodbdistribution == "modb"
+
+    - name: Prepare Container for Mongodb
+      shell: >
+        docker run -d --name={{ mongodbdockerdb }}
+        -p {{ mongodbportrange }}:27017-27019
+        --network "{{ mongodbdockerdb }}_network"
+        -v "/$(pwd)/setup_scripts/mongodb:/tmp/setup/"
+        -v "/$(pwd)/queries/mongodb:/tmp/mongodb/"
+        phusion/baseimage:focal-1.1.0
+      when: mongodbdistribution == "psmdb"
+
+    - name: Setup Percona Server Mongodb using setup script
+      shell: docker exec {{ mongodbdockerdb }} nohup bash -x /tmp/setup/mongodb_setup.sh {{ mongodbversion }} {{ mongodbsetup }} > /$(pwd)/setup_scripts/mongodb/mongodb_setup_log 2>&1
+      when: mongodbdistribution == "psmdb"
+
+    - name: Wait for Container to be ready
+      shell: docker logs {{ mongodbdockerdb }} 2>&1 | grep "waiting for connections on port 27017"
+      register: result
+      until: result.rc == 0
+      retries: 10
+      delay: 10
+      when: mongodbdistribution == "modb"
+
+    - name: Wait for Container to be ready
+      shell: cat /$(pwd)/setup_scripts/mongodb/mongodb_setup_log | grep "waiting for connections on port 27017"
+      register: result
+      until: result.rc == 0
+      retries: 10
+      delay: 10
+      when: mongodbdistribution == "psmdb"
+
+    - name: Get the Port for Primary in Mongodb Replica Set and Sharded Cluster Setups
+      shell: docker exec {{ mongodbdockerdb }} ./nodes/cl_primary.sh --eval 'db.runCommand("ismaster")' | grep "primary" | awk '{print $3}' | awk -F':' '{print $2}' | awk -F"\"" '{print $1}'
+      when: mongodbdistribution == "psmdb" and (mongodbsetup == "replicaset" or mongodbsetup == "sharding" )
+      register: primaryport
+
+    - name: Enable MongoDb Profiler on Docker official Mongodb
+      shell: docker exec {{ mongodbdockerdb }} mongo -u mongoadmin -p 'secret' "{{ item }}"
+      with_items:
+        - /tmp/mongodb/mongodb_user_setup.js
+      when: mongodbdistribution == "modb"
+
+    - name: Check if PMM Workload container for this Mongodb Deployment already running, kill it
+      shell: >
+        docker stop pmm{{ mongodbdockerdb }}-run && docker rm -fv pmm{{ mongodbdockerdb }}-run
+      when: mongodbdataload == "pmmworkload"
+      ignore_errors: true
+      tags:
+        - pmm-workload
+        - cleanup
+
+    - name: Build the Workload container if not already build
+      shell: >
+        touch docker-build.log | docker build --tag php-db {{ workloadscripts }} > docker-build.log 2>&1
+      when: mongodbdataload == "pmmworkload"
+      ignore_errors: true
+      tags:
+        - pmm-workload
+
+    - name: setup QAN Workload container with composer requirement for mongodb driver
+      shell: >
+        docker run --rm --name="pmm{{ mongodbdockerdb }}-run"
+        --network "{{ mongodbdockerdb }}_network"
+        -v "{{ workloadscripts }}:/usr/src/myapp"
+        -w /usr/src/myapp
+        php-db
+        composer require mongodb/mongodb
+      when: mongodbdataload == "pmmworkload"
+      tags:
+        - pmm-workload
+
+    - name: run QAN Workload Script for Official Mongodb
+      shell: >
+        docker run --rm --name="pmm{{ mongodbdockerdb }}-run" -d
+        --network "{{ mongodbdockerdb }}_network"
+        -e MONGODB_HOST={{ mongodbdockerdb }}
+        -e MONGODB_PORT=27017
+        -e MONGODB_USER=mongoadmin
+        -e MONGODB_PASSWORD=secret
+        -e TEST_TARGET_QPS=1000
+        -e TEST_COLLECTION=50
+        -e TEST_DB=10
+        -v "{{ workloadscripts }}:/usr/src/myapp"
+        -w /usr/src/myapp
+        php-db
+        php mongodb_query.php
+      when: mongodbdataload == "pmmworkload" and mongodbdistribution == "modb"
+      tags:
+        - pmm-workload
+
+    - name: run QAN Workload Script for Official Mongodb
+      shell: >
+        docker run --rm --name="pmm{{ mongodbdockerdb }}-run" -d
+        --network "{{ mongodbdockerdb }}_network"
+        -e MONGODB_HOST={{ mongodbdockerdb }}
+        -e MONGODB_PORT={{ primaryport.stdout }}
+        -e TEST_TARGET_QPS=1000
+        -e TEST_COLLECTION=50
+        -e TEST_DB=10
+        -v "{{ workloadscripts }}:/usr/src/myapp"
+        -w /usr/src/myapp
+        php-db
+        php mongodb_query.php
+      when: mongodbdataload == "pmmworkload" and mongodbdistribution == "psmdb"
+      tags:
+        - pmm-workload

playbooks/mysql.yml

@@ -0,0 +1,182 @@
+---
+- name: Running MYSQL Database as Docker Container using Ansible.
+  hosts: localhost
+  vars:
+    mysqldockerdb: "{{ lookup('vars', 'extra_mysqldockerdb', default=lookup('env','MYSQL_CONTAINER') | default('mysql57-db', true) ) }}"
+    mysqlport: "{{ lookup('vars', 'extra_mysqlport', default=lookup('env','MYSQL_PORT') | default('43306', true) ) }}"
+    mysqlversion: "{{ lookup('vars', 'extra_mysqlversion', default=lookup('env','MYSQL_VERSION') | default('5.7', true) ) }}"
+    mysqldistribution: "{{ lookup('vars', 'extra_mysqldistribution', default=lookup('env','MYSQL_DISTRIBUTION') | default('percona', true) ) }}"
+    mysqldataload: "{{ lookup('vars', 'extra_mysqldataload', default=lookup('env','MYSQL_DATALOAD') | default('pmmworkload', true) ) }}"
+    workloadscripts: "{{ lookup('vars', 'extra_workloadscripts', default=lookup('env','WORKLOAD_SCRIPTS') | default('/$(pwd)/workload/', true) ) }}"
+    mysqlhost: 127.0.0.1
+  tasks:
+    - name: Check for mysql container already running, kill it
+      shell: >
+        docker ps -q --filter "name={{ mysqldockerdb }}" | grep -q . && docker stop {{ mysqldockerdb }} && docker rm -fv {{ mysqldockerdb }}
+      ignore_errors: true
+
+    - name: Check for mysql container created previously, kill it
+      shell: >
+        docker rm {{ mysqldockerdb }}
+      ignore_errors: true
+
+    - name: delete network if exist
+      shell: docker network rm "{{ mysqldockerdb }}_network"
+      ignore_errors: true
+
+    - name: Create a network
+      shell: docker network create "{{ mysqldockerdb }}_network"
+
+    - name: Delete slowlog file if exist from previous runs
+      file:
+        path: "/tmp/mysql/{{ mysqldockerdb }}/mysql_log.log"
+        state: absent
+      become: yes
+      tags:
+        - slowlog-setup
+
+    - name: ensure slowlog directory exist
+      command: mkdir -p /tmp/mysql/{{ mysqldockerdb }}/
+      tags:
+        - slowlog-setup
+
+    - name: ensure slowlog file is writable by Docker container
+      command: chmod 777 -R  /tmp/mysql/{{ mysqldockerdb }}/
+      become: yes
+      tags:
+        - slowlog-setup
+
+    - name: Setting Source for Container based on Distribution requested by user [mysql]
+      set_fact: db="mysql"
+      when: mysqldistribution == "mysql"
+
+    - name: Setting Source for Container based on Distribution requested by user [percona]
+      set_fact: db="percona"
+      when: mysqldistribution == "percona"
+
+    - name: Setting Source for Container based on Distribution requested by user [mariadb]
+      set_fact: db="mariadb"
+      when: mysqldistribution == "mariadb"
+
+    - name: Launch MySQL Database Container
+      shell: >
+        docker run --name {{ mysqldockerdb }} -e MYSQL_ROOT_PASSWORD=password
+        -p {{ mysqlport }}:3306
+        -v /tmp/mysql/{{ mysqldockerdb }}:/tmp/mysql/{{ mysqldockerdb }}
+        -v "/$(pwd)/queries/mysql/users.sql:/docker-entrypoint-initdb.d/1.sql"
+        -v "/$(pwd)/queries/mysql/{{ db }}.sql:/docker-entrypoint-initdb.d/2.sql"
+        --network "{{ mysqldockerdb }}_network"
+        -d {{ db }}:{{ mysqlversion }}
+        --slow-query-log-file=/tmp/mysql/{{ mysqldockerdb }}/mysql_log.log
+        --net_read_timeout=100
+        --slow-query-log='ON'
+        --long-query-time=0
+        --log-slow-admin-statements='ON'
+        --log-slow-slave-statements='ON'
+      tags:
+        - container-mysql
+
+    - name: Wait for Container to be ready
+      shell: docker logs {{ mysqldockerdb }} 2>&1 | grep "ready for connections."
+      register: result
+      until: result.rc == 0
+      retries: 10
+      delay: 5
+
+    - name: Pause for 30 seconds for container buildup
+      pause:
+        seconds: 30
+
+    - name: Check for mysql slow log query
+      shell: docker exec {{ mysqldockerdb }} mysql -u root -ppassword -e "SET GLOBAL slow_query_log='ON';"
+      tags:
+        - slowlog-setup
+
+    - name: Enable Query Response time plugin for Percona Server 5.x
+      shell: docker exec {{ mysqldockerdb }} mysql -u root -ppassword -e "{{ item }}"
+      with_items:
+       - "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';"
+       - "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';"
+       - "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';"
+       - "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';"
+       - "SET GLOBAL query_response_time_stats=ON;"
+      when: mysqlversion is search("5") and  db == "percona"
+      ignore_errors: true
+      tags:
+        - slowlog-setup
+
+    - name: setup sysbench for Mysql DB deployment
+      shell: >
+        docker run --rm=true --name=sb-prepare --network "{{ mysqldockerdb }}_network" severalnines/sysbench sysbench
+        --db-driver=mysql
+        --oltp-table-size=100000
+        --oltp-tables-count=24
+        --threads=1
+        --mysql-host={{ mysqldockerdb }}
+        --mysql-port=3306
+        --mysql-user=sbtest
+        --mysql-password=sbtestsecret
+        /usr/share/sysbench/tests/include/oltp_legacy/parallel_prepare.lua run
+      when: mysqldataload == "sysbench"
+      tags:
+        - sysbench-mysql
+
+    - name: Check for sysbench container for this DB already running, kill it
+      shell: >
+        docker stop sb{{ mysqldockerdb }}-run && docker rm -fv sb{{ mysqldockerdb }}-run
+      when: mysqldataload == "sysbench"
+      ignore_errors: true
+      tags:
+        - sysbench-mysql
+
+    - name: run sysbench Read & Write Load for mysql DB
+      shell: >
+        docker run --name="sb{{ mysqldockerdb }}-run" -d --network "{{ mysqldockerdb }}_network" severalnines/sysbench
+        sysbench --db-driver=mysql --report-interval=2 --mysql-table-engine=innodb --oltp-table-size=100000
+        --oltp-tables-count=24
+        --threads=64
+        --time=99999
+        --mysql-host={{ mysqldockerdb }}
+        --mysql-port=3306
+        --mysql-user=sbtest
+        --mysql-password=sbtestsecret
+        /usr/share/sysbench/tests/include/oltp_legacy/oltp.lua run
+      when: mysqldataload == "sysbench"
+      tags:
+        - sysbench-mysql
+
+    - name: Check if PMM Workload container for this Deployment already running, kill it
+      shell: >
+        docker stop pmm{{ mysqldockerdb }}-run && docker rm -fv pmm{{ mysqldockerdb }}-run
+      when: mysqldataload == "pmmworkload"
+      ignore_errors: true
+      tags:
+        - pmm-workload
+
+    - name: Build the Workload container if not already build
+      shell: >
+        touch docker-build.log | docker build --tag php-db {{ workloadscripts }} > docker-build.log 2>&1
+      when: mysqldataload == "pmmworkload"
+      ignore_errors: true
+      tags:
+        - pmm-workload
+
+    - name: run QAN Workload Script for High Bucket size
+      shell: >
+        docker run --rm --name="pmm{{ mysqldockerdb }}-run" -d
+        --network "{{ mysqldockerdb }}_network"
+        -e MYSQL_HOST={{ mysqldockerdb }}
+        -e MYSQL_PORT=3306
+        -e MYSQL_USER=root
+        -e MYSQL_PASSWORD=password
+        -e TEST_TARGET_QPS=1000
+        -e TEST_QUERIES=100
+        -e TEST_TABLES=50
+        -e TEST_SCHEMAS=10
+        -v "{{ workloadscripts }}:/usr/src/myapp"
+        -w /usr/src/myapp
+        php-db
+        php schema_table_query.php
+      when: mysqldataload == "pmmworkload"
+      tags:
+        - pmm-workload