From 8f84826912e5d0ab8bafd8a1f38d715932b97576 Mon Sep 17 00:00:00 2001 From: Girjesh Rajoria Date: Mon, 27 Nov 2017 18:03:09 +0530 Subject: [PATCH 1/4] Cache invalidation Test for nfs-ganesha Cache invalidation test for nfs-ganesha on mount v3, v4.0 and v4.1 Signed-off-by: Girjesh Rajoria --- cache-invalidation/client.sh | 67 +++++++++++++ cache-invalidation/duffy.py | 164 +++++++++++++++++++++++++++++++ cache-invalidation/server.sh | 182 +++++++++++++++++++++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 cache-invalidation/client.sh create mode 100644 cache-invalidation/duffy.py create mode 100644 cache-invalidation/server.sh diff --git a/cache-invalidation/client.sh b/cache-invalidation/client.sh new file mode 100644 index 00000000..8ec7f05f --- /dev/null +++ b/cache-invalidation/client.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# +# Environment variables used: +# - SERVER: hostname or IP-address of the NFS-server +# - EXPORT: NFS-export to test (should start with "/") + +# if any command fails, the script should exit +set -e + +# enable some more output +set -x + +if [ $1 -eq 1 ] +then + [ -n "${SERVER}" ] + [ -n "${EXPORT}" ] + + # install build and runtime dependencies + echo "Install build and runtime dependencies" + yum -y install nfs-utils time + + echo "--------------------------------------------------" + echo "Running test on Mount Version $2" + echo "--------------------------------------------------" + + # mount + mkdir -p /mnt/nfs + mount -t nfs -o vers=$2 ${SERVER}:${EXPORT} /mnt/nfs + status=$? + if [ $status -ne 0 ] + then + echo "Failed Mounting for version 3 on client-2" + echo "Cache Invalidation Test: FAILURE" + exit $status + fi + +elif [ $1 -eq 2 ] +then + touch /mnt/nfs/file1.txt + > /mnt/nfs/file1.txt + echo "We Are" >> /mnt/nfs/file1.txt + +elif [ $1 -eq 3 ] +then + content=$(cat /mnt/nfs/file1.txt) + if [ "$content" == "We Are" ] + then + echo "REDHAT" >> /mnt/nfs/file1.txt + else + echo "Cache Invalidation Test: FAILURE" + exit -1 + fi + #unmount on client-2 + umount /mnt/nfs + +elif [ $1 -eq 4 ] +then + content=$(cat /mnt/nfs/file1.txt) + if [ "$content" != "We Are +REDHAT" ] + then + echo "Cache Invalidation Test: FAILURE" + exit -1 + fi + #unmount on client-1 + umount /mnt/nfs +fi diff --git a/cache-invalidation/duffy.py b/cache-invalidation/duffy.py new file mode 100644 index 00000000..dbe4ffbf --- /dev/null +++ b/cache-invalidation/duffy.py @@ -0,0 +1,164 @@ +# +# from: https://raw.githubusercontent.com/kbsingh/centos-ci-scripts/master/build_python_script.py +# +# This script uses the Duffy node management api to get fresh machines to run +# your CI tests on. Once allocated you will be able to ssh into that machine +# as the root user and setup the environ +# +# XXX: You need to add your own api key below, and also set the right cmd= line +# needed to run the tests +# +# Please note, this is a basic script, there is no error handling and there are +# no real tests for any exceptions. Patches welcome! + +import json, urllib, subprocess, sys, os, time + +url_base="http://admin.ci.centos.org:8080" +ver=os.getenv("CENTOS_VERSION") +arch=os.getenv("CENTOS_ARCH") +count=4 +server_script=os.getenv("SERVER_TEST_SCRIPT") +client_script=os.getenv("CLIENT_TEST_SCRIPT") + +# delay for 5 minutes (duffy timeout for rate limiting) +retry_delay=300 +# retry maximum 3 hours, that is 3 x 60 x 60 seconds +max_retries=((3 * 60 * 60) / retry_delay) + +# read the API key for Duffy from the ~/duffy.key file +fo=open("/home/nfs-ganesha/duffy.key") +api=fo.read().strip() +fo.close() + +# build the URL to request the system(s) +get_nodes_url="%s/Node/get?key=%s&ver=%s&arch=%s&count=%s" % (url_base,api,ver,arch,count) + +# request the system(s) +retries=0 +while retries < max_retries: + try: + dat=urllib.urlopen(get_nodes_url).read() + b=json.loads(dat) + # all is fine, break out of the loop + break + except ValueError, ve: + print("Failed to parse Duffy response: %s" % (dat)) + except Error, e: + print("An unexpected error occured: %s" % (e)) + + retries+=1 + print("Waiting %d seconds before retrying #%d..." % (retry_delay, retries)) + time.sleep(retry_delay) + + +# NFS-Ganesha Server (parameters need double escape, passed on ssh commandline) +server_env="export GERRIT_HOST='%s'" % os.getenv("GERRIT_HOST") +server_env+=" GERRIT_PROJECT='%s'" % os.getenv("GERRIT_PROJECT") +server_env+=" GERRIT_REFSPEC='%s'" % os.getenv("GERRIT_REFSPEC") +server_env+=" YUM_REPO='%s'" % os.getenv("YUM_REPO", "") +server_env+=" GLUSTER_VOLUME='%s'" % os.getenv("EXPORT") +server_env+=" ENABLE_ACL='%s'" % os.getenv("ENABLE_ACL", "") + +# add the export with environment to ~/.bashrc for server-1 +cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' +tee -a ~/.bashrc' <<< "%s" +""" % (b['hosts'][0], server_env) +subprocess.call(cmd, shell=True) + +cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + yum -y install curl && + curl -o server1 %s && bash server1 0 +'""" % (b['hosts'][0], server_script) +rtn_code=subprocess.call(cmd, shell=True) + +# add the export with environment to ~/.bashrc for server-2 +if rtn_code == 0: + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + tee -a ~/.bashrc' <<< "%s" + """ % (b['hosts'][1], server_env) + subprocess.call(cmd, shell=True) + + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + yum -y install curl && + curl -o server2 %s && bash server2 0 0 0 %s + '""" % (b['hosts'][1], server_script, b['hosts'][0]) + rtn_code=subprocess.call(cmd, shell=True) + +# check rtn_code and skip resume client-1 part after failure +if rtn_code == 0: + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + yum -y install curl && + curl -o server1 %s && bash server1 - - 0 + '""" % (b['hosts'][0], server_script) + rtn_code=subprocess.call(cmd, shell=True) + +# check rtn_code and skip client environment part after failure +if rtn_code == 0: + # NFS-Client (parameters need double escape, passed on ssh commandline) + client_env="export SERVER='%s'" % b['hosts'][0] + client_env+=" EXPORT='/%s'" % os.getenv("EXPORT") + client_env+=" TEST_PARAMETERS='%s'" % os.getenv("TEST_PARAMETERS", "") + + # add the export with environment to ~/.bashrc for client-1 + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + tee -a ~/.bashrc' <<< "%s" + """ % (b['hosts'][2], client_env) + rtn_code=subprocess.call(cmd, shell=True) + + # NFS-Client (parameters need double escape, passed on ssh commandline) + client_env="export SERVER='%s'" % b['hosts'][1] + client_env+=" EXPORT='/%s'" % os.getenv("EXPORT") + client_env+=" TEST_PARAMETERS='%s'" % os.getenv("TEST_PARAMETERS", "") + + # add the export with environment to ~/.bashrc for client-2 + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + tee -a ~/.bashrc' <<< "%s" + """ % (b['hosts'][3], client_env) + rtn_code=subprocess.call(cmd, shell=True) + + # check rtn_code and skip rest test part after failure +if rtn_code == 0: + versions = [3 , 4.0, 4.1] + for version in versions: + # running client-1 script + client_script = client_script.strip(" ") + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + curl -o client1 %s && bash client1 1 %.1f + '""" % (b['hosts'][2], client_script, version) + rtn_code=subprocess.call(cmd, shell=True) + + # check rtn_code and skip client-2 part after failure + if rtn_code == 0: + # running client-2 script + client_script = client_script.strip(" ") + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + curl -o client2 %s && bash client2 1 %.1f + '""" % (b['hosts'][3], client_script, version) + rtn_code=subprocess.call(cmd, shell=True) + + # check rtn_code and skip client-1 write to a file part after failure + if rtn_code == 0: + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + curl -o client1 %s && bash client1 2 + '""" % (b['hosts'][2], client_script) + rtn_code=subprocess.call(cmd, shell=True) + + # check rtn_code and skip client-2 read and write to the file part after failure + if rtn_code == 0: + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + curl -o client2 %s && bash client2 3 + '""" % (b['hosts'][3], client_script) + rtn_code=subprocess.call(cmd, shell=True) + + # check rtn_code and skip client-1 write to a file part after failure + if rtn_code == 0: + cmd="""ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@%s ' + curl -o client1 %s && bash client1 4 + '""" % (b['hosts'][2], client_script) + rtn_code=subprocess.call(cmd, shell=True) + +# return the system(s) to duffy +done_nodes_url="%s/Node/done?key=%s&ssid=%s" % (url_base, api, b['ssid']) +das=urllib.urlopen(done_nodes_url).read() + +sys.exit(rtn_code) diff --git a/cache-invalidation/server.sh b/cache-invalidation/server.sh new file mode 100644 index 00000000..fd18f5f3 --- /dev/null +++ b/cache-invalidation/server.sh @@ -0,0 +1,182 @@ +#!/bin/sh +# +# Setup a simple gluster environment and export a volume through NFS-Ganesha. +# +# This script uses the following environment variables: +# - GLUSTER_VOLUME: name of the gluster volume to create +# this name will also be used as name for the export +# +# The YUM_REPO and GERRIT_* variables are mutually exclusive. +# +# - YUM_REPO: URL to the yum repository (.repo file) for the NFS-Ganesha +# packages. When this option is used, libntirpc-latest is enabled +# as well. Leave empty in case patches from Gerrit need testing. +# +# - GERRIT_HOST: when triggered from a new patch submission, this is set to the +# git server that contains the repository to use. +# +# - GERRIT_PROJECT: project that triggered the build (like ffilz/nfs-ganesha). +# +# - GERRIT_REFSPEC: git tree-ish that can be fetched and checked-out for testing. + +# abort if anything fails +set -e + +[ -n "${GLUSTER_VOLUME}" ] + +# be a little bit more verbose +set -x + +if [ $1 -eq 0 ] +then + # enable repositories + yum -y install centos-release-gluster yum-utils + + # make sure rpcbind is running + yum -y install rpcbind + systemctl start rpcbind + + # CentOS 7.4.1708 has an SELinux issue that prevents NFS-Ganesha from creating + # the /var/log/ganesha/ganesha.log file. Starting ganesha.nfsd fails due to + # this. + echo 'TODO: this is BAD, needs a fix in the selinux-policy' + setenforce 0 + + if [ -n "${YUM_REPO}" ] + then + yum-config-manager --add-repo=http://artifacts.ci.centos.org/nfs-ganesha/nightly/libntirpc/libntirpc-latest.repo + yum-config-manager --add-repo=${YUM_REPO} + + # install the latest version of gluster + yum -y install nfs-ganesha nfs-ganesha-gluster glusterfs-ganesha + + # start nfs-ganesha service + if ! systemctl start nfs-ganesha + then + echo "+++ systemctl status nfs-ganesha.service +++" + systemctl status nfs-ganesha.service + echo "+++ journalctl -xe +++" + journalctl -xe + exit 1 + fi + else + [ -n "${GERRIT_HOST}" ] + [ -n "${GERRIT_PROJECT}" ] + [ -n "${GERRIT_REFSPEC}" ] + + GIT_REPO=$(basename "${GERRIT_PROJECT}") + GIT_URL="https://${GERRIT_HOST}/${GERRIT_PROJECT}" + + # install NFS-Ganesha build dependencies + yum -y --enablerepo=centos-gluster*-test install glusterfs-api-devel + yum -y install git bison flex cmake gcc-c++ libacl-devel krb5-devel \ + dbus-devel libnfsidmap-devel libwbclient-devel libcap-devel \ + libblkid-devel rpm-build redhat-rpm-config + + git init "${GIT_REPO}" + pushd "${GIT_REPO}" + + git fetch "${GIT_URL}" "${GERRIT_REFSPEC}" + git checkout -b "${GERRIT_REFSPEC}" FETCH_HEAD + + # update libntirpc + git submodule update --init || git submodule sync + + mkdir build + pushd build + + cmake -DCMAKE_BUILD_TYPE=Maintainer -DBUILD_CONFIG=everything ../src + make dist + rpmbuild -ta --define "_srcrpmdir $PWD" --define "_rpmdir $PWD" *.tar.gz + rpm_arch=$(rpm -E '%{_arch}') + ganesha_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p *.src.rpm) + if [ -e ${rpm_arch}/libntirpc-devel*.rpm ]; then + ntirpc_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p ${rpm_arch}/libntirpc-devel*.rpm) + ntirpc_rpm=${rpm_arch}/libntirpc-${ntirpc_version}.${rpm_arch}.rpm + fi + yum -y install ${ntirpc_rpm} ${rpm_arch}/nfs-ganesha-{,gluster-}${ganesha_version}.${rpm_arch}.rpm + + # start nfs-ganesha service with an empty configuration + > /etc/ganesha/ganesha.conf + if ! systemctl start nfs-ganesha + then + echo "+++ systemctl status nfs-ganesha.service +++" + systemctl status nfs-ganesha.service + echo "+++ journalctl -xe +++" + journalctl -xe + exit 1 + fi + fi + + # TODO: open only the ports needed? + # disable the firewall, otherwise the client can not connect + systemctl stop firewalld || service iptables stop + + # TODO: SELinux prevents creating special files on Gluster bricks (bz#1331561) + setenforce 0 + + # install and start gluster + yum -y install glusterfs-server + systemctl start glusterd + mkdir -p /bricks/${GLUSTER_VOLUME} +fi + +if [ $2 -eq 0 ] +then + # create and start gluster volume + gluster peer probe $4 + gluster volume create ${GLUSTER_VOLUME} \ + replica 2 \ + $(hostname --fqdn):/bricks/${GLUSTER_VOLUME}/b{1,2} $4:/bricks/${GLUSTER_VOLUME}/b{1,2} force + + gluster volume start ${GLUSTER_VOLUME} force + + # cache-invalidation on for the volume + gluster v set ${GLUSTER_VOLUME} features.cache-invalidation on +fi + +if [ $3 -eq 0 ] +then + # Export the volume + mkdir -p /usr/libexec/ganesha + cd /usr/libexec/ganesha + yum -y install wget + wget https://raw.githubusercontent.com/gluster/glusterfs/release-3.10/extras/ganesha/scripts/create-export-ganesha.sh + wget https://raw.githubusercontent.com/gluster/glusterfs/release-3.10/extras/ganesha/scripts/dbus-send.sh + chmod 755 create-export-ganesha.sh dbus-send.sh + + /usr/libexec/ganesha/create-export-ganesha.sh /etc/ganesha on ${GLUSTER_VOLUME} + /usr/libexec/ganesha/dbus-send.sh /etc/ganesha on ${GLUSTER_VOLUME} + + # wait till server comes out of grace period + sleep 90 + + # basic check if the export is available, some debugging if not + if ! showmount -e | grep -q -w -e "${GLUSTER_VOLUME}" + then + echo "+++ /var/log/ganesha.log +++" + cat /var/log/ganesha.log + echo + echo "+++ /etc/ganesha/ganesha.conf +++" + grep --with-filename -e '' /etc/ganesha/ganesha.conf + echo + echo "+++ /etc/ganesha/exports/*.conf +++" + grep --with-filename -e '' /etc/ganesha/exports/*.conf + echo + echo "Export ${GLUSTER_VOLUME} is not available" + exit 1 + fi + + #Enabling ACL for the volume if ENABLE_ACL param is set to True + if [ "${ENABLE_ACL}" == "True" ] + then + conf_file="/etc/ganesha/exports/export."${GLUSTER_VOLUME}".conf" + sed -i s/'Disable_ACL = .*'/'Disable_ACL = false;'/g ${conf_file} + cat ${conf_file} + + #Parsing export id from volume export conf file + export_id=$(grep 'Export_Id' ${conf_file} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\([0-9]*\).*/\1/') + + dbus-send --type=method_call --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport string:${conf_file} string:"EXPORT(Export_Id = ${export_id})" + fi +fi From 2d31f1a8d9500ae3d981cec1d08b7b6707685376 Mon Sep 17 00:00:00 2001 From: Girjesh Rajoria <32087575+grajoria@users.noreply.github.com> Date: Tue, 28 Nov 2017 00:05:58 +0530 Subject: [PATCH 2/4] Update client.sh --- cache-invalidation/client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache-invalidation/client.sh b/cache-invalidation/client.sh index 8ec7f05f..e79c22a3 100644 --- a/cache-invalidation/client.sh +++ b/cache-invalidation/client.sh @@ -29,7 +29,7 @@ then status=$? if [ $status -ne 0 ] then - echo "Failed Mounting for version 3 on client-2" + echo "Failed Mounting for version $2 on client-2" echo "Cache Invalidation Test: FAILURE" exit $status fi From ea072153865c7c61034e19b699583adc75a700d7 Mon Sep 17 00:00:00 2001 From: Girjesh Rajoria <32087575+grajoria@users.noreply.github.com> Date: Tue, 28 Nov 2017 00:08:21 +0530 Subject: [PATCH 3/4] Update server.sh --- cache-invalidation/server.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/cache-invalidation/server.sh b/cache-invalidation/server.sh index fd18f5f3..11eecdd6 100644 --- a/cache-invalidation/server.sh +++ b/cache-invalidation/server.sh @@ -112,9 +112,6 @@ then # disable the firewall, otherwise the client can not connect systemctl stop firewalld || service iptables stop - # TODO: SELinux prevents creating special files on Gluster bricks (bz#1331561) - setenforce 0 - # install and start gluster yum -y install glusterfs-server systemctl start glusterd From 33b9c8e5b9379203ebbc7db2d9e5e37836d9edc9 Mon Sep 17 00:00:00 2001 From: Girjesh Rajoria <32087575+grajoria@users.noreply.github.com> Date: Mon, 18 Dec 2017 15:12:46 +0530 Subject: [PATCH 4/4] Update client.sh --- cache-invalidation/client.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache-invalidation/client.sh b/cache-invalidation/client.sh index e79c22a3..ddcf4adc 100644 --- a/cache-invalidation/client.sh +++ b/cache-invalidation/client.sh @@ -51,7 +51,7 @@ then exit -1 fi #unmount on client-2 - umount /mnt/nfs + umount -l /mnt/nfs elif [ $1 -eq 4 ] then @@ -63,5 +63,5 @@ REDHAT" ] exit -1 fi #unmount on client-1 - umount /mnt/nfs + umount -l /mnt/nfs fi