test: Add manual test for cross-shard balancing

It's pretty long, so not for automatic execition 2-shards tests: {'shard_0': {'iops': 88204.3828, 'shares': 100}, 'shard_1': {'iops': 89686.5156, 'shares': 100}} IOPS ratio 1.02, expected 1.0, deviation 1% {'shard_0': {'iops': 60321.3125, 'shares': 100}, 'shard_1': {'iops': 117566.406, 'shares': 200}} IOPS ratio 1.95, expected 2.0, deviation 2% {'shard_0': {'iops': 37326.2422, 'shares': 100}, 'shard_1': {'iops': 140555.062, 'shares': 400}} IOPS ratio 3.77, expected 4.0, deviation 5% {'shard_0': {'iops': 21547.6152, 'shares': 100}, 'shard_1': {'iops': 156309.891, 'shares': 800}} IOPS ratio 7.25, expected 8.0, deviation 9% 3-shards tests: {'shard_0': {'iops': 45211.9336, 'shares': 100}, 'shard_1': {'iops': 45211.9766, 'shares': 100}, 'shard_2': {'iops': 87412.9453, 'shares': 200}} shard-1 IOPS ratio 1.0, expected 1.0, deviation 0% shard-2 IOPS ratio 1.93, expected 2.0, deviation 3% {'shard_0': {'iops': 30992.2188, 'shares': 100}, 'shard_1': {'iops': 30992.2812, 'shares': 100}, 'shard_2': {'iops': 115887.609, 'shares': 400}} shard-1 IOPS ratio 1.0, expected 1.0, deviation 0% shard-2 IOPS ratio 3.74, expected 4.0, deviation 6% {'shard_0': {'iops': 19279.6348, 'shares': 100}, 'shard_1': {'iops': 19279.6934, 'shares': 100}, 'shard_2': {'iops': 139316.828, 'shares': 800}} shard-1 IOPS ratio 1.0, expected 1.0, deviation 0% shard-2 IOPS ratio 7.23, expected 8.0, deviation 9% {'shard_0': {'iops': 26505.9082, 'shares': 100}, 'shard_1': {'iops': 53011.9922, 'shares': 200}, 'shard_2': {'iops': 98369.4453, 'shares': 400}} shard-1 IOPS ratio 2.0, expected 2.0, deviation 0% shard-2 IOPS ratio 3.71, expected 4.0, deviation 7% {'shard_0': {'iops': 17461.8145, 'shares': 100}, 'shard_1': {'iops': 34923.8438, 'shares': 200}, 'shard_2': {'iops': 125470.43, 'shares': 800}} shard-1 IOPS ratio 2.0, expected 2.0, deviation 0% shard-2 IOPS ratio 7.19, expected 8.0, deviation 10% {'shard_0': {'iops': 14812.3037, 'shares': 100}, 'shard_1': {'iops': 58262, 'shares': 400}, 'shard_2': {'iops': 104794.633, 'shares': 800}} shard-1 IOPS ratio 3.93, expected 4.0, deviation 1% shard-2 IOPS ratio 7.07, expected 8.0, deviation 11% Signed-off-by: Pavel Emelyanov <[email protected]>
scylladb · Jun 18, 2024 · a2b8ec6 · a2b8ec6
1 parent 2b4064a
commit a2b8ec6
Showing 1 changed file with 150 additions and 0 deletions.
diff --git a/tests/manual/iosched-smp.py b/tests/manual/iosched-smp.py
@@ -0,0 +1,150 @@
+#!/bin/env python3
+
+import os.path
+import yaml
+import shutil
+import subprocess
+import argparse
+from functools import reduce
+
+parser = argparse.ArgumentParser(description='IO scheduler tester')
+parser.add_argument('--directory', help='Directory to run on', default='/mnt')
+parser.add_argument('--seastar-build-dir', help='Path to seastar build directory', default='./build/dev/', dest='bdir')
+parser.add_argument('--duration', help='One run duration', default=60)
+args = parser.parse_args()
+
+class iotune:
+    def __init__(self, args):
+        self._iotune = args.bdir + '/apps/iotune/iotune'
+        self._dir = args.directory
+
+    def ensure_io_properties(self):
+        if os.path.exists('io_properties.yaml'):
+            print('Using existing io_properties file')
+        else:
+            print('Running iotune')
+            subprocess.check_call([self._iotune, '--evaluation-directory', self._dir, '-c1', '--properties-file', 'io_properties.yaml'])
+
+class ioinfo:
+    def __init__(self, args):
+        self._ioinfo = args.bdir + '/apps/io_tester/ioinfo'
+        self._dir = args.directory
+        res = subprocess.check_output([self._ioinfo, '--directory', self._dir, '--io-properties-file', 'io_properties.yaml'])
+        self._info = yaml.safe_load(res)
+
+    def min_read_length(self):
+        return 4
+
+    def max_read_length(self):
+        return min(self._info['disk_read_max_length'] / 1024, 128)
+
+    def min_write_length(self):
+        return 4
+
+    def max_write_length(self):
+        return min(self._info['disk_write_max_length'] / 1024, 64)
+
+
+class job:
+    def __init__(self, typ, req_size_kb, prl, shards, shares):
+        self._typ = typ
+        self._req_size = req_size_kb
+        self._prl = prl
+        self._shares = shares
+        self._shards = shards
+
+    def to_conf_entry(self, name):
+        return {
+            'name': name,
+            'shards': self._shards,
+            'type': self._typ,
+            'shard_info': {
+                'parallelism': self._prl,
+                'reqsize': f'{self._req_size}kB',
+                'shares': self._shares
+            }
+        }
+
+    def shares(self):
+        return self._shares
+
+
+class io_tester:
+    def __init__(self, args, smp):
+        self._jobs = []
+        self._duration = args.duration
+        self._io_tester = args.bdir + '/apps/io_tester/io_tester'
+        self._dir = args.directory
+        self._use_fraction = 0.1
+        self._smp = smp
+
+    def add_job(self, name, job):
+        self._jobs.append(job.to_conf_entry(name))
+
+    def _setup_data_sizes(self):
+        du = shutil.disk_usage(self._dir)
+        one_job_space_mb = int(du.free * self._use_fraction / len(self._jobs) / (100*1024*1024)) * 100 # round down to 100MB
+        for j in self._jobs:
+            j['data_size'] = f'{one_job_space_mb}MB'
+
+    def run(self):
+        if not self._jobs:
+            raise 'Empty jobs'
+
+        self._setup_data_sizes()
+        yaml.dump(self._jobs, open('conf.yaml', 'w'))
+        self._proc = subprocess.Popen([self._io_tester, '--storage', self._dir, f'-c{self._smp}', '--num-io-groups', '1', '--conf', 'conf.yaml', '--duration', f'{self._duration}', '--io-properties-file', 'io_properties.yaml'], stdout=subprocess.PIPE)
+        res = self._proc.communicate()
+        res = res[0].split(b'---\n')[1]
+        return yaml.safe_load(res)
+
+
+def run_jobs(jobs, args, smp):
+    iot = io_tester(args, smp)
+    results = {}
+    for j in jobs:
+        iot.add_job(j, jobs[j])
+        results[j] = { 'iops': 0, 'shares': jobs[j].shares() }
+
+    out = iot.run()
+    statuses = {}
+
+    for j in results:
+        for shard in out:
+            if j in shard:
+                results[j]['iops'] += shard[j]['IOPS']
+
+    return results
+
+
+iotune(args).ensure_io_properties()
+ioinf = ioinfo(args)
+
+for s in [ 100, 200, 400, 800 ]:
+    def ratios(res, idx):
+        shares_ratio = float(res[f'shard_{idx}']['shares']) / float(res['shard_0']['shares'])
+        iops_ratio = float(res[f'shard_{idx}']['iops']) / float(res['shard_0']['iops'])
+        return (shares_ratio, iops_ratio)
+
+    res = run_jobs({
+            'shard_0': job('randread', ioinf.min_read_length(), 16, ['0'], 100),
+            'shard_1': job('randread', ioinf.min_read_length(), 24, ['1'], s),
+    }, args, 2)
+    print(f'{res}')
+    shares_ratio, iops_ratio = ratios(res, 1)
+    print(f'IOPS ratio {iops_ratio:.3}, expected {shares_ratio:.3}, deviation {int(abs(shares_ratio - iops_ratio)*100.0/shares_ratio)}%')
+
+    for s2 in [ 200, 400, 800 ]:
+        if s2 <= s:
+            continue
+
+        res = run_jobs({
+                'shard_0': job('randread', ioinf.min_read_length(), 16, ['0'], 100),
+                'shard_1': job('randread', ioinf.min_read_length(), 24, ['1'], s),
+                'shard_2': job('randread', ioinf.min_read_length(), 32, ['2'], s2),
+        }, args, 3)
+        print(f'{res}')
+        shares_ratio, iops_ratio = ratios(res, 1)
+        print(f'shard-1 IOPS ratio {iops_ratio:.3}, expected {shares_ratio:.3}, deviation {int(abs(shares_ratio - iops_ratio)*100.0/shares_ratio)}%')
+        shares_ratio, iops_ratio = ratios(res, 2)
+        print(f'shard-2 IOPS ratio {iops_ratio:.3}, expected {shares_ratio:.3}, deviation {int(abs(shares_ratio - iops_ratio)*100.0/shares_ratio)}%')