rename files

experiments/analysis/analysis_merge.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+python3 merge_alloc_discrete.py &
+python3 merge_frag_discrete.py &
+python3 merge_frag_ratio_discrete.py &
+python3 merge_fail_pods.py &

experiments/analysis/merge_alloc_discrete_0920.py → experiments/analysis/merge_alloc_discrete.py

@@ -4,8 +4,7 @@
 
 RESULTDIR="analysis_results"
 DATADIR="data"
-RESULTDIR="analysis_results"
-DATADIR="data"
+
 filepath = os.path.abspath(__file__)
 root = Path(filepath).parents[1] # 0524
 data = root / DATADIR
@@ -18,12 +17,12 @@
 # │   │   ├── analysis_new.csv
 # │   │   └── analysis.csv
 # │   ├── [d] images
-# │   ├── merge_0913.py
-# │   ├── plot_0810_dotprod_extend.py
+# │   ├── merge.py
+# │   ├── plot.py
 # ├── [d] data
-# │   ├── [d] cluster_hhpai_na61-pod_hhpai_na61-0820_no_time
-# │   │   ├── [d] 01-FragShare
-# │   │   │   ├── [d] 0.6
+# │   ├── [d] openb_pod_list_default
+# │   │   ├── [d] 06-FGD
+# │   │   │   ├── [d] 1.3
 # │   │   │   │   ├── [d] 42
 # ......
 

experiments/analysis/merge_fail_pods_1010.py → experiments/analysis/merge_fail_pods.py

@@ -2,10 +2,11 @@
 import pandas as pd
 from pathlib import Path
 
-DATADIR="data"
 IN_FILE="analysis_fail.out"
-RESULTDIR="analysis_results"
 OUT_CSV="analysis_fail.csv"
+RESULTDIR="analysis_results"
+DATADIR="data"
+
 filepath = os.path.abspath(__file__)
 root = Path(filepath).parents[1] # 0524
 data = root / DATADIR
@@ -18,14 +19,15 @@
 # │   │   ├── analysis_new.csv
 # │   │   └── analysis.csv
 # │   ├── [d] images
-# │   ├── merge_0913.py
-# │   ├── plot_0810_dotprod_extend.py
+# │   ├── merge.py
+# │   ├── plot.py
 # ├── [d] data
-# │   ├── [d] cluster_hhpai_na61-pod_hhpai_na61-0820_no_time
-# │   │   ├── [d] 01-FragShare
-# │   │   │   ├── [d] 0.6
+# │   ├── [d] openb_pod_list_default
+# │   │   ├── [d] 06-FGD
+# │   │   │   ├── [d] 1.3
 # │   │   │   │   ├── [d] 42
 # ......
+
 def get_total_num_gpu(workload):
     return 6212
 

experiments/analysis/merge_frag_discrete_0928.py → experiments/analysis/merge_frag_discrete.py

@@ -17,14 +17,15 @@
 # │   │   ├── analysis_new.csv
 # │   │   └── analysis.csv
 # │   ├── [d] images
-# │   ├── merge_0913.py
-# │   ├── plot_0810_dotprod_extend.py
+# │   ├── merge.py
+# │   ├── plot.py
 # ├── [d] data
-# │   ├── [d] cluster_hhpai_na61-pod_hhpai_na61-0820_no_time
-# │   │   ├── [d] 01-FragShare
-# │   │   │   ├── [d] 0.6
+# │   ├── [d] openb_pod_list_default
+# │   │   ├── [d] 06-FGD
+# │   │   │   ├── [d] 1.3
 # │   │   │   │   ├── [d] 42
 # ......
+
 def get_total_num_gpu(workload):
     return 6212
 

experiments/analysis/merge_frag_ratio_discrete_0928.py → experiments/analysis/merge_frag_ratio_discrete.py

@@ -4,6 +4,7 @@
 
 RESULTDIR="analysis_results"
 DATADIR="data"
+
 filepath = os.path.abspath(__file__)
 root = Path(filepath).parents[1] # 0524
 data = root / DATADIR
@@ -16,12 +17,12 @@
 # │   │   ├── analysis_new.csv
 # │   │   └── analysis.csv
 # │   ├── [d] images
-# │   ├── merge_0913.py
-# │   ├── plot_0810_dotprod_extend.py
+# │   ├── merge.py
+# │   ├── plot.py
 # ├── [d] data
-# │   ├── [d] cluster_hhpai_na61-pod_hhpai_na61-0820_no_time
-# │   │   ├── [d] 01-FragShare
-# │   │   │   ├── [d] 0.6
+# │   ├── [d] openb_pod_list_default
+# │   │   ├── [d] 06-FGD
+# │   │   │   ├── [d] 1.3
 # │   │   │   │   ├── [d] 42
 # ......
 

experiments/plot/.gitignore

@@ -1,2 +1,3 @@
 *.csv
 *.pdf
+!expected_results/*.pdf

experiments/plot/plot_paib_alloc.py → experiments/plot/plot_openb_alloc.py

@@ -1,18 +1,16 @@
 
 # %%
-# 1016
 import matplotlib
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from pathlib import Path
-from IPython.display import display
 from utils import parse_workload_name, POLICY_ABBR_DICT
 
-PAPER_PLOT=True # False: Plot with thinner lines for DingTalk or Doc usages
+PAPER_PLOT=True # False: Plot with thinner lines for Presentation
 SAVEFIG=True   # False: plt.show()
 TUNE_RATIO = 1.3
-FIGNAME = "paib_alloc.pdf"
+FIGNAME = "openb_alloc.pdf"
 
 workload = 'openb_pod_list_default'
 

experiments/plot/plot_paib_frag_amount.py → experiments/plot/plot_openb_frag_amount.py

@@ -1,19 +1,17 @@
 
 # %%
-# 1016
 import matplotlib
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from pathlib import Path
-from IPython.display import display
 from utils import parse_workload_name, POLICY_ABBR_DICT
 
 TYPE='frag_amount'
-PAPER_PLOT=True # False: Plot with thinner lines for DingTalk or Doc usages
+PAPER_PLOT=True # False: Plot with thinner lines for Presentation
 SAVEFIG=False    # False: plt.show()
 TUNE_RATIO = 1.3
-FIGNAME = "paib_%s.pdf"
+FIGNAME = "openb_%s.pdf"
 
 workload = 'openb_pod_list_default'
 
@@ -42,10 +40,6 @@
 
     dfn['workload'] = dfn.workload.apply(parse_workload_name)
 
-    workload_order = ['EightGpu80', 'EightGpu60', 'FourGpu80', 'FourGpu60', 'TwoGpu80', 'TwoGpu60', 'OneGpu80', 'OneGpu60', 'ShareGpu80', 'ShareGpu60', 'hhpai_0820','hhpai_0905','hhpai_mvap_0820', 'hhpai_mvap_0905', 'mit', 'mvap', 'paib']
-    workload_order_dict = dict(zip(workload_order, range(1, len(workload_order)+1)))
-    dfn.workload = dfn.workload.apply(lambda x: x if x not in workload_order_dict else "%02d-%s" % (workload_order_dict[x], x))
-
     # display(dfn)
     # print("SC_POLICY_LIST=[%s]" % (",".join("'%s'" % x for x in list(dfn.sc_policy.unique()))))
 
@@ -68,33 +62,9 @@
     dfnp.sc_policy = dfnp.sc_policy.apply(lambda x: POLICY_ABBR_DICT.get(x, x))
     dfp_dict[type] = dfnp
 
-# openb, production workloads:
-
-# openb, multi-GPUs workloads
-# workload = 'cluster_openb-pod_openb-0820_a20aoc_gpu_nospec'
-# workload = 'cluster_openb-pod_openb-0820_a30aoc_gpu_nospec'
-# workload = 'cluster_openb-pod_openb-0820_a40aoc_gpu_nospec'
-# workload = 'cluster_openb-pod_openb-0820_a50aoc_gpu_nospec'
-
-# openb, non-GPU workloads
-# workload = 'cluster_openb-pod_openb-0820_nocpu_gpu_nospec'
-
-# openb, heterogeneous GPU types
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_05'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_10'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_20'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_25'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec'
-
-# mvap, GPU-sharing workloads
-# workload = 'cluster_mvap-pod_mvap-0820_nomem_no_time-cap_1.0'
-
-# policy_keep = ['FGD', 'Clustering', 'Packing', 'DotProd', 'BestFit', 'Random']
 policy_keep = ['Random', 'DotProd', 'Clustering', 'Packing', 'BestFit', 'FGD']
 policy_keepr = ['FGD', 'BestFit', 'Packing', 'Clustering', 'DotProd', 'Random']
 
-# TYPE=list(FILEDICT.keys())[1]
-# TYPE='frag_ratio'
 # ['alloc', 'frag_amount', 'frag_ratio']
 dfnp = dfp_dict[TYPE]
 

experiments/plot/plot_paib_frag_ratio.py → experiments/plot/plot_openb_frag_ratio.py

@@ -1,19 +1,17 @@
 
 # %%
-# 1016
 import matplotlib
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from pathlib import Path
-from IPython.display import display
 from utils import parse_workload_name, POLICY_ABBR_DICT
 
 TYPE='frag_ratio'
-PAPER_PLOT=True # False: Plot with thinner lines for DingTalk or Doc usages
+PAPER_PLOT=True # False: Plot with thinner lines for Presentation
 SAVEFIG=False    # False: plt.show()
 TUNE_RATIO = 1.3
-FIGNAME = "paib_%s.pdf"
+FIGNAME = "openb_%s.pdf"
 
 workload = 'openb_pod_list_default'
 
@@ -42,10 +40,6 @@
 
     dfn['workload'] = dfn.workload.apply(parse_workload_name)
 
-    workload_order = ['EightGpu80', 'EightGpu60', 'FourGpu80', 'FourGpu60', 'TwoGpu80', 'TwoGpu60', 'OneGpu80', 'OneGpu60', 'ShareGpu80', 'ShareGpu60', 'hhpai_0820','hhpai_0905','hhpai_mvap_0820', 'hhpai_mvap_0905', 'mit', 'mvap', 'paib']
-    workload_order_dict = dict(zip(workload_order, range(1, len(workload_order)+1)))
-    dfn.workload = dfn.workload.apply(lambda x: x if x not in workload_order_dict else "%02d-%s" % (workload_order_dict[x], x))
-
     # display(dfn)
     # print("SC_POLICY_LIST=[%s]" % (",".join("'%s'" % x for x in list(dfn.sc_policy.unique()))))
 
@@ -68,33 +62,9 @@
     dfnp.sc_policy = dfnp.sc_policy.apply(lambda x: POLICY_ABBR_DICT.get(x, x))
     dfp_dict[type] = dfnp
 
-# openb, production workloads:
-
-# openb, multi-GPUs workloads
-# workload = 'cluster_openb-pod_openb-0820_a20aoc_gpu_nospec'
-# workload = 'cluster_openb-pod_openb-0820_a30aoc_gpu_nospec'
-# workload = 'cluster_openb-pod_openb-0820_a40aoc_gpu_nospec'
-# workload = 'cluster_openb-pod_openb-0820_a50aoc_gpu_nospec'
-
-# openb, non-GPU workloads
-# workload = 'cluster_openb-pod_openb-0820_nocpu_gpu_nospec'
-
-# openb, heterogeneous GPU types
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_05'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_10'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_20'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec_25'
-# workload = 'cluster_openb-pod_openb-0820_gpu_gpuspec'
-
-# mvap, GPU-sharing workloads
-# workload = 'cluster_mvap-pod_mvap-0820_nomem_no_time-cap_1.0'
-
-# policy_keep = ['FGD', 'Clustering', 'Packing', 'DotProd', 'BestFit', 'Random']
 policy_keep = ['Random', 'DotProd', 'Clustering', 'Packing', 'BestFit', 'FGD']
 policy_keepr = ['FGD', 'BestFit', 'Packing', 'Clustering', 'DotProd', 'Random']
 
-# TYPE=list(FILEDICT.keys())[1]
-# TYPE='frag_amount'
 # ['alloc', 'frag_amount', 'frag_ratio']
 dfnp = dfp_dict[TYPE]
 

experiments/plot/plot_paib_gpushare_alloc_bar.py → experiments/plot/plot_openb_gpushare_alloc_bar.py

@@ -1,22 +1,16 @@
 
 # %%
-# 1016
-from email import policy
-from ssl import ALERT_DESCRIPTION_HANDSHAKE_FAILURE
 import matplotlib
-from matplotlib import style
-from matplotlib import hatch
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from pathlib import Path
-from IPython.display import display
 from utils import parse_workload_name, POLICY_ABBR_DICT
 
-PAPER_PLOT=False # False: Plot with thinner lines for DingTalk or Doc usages
+PAPER_PLOT=False # False: Plot with thinner lines for Presentation
 SAVEFIG=True     # False: plt.show()
 TUNE_RATIO = 1.3
-FIGNAME = "paib_gpushare_alloc_bar.pdf"
+FIGNAME = "openb_gpushare_alloc_bar.pdf"
 
 # openb, GPU-sharing workloads
 workloads = ['openb_pod_list_gpushare20',
@@ -73,28 +67,13 @@
     dfnp.sc_policy = dfnp.sc_policy.apply(lambda x: POLICY_ABBR_DICT.get(x, x))
     dfp_dict[type] = dfnp
 
-
-# policy_keep = ['FGD', 'Packing', 'Clustering', 'DotProd', 'BestFit', 'Random']
 policy_keep = ['FGD', 'BestFit', 'Packing', 'Clustering', 'DotProd', 'Random']
-# policy_keep = ['BestFit', 'FGD Coarse', 'FGD']
-
 
 # ['alloc', 'frag_amount', 'frag_ratio']
 dfnp = dfp_dict['alloc']
 
 yhead = 30
 dfnpp = dfnp[dfnp.workload.isin(workloads)][dfnp.arrive_rate==100].copy()
-# print(dfnpp[dfnpp.workload == workloads[4]].groupby(by='sc_policy').mean())
-# print(dfnpp['sc_policy', ])
-# dfnpp.workload = dfnpp.workload.apply(lambda x: 
-# {
-#     'cluster_openb-pod_openb-0820_cpu037_gpu_nospec': 'CPU037',
-#     'cluster_openb-pod_openb-0820_cpu050_gpu_nospec': 'CPU050',
-#     'cluster_openb-pod_openb-0820_cpu072_gpu_nospec': 'CPU072',
-#     'cluster_openb-pod_openb-0820_cpu100_gpu_nospec': 'CPU100',
-#     'cluster_openb-pod_openb-0820_cpu200_gpu_nospec': 'CPU200',
-#     'cluster_openb-pod_openb-0820_cpu235_gpu_nospec': 'CPU235',
-# }.get(x, x))
 dfnpp.workload = dfnpp.workload.apply(lambda x:
 {
     'openb_pod_list_gpushare20': '20%',
@@ -119,16 +98,12 @@
 #     bars.bar_label(container, label_type='edge', fmt='%0.1f%%', padding=5)
 bars.bar_label(bars.containers[0], label_type='edge', fmt='%0.1f%%', padding=5)
 
-plt.xlabel('Percentage of GPUs occupied by GPU-sharing workloads')
+plt.xlabel('Percentage of GPUs occupied by GPU-sharing workloads') # plt.xlabel('Arrived Workload (in Percentage of Cluster GPU Capacity)')
 plt.ylabel('Unallocated GPU (%)')
 
 plt.legend()
-# plt.xlabel('Arrived Workload (in Percentage of Cluster GPU Capacity)')
-# plt.ylabel('Unallocated GPU (%)')
-# plt.xlim(100-yhead, None)
 plt.ylim(0, 21.7)
 # plt.title("%s" % (workload))
-# plt.show()
 
 plt.grid(linestyle='-.', alpha=0.8, axis='y')
 plt.legend(ncol=3, loc='lower center')