diff --git a/health_check.py b/health_check.py index 1f7a826..f494529 100644 --- a/health_check.py +++ b/health_check.py @@ -205,3 +205,6 @@ def health_check(): check_virtual_environment() check_python_version() check_and_install_requirements() + +if __name__ == "__main__": + health_check() \ No newline at end of file diff --git a/quisby/benchmarks/auto_hpl/extract.py b/quisby/benchmarks/auto_hpl/extract.py index bd922a8..1a0a3b4 100644 --- a/quisby/benchmarks/auto_hpl/extract.py +++ b/quisby/benchmarks/auto_hpl/extract.py @@ -3,13 +3,21 @@ from quisby.pricing import cloud_pricing from quisby.benchmarks.linpack.extract import linpack_format_data +from quisby.util import read_config + def extract_auto_hpl_data(path, system_name): + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") + if path.endswith(".csv"): with open(path) as file: results = [] file_data = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) if len(file_data) > 1: header_row = file_data[-2].strip().split(":") @@ -24,8 +32,8 @@ def extract_auto_hpl_data(path, system_name): ) if results: - return results + return results, summary_data else: - return None + return None, None diff --git a/quisby/benchmarks/coremark/compare.py b/quisby/benchmarks/coremark/compare.py index deaa893..aa07ce8 100644 --- a/quisby/benchmarks/coremark/compare.py +++ b/quisby/benchmarks/coremark/compare.py @@ -8,11 +8,33 @@ get_sheet, create_sheet, clear_sheet_data, clear_sheet_charts, ) -from quisby.util import combine_two_array_alternating, merge_lists_alternately +from quisby.util import combine_two_array_alternating, merge_lists_alternately, read_config from quisby.benchmarks.coremark.graph import graph_coremark_data +import re -def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name"]): +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None + + +def compare_inst(item1, item2): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "localhost": + return True + elif cloud_type == "aws": + return item1.split(".")[0] == item2.split(".")[0] + elif cloud_type == "gcp": + return item1.split("-")[0], item2.split("-")[0] + elif cloud_type == "azure": + return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) + + +def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -29,13 +51,23 @@ def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name= for value in list_1: for ele in list_2: # Check max throughput - if value[0][0] in table_name and ele[0][0] in table_name: - results.append([""]) - for item1 in value: - for item2 in ele: - if item1[0] == item2[0]: - results = merge_lists_alternately(results, item1, item2) - break + if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results = merge_lists_alternately(results, item1, item2) + break + + elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results.append(item1) + break elif value[1][0] == ele[1][0]: if value[0][0] == ele[0][0]: @@ -69,4 +101,4 @@ def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name= test_name = "coremark" compare_coremark_results(spreadsheets, "", test_name, - table_name=["System Name"]) + table_name=["System Name"]) \ No newline at end of file diff --git a/quisby/benchmarks/coremark/coremark.py b/quisby/benchmarks/coremark/coremark.py index b077f9a..9465a14 100644 --- a/quisby/benchmarks/coremark/coremark.py +++ b/quisby/benchmarks/coremark/coremark.py @@ -1,9 +1,17 @@ """ Custom key to sort the data base don instance name """ +from itertools import groupby + from quisby import custom_logger import re from quisby.util import read_config +from quisby.pricing.cloud_pricing import get_cloud_pricing + +from quisby.util import process_instance + +from quisby.util import mk_int + def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) @@ -15,57 +23,118 @@ def extract_prefix_and_number(input_string): return None, None, None - def custom_key(item): - cloud_type = read_config("cloud","cloud_type") + cloud_type = read_config("cloud", "cloud_type") if item[1][0] == "localhost": - return (item[1][0]) + return item[1][0] elif cloud_type == "aws": - instance_type =item[1][0].split(".")[0] - instance_number = item[1][0].split(".")[1] - return (instance_type, instance_number) + instance_name = item[1][0] + instance_type = instance_name.split(".")[0] + instance_number = instance_name.split(".")[1] + return instance_type, instance_number elif cloud_type == "gcp": instance_type = item[1][0].split("-")[0] instance_number = int(item[1][0].split('-')[-1]) - return (instance_type, instance_number) + return instance_type, instance_number + elif cloud_type == "azure": + instance_type, instance_number, version = extract_prefix_and_number(item[1][0]) + return instance_type, version, instance_number + + +def calc_price_performance(inst, avg): + region = read_config("cloud", "region") + cloud_type = read_config("cloud", "cloud_type") + os_type = read_config("test", "os_type") + cost_per_hour = None + price_perf = 0.0 + try: + cost_per_hour = get_cloud_pricing( + inst, region, cloud_type.lower(), os_type) + price_perf = float(avg)/float(cost_per_hour) + except Exception as exc: + custom_logger.debug(str(exc)) + custom_logger.error("Error calculating value !") + return cost_per_hour, price_perf + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) elif cloud_type == "azure": - instance_type, instance_number, version=extract_prefix_and_number(item[1][0]) - return (instance_type, instance_number) + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) -def create_summary_coremark_data(results,OS_RELEASE): +def create_summary_coremark_data(results, OS_RELEASE, sorted_results=None): final_results = [] - cal_data = [["System name", "test passes_"+OS_RELEASE]] # Sort data based on instance name - sorted_data = sorted(results, key=custom_key) - - # Add summary data - for item in sorted_data: - sum = 0 - avg = 0 - iterations = 0 - for index in range(3,len(item)): - sum = sum + float(item[index][1]) - iterations = iterations + 1 - avg = float(sum/iterations) - cal_data.append([item[1][0],avg]) - - final_results += item - final_results += [[""]] - final_results += cal_data + results = list(filter(None, results)) + sort_data(results) + + for _, items in group_data(results): + cal_data = [["System name", "Test_passes-" + OS_RELEASE]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + # sorted_results.extend(sorted_data) + cost_per_hour, price_per_perf = [], [] + + # Add summary data + for item in sorted_data: + sum = 0 + avg = 0 + iterations = 0 + for index in range(3, len(item)): + sum = sum + float(item[index][1]) + iterations = iterations + 1 + avg = float(sum/iterations) + try: + cph, pp = calc_price_performance(item[1][0], avg) + except Exception as exc: + custom_logger.error(str(exc)) + break + cal_data.append([item[1][0], avg]) + price_per_perf.append([item[1][0], pp]) + cost_per_hour.append([item[1][0], cph]) + sorted_results=[[""]] + sorted_results += cal_data + sorted_results.append([""]) + sorted_results.append(["Cost/Hr"]) + sorted_results += cost_per_hour + sorted_results.append([""]) + sorted_results.append(["Price-perf", f"Passes/$-{OS_RELEASE}"]) + sorted_results += price_per_perf + final_results.extend(sorted_results) return final_results + def extract_coremark_data(path, system_name, OS_RELEASE): """""" results = [] processed_data =[] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith(".csv"): with open(path) as file: coremark_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: return None except Exception as exc: @@ -88,5 +157,7 @@ def extract_coremark_data(path, system_name, OS_RELEASE): iteration = iteration + 1 results.append(processed_data) - return results + return results, summary_data + + diff --git a/quisby/benchmarks/coremark/graph.py b/quisby/benchmarks/coremark/graph.py index 8494a95..3880cc3 100644 --- a/quisby/benchmarks/coremark/graph.py +++ b/quisby/benchmarks/coremark/graph.py @@ -101,13 +101,21 @@ def graph_coremark_data(spreadsheetId, range, action): header_row = [] sheetId = -1 + for index, row in enumerate(data): if "System name" in row: start_index = index header_row.extend(row) + title = "%s : %s" % (range, "Test Passes") + subtitle = "Average Test Passes" + elif "Price-perf" in row: + start_index = index + header_row.extend(row) + title = "%s : %s" % (range, "Price-Performance") + subtitle = "Passes/$" if start_index: if not row: - end_index = index - 1 + end_index = index if index + 1 == len(data): end_index = index + 1 @@ -124,20 +132,35 @@ def graph_coremark_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s" % (range, "Test passes"), + "title": title, + "subtitle": subtitle + " : "+graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ { - "position": "LEFT_AXIS", - "title": "Test passes" - }, - { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "BOTTOM_AXIS", - "title": "Machine types", + "title": "System"}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + "position": "LEFT_AXIS", + "title": graph_data[0][1].split("-")[0], }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -168,8 +191,11 @@ def graph_coremark_data(spreadsheetId, range, action): "anchorCell": { "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, - "columnIndex": column_count + 1, - } + "columnIndex": column_count + GRAPH_COL_INDEX, + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } @@ -199,4 +225,3 @@ def graph_coremark_data(spreadsheetId, range, action): update_conditional_formatting(spreadsheetId, sheetId, col, threshold) - diff --git a/quisby/benchmarks/coremark_pro/compare.py b/quisby/benchmarks/coremark_pro/compare.py index 2cbb69a..333ba79 100644 --- a/quisby/benchmarks/coremark_pro/compare.py +++ b/quisby/benchmarks/coremark_pro/compare.py @@ -8,7 +8,28 @@ get_sheet, create_sheet, clear_sheet_data, clear_sheet_charts, ) -from quisby.util import merge_lists_alternately +from quisby.util import merge_lists_alternately,read_config +import re + + +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + return prefix + return None + + +def compare_inst(item1, item2): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "localhost": + return True + elif cloud_type == "aws": + return item1.split(".")[0] == item2.split(".")[0] + elif cloud_type == "gcp": + return item1.split("-")[0], item2.split("-")[0] + elif cloud_type == "azure": + return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) def compare_coremark_pro_results(spreadsheets, spreadsheetId, test_name, table_name=["System name"]): @@ -28,15 +49,24 @@ def compare_coremark_pro_results(spreadsheets, spreadsheetId, test_name, table_n for value in list_1: for ele in list_2: # Check max throughput - if value[0][0] == ele[0][0] and value[1][0] == ele[1][0]: - if value[1][0].split(".")[0] == ele[1][0].split(".")[0]: + if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results = merge_lists_alternately(results, item1, item2) + break + + elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": + if compare_inst(value[1][0], ele[1][0]): results.append([""]) for item1 in value: for item2 in ele: if item1[0] == item2[0]: - results = merge_lists_alternately(results, item1, item2) + results.append(item1) break + elif value[1][0] == ele[1][0]: if value[0][0] == ele[0][0]: results.append([""]) @@ -67,4 +97,4 @@ def compare_coremark_pro_results(spreadsheets, spreadsheetId, test_name, table_n test_name = "coremark_pro" compare_coremark_pro_results(spreadsheets, "", test_name, - table_name=["System Name"]) + table_name=["System Name"]) \ No newline at end of file diff --git a/quisby/benchmarks/coremark_pro/coremark_pro.py b/quisby/benchmarks/coremark_pro/coremark_pro.py index 52c3e57..25fa0c1 100644 --- a/quisby/benchmarks/coremark_pro/coremark_pro.py +++ b/quisby/benchmarks/coremark_pro/coremark_pro.py @@ -1,7 +1,11 @@ import re +from itertools import groupby from quisby import custom_logger from quisby.util import read_config +from quisby.pricing.cloud_pricing import get_cloud_pricing + +from quisby.util import process_instance, mk_int def extract_prefix_and_number(input_string): @@ -29,47 +33,116 @@ def custom_key(item): return instance_type, instance_number elif cloud_type == "azure": instance_type, instance_number, version = extract_prefix_and_number(item[1][0]) - return instance_type, instance_number + return instance_type, version, instance_number except Exception as exc: custom_logger.error(str(exc)) return "", "" +def calc_price_performance(inst, avg): + region = read_config("cloud", "region") + cloud_type = read_config("cloud", "cloud_type") + os_type = read_config("test", "os_type") + cost_per_hour = None + price_perf = 0.0 + try: + cost_per_hour = get_cloud_pricing( + inst, region, cloud_type.lower(), os_type) + price_perf = float(avg)/float(cost_per_hour) + except Exception as exc: + custom_logger.debug(str(exc)) + custom_logger.error("Error calculating value !") + return cost_per_hour, price_perf + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + + def create_summary_coremark_pro_data(results, OS_RELEASE): - final_results = [] - multi_iter = [["Multi Iterations"], ["System name", "Score_" + OS_RELEASE]] - single_iter = [["Single Iterations"], ["System name", "Score_" + OS_RELEASE]] + ret_results = [] + # Sort data based on instance name - sorted_data = sorted(results, key=custom_key) - # Add summary data - for item in sorted_data: - for index in range(3, len(item)): - multi_iter.append([item[1][0], item[index][1]]) - single_iter.append([item[1][0], item[index][2]]) - # final_results += item - final_results += [[""]] - final_results += multi_iter - final_results += [[""]] - final_results += single_iter - return final_results + results = list(filter(None, results)) + sort_data(results) + results = group_data(results) + for _, items in results: + multi_iter = [["Multi Iterations"], ["System name", "Score-" + OS_RELEASE]] + single_iter = [["Single Iterations"], ["System name", "Score-" + OS_RELEASE]] + cal_data = [["System name", "Test_passes-" + OS_RELEASE]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + # Add summary data + cost_per_hour, price_perf_single, price_perf_multi = [], [],[] + for item in sorted_data: + for index in range(3, len(item)): + multi_iter.append([item[1][0], item[index][1]]) + single_iter.append([item[1][0], item[index][2]]) + try: + cph, ppm = calc_price_performance(item[1][0], item[index][1]) + cph, pps = calc_price_performance(item[1][0], item[index][2]) + except Exception as exc: + custom_logger.error(str(exc)) + break + price_perf_multi.append([item[1][0], ppm]) + price_perf_single.append([item[1][0], pps]) + cost_per_hour.append([item[1][0], cph]) + # final_results += item + final_results =[[""]] + final_results += single_iter + final_results.append([""]) + final_results.append(["Cost/Hr"]) + final_results += cost_per_hour + final_results.extend([[""], ["Single Iterations"]]) + final_results.append(["Price-perf", f"Score/$-{OS_RELEASE}"]) + final_results += price_perf_single + final_results += [[""]] + final_results += multi_iter + final_results.extend([[""], ["Multi Iterations"]]) + final_results.append(["Price-perf", f"Score/$-{OS_RELEASE}"]) + final_results += price_perf_multi + ret_results.extend(final_results) + return ret_results def extract_coremark_pro_data(path, system_name, OS_RELEASE): """""" results = [] processed_data = [] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith(".csv"): with open(path) as file: coremark_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: - return None + return None,None except Exception as exc: custom_logger.debug(str(exc)) custom_logger.error("Unable to extract data from csv file for coremark_pro") - return None + return None, None for index, data in enumerate(coremark_results): coremark_results[index] = data.strip("\n").split(":") @@ -84,4 +157,4 @@ def extract_coremark_pro_data(path, system_name, OS_RELEASE): elif "Score" in row: processed_data.append(["Score", row[1], row[2]]) results.append(processed_data) - return results + return results, summary_data \ No newline at end of file diff --git a/quisby/benchmarks/coremark_pro/graph.py b/quisby/benchmarks/coremark_pro/graph.py index 126196d..af9eebd 100644 --- a/quisby/benchmarks/coremark_pro/graph.py +++ b/quisby/benchmarks/coremark_pro/graph.py @@ -103,12 +103,19 @@ def graph_coremark_pro_data(spreadsheetId, range, action): header = [] sheetId = -1 - for index, row in enumerate(data): if "System name" in row: start_index = index + header.extend(row) iteration = data[index - 1][0] + title = "%s : %s" % (range, "Score") + subtitle = iteration + elif "Price-perf" in row: + start_index = index header.extend(row) + iteration = data[index - 1][0] + title = "%s : %s" % (range, "Price-Performance") + subtitle = "%s : %s" % ("Score/$", iteration) if start_index: if not row: end_index = index @@ -130,17 +137,35 @@ def graph_coremark_pro_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s : %s" % (range, "score", iteration), + "title": title, + "subtitle": subtitle+" : "+graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ - {"position": "BOTTOM_AXIS", "title": ""}, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + "position": "BOTTOM_AXIS", + "title": "System"}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", - "title": "Score", + "title": graph_data[0][1].split("-")[0], }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -172,7 +197,10 @@ def graph_coremark_pro_data(spreadsheetId, range, action): "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, "columnIndex": column_count + GRAPH_COL_INDEX, - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } @@ -183,7 +211,7 @@ def graph_coremark_pro_data(spreadsheetId, range, action): GRAPH_ROW_INDEX += 20 GRAPH_COL_INDEX = 1 else: - GRAPH_ROW_INDEX = end_index + GRAPH_COL_INDEX += 6 body = {"requests": requests} @@ -199,4 +227,4 @@ def graph_coremark_pro_data(spreadsheetId, range, action): if not threshold: threshold = "5" for col in diff_col: - update_conditional_formatting(spreadsheetId, sheetId, col, threshold) + update_conditional_formatting(spreadsheetId, sheetId, col, threshold) \ No newline at end of file diff --git a/quisby/benchmarks/linpack/comparison.py b/quisby/benchmarks/linpack/comparison.py index f2c5f5d..e28cb71 100644 --- a/quisby/benchmarks/linpack/comparison.py +++ b/quisby/benchmarks/linpack/comparison.py @@ -38,7 +38,7 @@ def compare_linpack_results(spreadsheets, spreadsheetId, test_name): value[4], value[5], ele[5], - "Price/Perf % Diff", + "Price-perf % Diff", ] ) break diff --git a/quisby/benchmarks/linpack/extract.py b/quisby/benchmarks/linpack/extract.py index 0b21c7d..ac7bd57 100644 --- a/quisby/benchmarks/linpack/extract.py +++ b/quisby/benchmarks/linpack/extract.py @@ -31,16 +31,6 @@ def linpack_format_data(**kwargs): system_name, region, cloud_type.lower() ) - results.append( - [ - "System", - "Cores", - f"GFLOPS-{os_release}", - f"GFLOP Scaling-{os_release}", - "Cost/hr", - f"Price/Perf-{os_release}", - ] - ) results.append( [ system_name, @@ -64,6 +54,9 @@ def extract_linpack_data(path, system_name): results = [] no_of_cores = None gflops = None + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") summary_file = path @@ -77,8 +70,10 @@ def extract_linpack_data(path, system_name): last_row = list_data[-1] gflops = last_row["MB/sec"] threads = last_row["threads"] + sum_path = summary_file.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: - return results + return results, summary_data for file_path in glob.glob(path + f"/linpack*_threads_{threads}_*"): with open(file_path) as txt_file: @@ -96,4 +91,4 @@ def extract_linpack_data(path, system_name): gflops=gflops, ) - return results + return results, summary_data diff --git a/quisby/benchmarks/linpack/graph.py b/quisby/benchmarks/linpack/graph.py index 2ac7ddd..e9b0d2d 100644 --- a/quisby/benchmarks/linpack/graph.py +++ b/quisby/benchmarks/linpack/graph.py @@ -45,7 +45,7 @@ def graph_linpack_compare(spreadsheetId, test_name, action): Graphs: - GFLOP and GFLOPS scaling - - Price/perf + - Price-perf :sheet: sheet API function :spreadsheetId @@ -91,10 +91,10 @@ def graph_linpack_compare(spreadsheetId, test_name, action): "addChart": { "chart": { "spec": { - "title": "%s and %s" % (header_row[2], header_row[3]), + "title": "%s : %s and %s" % (test_name, header_row[2], header_row[3]), "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ { "position": "BOTTOM_AXIS", @@ -203,15 +203,15 @@ def graph_linpack_compare(spreadsheetId, test_name, action): sheet.batchUpdate(spreadsheetId=spreadsheetId, body=body).execute() - # PRICE/PERF graph + # Price-perf graph requests = { "addChart": { "chart": { "spec": { - "title": "%s and %s" % (header_row[9], header_row[10]), + "title": "%s : Price-Performance" % (test_name), "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ { "position": "BOTTOM_AXIS", @@ -324,16 +324,27 @@ def graph_linpack_compare(spreadsheetId, test_name, action): "addChart": { "chart": { "spec": { - "title": "GFLOP Scaling", + "title": "%s : GFLOP Scaling" %(test_name), "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "BOTTOM_AXIS", "title": "", }, - {"position": "LEFT_AXIS", "title": "GFlops Scaling "}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + "position": "LEFT_AXIS", "title": "GFlops Scaling "}, ], "domains": [ { @@ -414,7 +425,10 @@ def graph_linpack_compare(spreadsheetId, test_name, action): "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, "columnIndex": 12, - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } @@ -448,7 +462,7 @@ def graph_linpack_data(spreadsheetId, test_name, action): Graphs: - GFLOP and GFLOPS scaling - - Price/perf + - Price-perf :sheet: sheet API function :spreadsheetId @@ -460,10 +474,10 @@ def graph_linpack_data(spreadsheetId, test_name, action): GFLOPS_PLOT_RANGE = "C" PRICE_PER_PERF_RANGE = "E" GRAPH_COL_INDEX = 5 - GRAPH_ROW_INDEX = 0 - start_index, end_index = None, None - data = read_sheet(spreadsheetId, test_name) + last_row = len(data) + GRAPH_ROW_INDEX = last_row + 1 + start_index, end_index = None, None header_row = data[0] for index, row in enumerate(data): @@ -490,20 +504,30 @@ def graph_linpack_data(spreadsheetId, test_name, action): "addChart": { "chart": { "spec": { - "title": "%s and %s" - % (header_row[2], header_row[3]), + "title": "%s : %s and %s" + % (test_name, header_row[2].split("-")[0], header_row[3].split("-")[0]), "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "BOTTOM_AXIS", "title": "%s" % (header_row[0]), }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", "title": "%s and %s" - % (header_row[2], header_row[3]), + % (header_row[2].lower(), header_row[3].lower()), }, ], "domains": [ @@ -567,15 +591,18 @@ def graph_linpack_data(spreadsheetId, test_name, action): "anchorCell": { "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, - "columnIndex": column_count + 1, - } + "columnIndex": 0, + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } } } - # PRICE/PERF graph + # Price-perf graph body = {"requests": requests} sheet.batchUpdate(spreadsheetId=spreadsheetId, body=body).execute() @@ -584,16 +611,26 @@ def graph_linpack_data(spreadsheetId, test_name, action): "addChart": { "chart": { "spec": { - "title": "%s " % (header_row[5]), + "title": "%s : Price-Performance" %(test_name), "basicChart": { "chartType": "COLUMN", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "BOTTOM_AXIS", "title": "%s" % (header_row[0]), }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", "title": "%s " % (header_row[5]), }, @@ -642,8 +679,11 @@ def graph_linpack_data(spreadsheetId, test_name, action): "anchorCell": { "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, - "columnIndex": column_count + 7, - } + "columnIndex": 6, + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } diff --git a/quisby/benchmarks/linpack/summary.py b/quisby/benchmarks/linpack/summary.py index c7e0e1c..6330cf2 100644 --- a/quisby/benchmarks/linpack/summary.py +++ b/quisby/benchmarks/linpack/summary.py @@ -28,19 +28,50 @@ def custom_key(item): return instance_type, instance_number elif cloud_type == "azure": instance_type, instance_number, version=extract_prefix_and_number(item[0]) - return instance_type, instance_number + return instance_type, version, instance_number + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[0], "family", "version","sub_family","feature")) + + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[0], "family", "version", "sub_family"))) def create_summary_linpack_data(results, OS_RELEASE): sorted_results = [] + header = [] + header.append( + [ + "System", + "Cores", + f"GFLOPS-{OS_RELEASE}", + f"GFLOP Scaling-{OS_RELEASE}", + "Cost/hr", + f"Price-perf-{OS_RELEASE}", + ] + ) results = list(filter(None, results)) - header_row = [results[0]] - results = [row for row in results if row[0] != "System"] - - results.sort(key=lambda x: str(process_instance(x[0], "family", "version", "feature"))) + sort_data(results) + #results.sort(key=lambda x: str((x[0], "family", "version","sub_family", "feature"))) - for _, items in groupby(results, key=lambda x: process_instance(x[0], "family", "version", "feature")): + for _, items in group_data(results): items = list(items) sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[0], "size"))) cpu_scale, base_gflops = None, None @@ -55,11 +86,10 @@ def create_summary_linpack_data(results, OS_RELEASE): cpu_scaling = 0 gflops_scaling = float(row[2]) / (int(row[1]) - cpu_scale) / base_gflops if cpu_scaling != 0 else 1 sorted_data[index][3] = format(gflops_scaling, ".4f") - sorted_data = sorted(sorted_data, key=custom_key) res = [] for item in sorted_data: res.append(item) - sorted_results += header_row + res + sorted_results += header + res # sorted_results += header_row + sorted_data return sorted_results diff --git a/quisby/benchmarks/passmark/compare.py b/quisby/benchmarks/passmark/compare.py index f8a3467..5dc0c17 100644 --- a/quisby/benchmarks/passmark/compare.py +++ b/quisby/benchmarks/passmark/compare.py @@ -8,10 +8,32 @@ get_sheet, create_sheet, clear_sheet_data, clear_sheet_charts, ) -from quisby.util import merge_lists_alternately +from quisby.util import merge_lists_alternately,read_config +import re -def compare_passmark_results(spreadsheets, spreadsheetId, test_name, table_name=["SYSTEM_NAME"]): +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None + + +def compare_inst(item1, item2): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "localhost": + return True + elif cloud_type == "aws": + return item1.split(".")[0] == item2.split(".")[0] + elif cloud_type == "gcp": + return item1.split("-")[0], item2.split("-")[0] + elif cloud_type == "azure": + return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) + + +def compare_passmark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -28,14 +50,21 @@ def compare_passmark_results(spreadsheets, spreadsheetId, test_name, table_name= for value in list_1: for ele in list_2: # Check max throughput - if value[0][0] in table_name and ele[0][0] in table_name: + if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: results.append([""]) - results.append(value[0]) - for item1 in value[1:]: - for item2 in ele[1:]: + for item1 in value: + for item2 in ele: if item1[0] == item2[0]: results = merge_lists_alternately(results, item1, item2) break + elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results.append(item1) + break elif value[1][0] == ele[1][0]: if value[0][0] == ele[0][0]: @@ -67,4 +96,4 @@ def compare_passmark_results(spreadsheets, spreadsheetId, test_name, table_name= test_name = "passmark" compare_passmark_results(spreadsheets, "", test_name, - table_name=["SYSTEM_NAME"]) + table_name=["SYSTEM_NAME"]) \ No newline at end of file diff --git a/quisby/benchmarks/passmark/graph.py b/quisby/benchmarks/passmark/graph.py index 3acf5ab..5866799 100644 --- a/quisby/benchmarks/passmark/graph.py +++ b/quisby/benchmarks/passmark/graph.py @@ -90,12 +90,13 @@ def create_series_range_list_passmark_compare(column_count, sheetId, start_index def graph_passmark_data(spreadsheetId, range, action): - GRAPH_COL_INDEX = 3 - GRAPH_ROW_INDEX = 10 + GRAPH_COL_INDEX = 1 + GRAPH_ROW_INDEX = 1 start_index = 0 end_index = 0 sheetId = -1 diff_col = [3] + row_val = 1 data = read_sheet(spreadsheetId, range) @@ -104,11 +105,23 @@ def graph_passmark_data(spreadsheetId, range, action): for index, row in enumerate(data): for col in row: - if "GEOMEAN" in col: + if "System name" in col: start_index = index + if row_val == 1: + row_val = start_index + title = "%s : %s" % (range, "Geomean") + subtitle = "" + + elif "Price-perf" in row: + start_index = index + if row_val == 1: + row_val = start_index + title = "%s : %s" % (range, "Price-Performance") + subtitle = "Geomean/$" + if start_index: if not row: - end_index = index - 1 + end_index = index if index + 1 == len(data): end_index = index + 1 @@ -127,17 +140,35 @@ def graph_passmark_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s" % (range, "GEOMEAN"), + "title": title, + "subtitle": graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ - {"position": "BOTTOM_AXIS", "title": ""}, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + "position": "BOTTOM_AXIS", + "title": "System"}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", - "title": "Geomean", + "title": graph_data[0][1].lower(), }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -167,9 +198,12 @@ def graph_passmark_data(spreadsheetId, range, action): "overlayPosition": { "anchorCell": { "sheetId": sheetId, - "rowIndex": GRAPH_ROW_INDEX, + "rowIndex": GRAPH_ROW_INDEX , "columnIndex": column_count + GRAPH_COL_INDEX, - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } @@ -196,4 +230,4 @@ def graph_passmark_data(spreadsheetId, range, action): if not threshold: threshold = "5" for col in diff_col: - update_conditional_formatting(spreadsheetId, sheetId, col, threshold) + update_conditional_formatting(spreadsheetId, sheetId, col, threshold) \ No newline at end of file diff --git a/quisby/benchmarks/passmark/passmark.py b/quisby/benchmarks/passmark/passmark.py index e43ee68..6e67b04 100644 --- a/quisby/benchmarks/passmark/passmark.py +++ b/quisby/benchmarks/passmark/passmark.py @@ -1,7 +1,23 @@ +from itertools import groupby + from scipy.stats import gmean from quisby import custom_logger from quisby.util import read_config +from quisby.pricing.cloud_pricing import get_cloud_pricing +import re + +from quisby.util import process_instance, mk_int + + +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + number = int(match.group(2)) + suffix = match.group(3) + return prefix, number, suffix + return None, None, None def custom_key(item): @@ -16,69 +32,110 @@ def custom_key(item): instance_type = item[0].split("-")[0] instance_number = int(item[0].split('-')[-1]) return instance_type, instance_number + elif cloud_type == "azure": + instance_type, instance_number, version= extract_prefix_and_number(item[0]) + return instance_type, version, instance_number + + +def calc_price_performance(inst, avg): + region = read_config("cloud", "region") + cloud_type = read_config("cloud", "cloud_type") + os_type = read_config("test", "os_type") + cost_per_hour = None + try: + cost_per_hour = get_cloud_pricing( + inst, region, cloud_type.lower(), os_type) + price_perf = float(avg)/float(cost_per_hour) + except Exception as exc: + custom_logger.debug(str(exc)) + custom_logger.error("Error calculating value !") + return cost_per_hour, price_perf + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) def create_summary_passmark_data(data, OS_RELEASE): - results = [ - ['SYSTEM', "NO_OF_TEST_PROCESSES", "CPU_INTEGER_MATH", "CPU_FLOATINGPOINT_MATH", "CPU_PRIME", "CPU_SORTING", - "CPU_ENCRYPTION", "CPU_COMPRESSION", "CPU_SINGLETHREAD", "CPU_PHYSICS", "CPU_MATRIX_MULT_SSE", "CPU_mm", - "CPU_sse", "CPU_fma", "CPU_avx", "CPU_avx512", "m_CPU_enc_SHA", "m_CPU_enc_AES", "m_CPU_enc_ECDSA", - "ME_ALLOC_S", "ME_READ_S", "ME_READ_L", "ME_WRITE", "ME_LARGE", "ME_LATENCY", "ME_THREADED", "SUMM_CPU", - "SUMM_ME" - ]] - processed_data = None - gmean_data = [] - SYSTEM_GEOMEAN = [] - end_index = 0 - start_index = 0 - system = "" - # Add summary data - for index, row in enumerate(data): - if row == [""]: - if processed_data: - results.append(processed_data) - SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - processed_data = [] + ret_results = [] + results = list(filter(None, data)) + sort_data(results) + results = group_data(results) + for _, items in results: + mac_data = [["System name", "Geomean-" + OS_RELEASE]] + cost_data = [["Cost/Hr"]] + price_perf_data = [["Price-perf", f"Geomean/$-{OS_RELEASE}"]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + cost_per_hour, price_per_perf = [], [] + # Add summary data + for index, row in enumerate(sorted_data): + inst = row[1][0] gmean_data = [] - system = "" - start_index = end_index + 1 - end_index = 0 - elif start_index: - system = row[0] - processed_data.append(system) - end_index = start_index + 1 - start_index = 0 - elif end_index: - if not row[0] == 'NumTestProcesses': - gmean_data.append(float(row[1])) - processed_data.append(row[1]) - results.append(processed_data) - SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - results.append([""]) - results.append(["SYSTEM_NAME", "GEOMEAN-" + str(OS_RELEASE)]) - sorted_data = sorted(SYSTEM_GEOMEAN, key=custom_key) - for item in sorted_data: - results.append(item) - return results + for i in range(2, len(row)): + try: + gmean_data.append(float(row[i][1].strip())) + except Exception as exc: + gmean_data.append(0.0) + gdata = gmean(gmean_data) + try: + cph, pp = calc_price_performance(inst, gdata) + except Exception as exc: + custom_logger.error(str(exc)) + continue + + mac_data.append([inst, gdata]) + cost_data.append([inst, cph]) + price_perf_data.append([inst, pp]) + ret_results.append([""]) + ret_results.extend(mac_data) + ret_results.append([""]) + ret_results.extend(cost_data) + ret_results.append([""]) + ret_results.extend(price_perf_data) + return ret_results def extract_passmark_data(path, system_name, OS_RELEASE): """""" results = [] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith("results.csv"): with open(path) as file: coremark_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) + else: return None except Exception as exc: custom_logger.error(str(exc)) - return None + return None, None for index, data in enumerate(coremark_results): coremark_results[index] = data.strip("\n").split(":") results.append([""]) results.append([system_name]) results.extend(coremark_results) - return results + return [results], summary_data \ No newline at end of file diff --git a/quisby/benchmarks/phoronix/compare.py b/quisby/benchmarks/phoronix/compare.py index e5025a8..c6c6e79 100644 --- a/quisby/benchmarks/phoronix/compare.py +++ b/quisby/benchmarks/phoronix/compare.py @@ -8,10 +8,31 @@ get_sheet, create_sheet, clear_sheet_data, clear_sheet_charts, ) -from quisby.util import merge_lists_alternately +from quisby.util import merge_lists_alternately,read_config +import re +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None -def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name=["SYSTEM_NAME"]): + +def compare_inst(item1, item2): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "localhost": + return True + elif cloud_type == "aws": + return item1.split(".")[0] == item2.split(".")[0] + elif cloud_type == "gcp": + return item1.split("-")[0], item2.split("-")[0] + elif cloud_type == "azure": + return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) + + +def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name=["System name", "Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -28,13 +49,23 @@ def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name= for value in list_1: for ele in list_2: # Check max throughput - if value[0][0] in table_name and ele[0][0] in table_name: - results.append([""]) - for item1 in value: - for item2 in ele: - if item1[0] == item2[0]: - results = merge_lists_alternately(results, item1, item2) - break + if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results = merge_lists_alternately(results, item1, item2) + break + + elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results.append(item1) + break elif value[1][0] == ele[1][0]: if value[0][0] == ele[0][0]: @@ -66,4 +97,4 @@ def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name= test_name = "phoronix" graph_phoronix_data(spreadsheets, "", test_name, - table_name=["SYSTEM_NAME"]) + table_name=["SYSTEM_NAME"]) \ No newline at end of file diff --git a/quisby/benchmarks/phoronix/graph.py b/quisby/benchmarks/phoronix/graph.py index 14bb1ea..b7a4a88 100644 --- a/quisby/benchmarks/phoronix/graph.py +++ b/quisby/benchmarks/phoronix/graph.py @@ -103,11 +103,18 @@ def graph_phoronix_data(spreadsheetId, range, action): for index, row in enumerate(data): for col in row: - if "GEOMEAN" in col: + if "System name" in col: start_index = index + title = "%s : %s" % (range, "Geomean") + subtitle="" + elif "Price-perf" in row: + start_index = index + title = "%s : %s" % (range, "Price-Performance") + subtitle = "Geomean/$" + if start_index: if not row: - end_index = index - 1 + end_index = index if index + 1 == len(data): end_index = index + 1 @@ -126,17 +133,36 @@ def graph_phoronix_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s" % (range, "GEOMEAN"), + "title": title, + "subtitle": graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ - {"position": "BOTTOM_AXIS", "title": ""}, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + "position": "BOTTOM_AXIS", + "title": "System" + }, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", - "title": "Geomean", + "title": graph_data[0][1].lower(), }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -168,7 +194,10 @@ def graph_phoronix_data(spreadsheetId, range, action): "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, "columnIndex": column_count + GRAPH_COL_INDEX, - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } @@ -195,4 +224,4 @@ def graph_phoronix_data(spreadsheetId, range, action): if not threshold: threshold = "5" for col in diff_col: - update_conditional_formatting(spreadsheetId, sheetId, col, threshold) + update_conditional_formatting(spreadsheetId, sheetId, col, threshold) \ No newline at end of file diff --git a/quisby/benchmarks/phoronix/phoronix.py b/quisby/benchmarks/phoronix/phoronix.py index 83ab3c8..0a3e8bb 100644 --- a/quisby/benchmarks/phoronix/phoronix.py +++ b/quisby/benchmarks/phoronix/phoronix.py @@ -1,9 +1,14 @@ +from itertools import groupby + from scipy.stats import gmean from quisby import custom_logger from quisby.util import read_config +from quisby.pricing.cloud_pricing import get_cloud_pricing import re +from quisby.util import process_instance, mk_int + def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) @@ -28,52 +33,99 @@ def custom_key(item): instance_number = int(item[0].split('-')[-1]) return instance_type, instance_number elif cloud_type == "azure": - instance_type, instance_number, version=extract_prefix_and_number(item[0]) - return (instance_type, instance_number) + instance_type, instance_number, version= extract_prefix_and_number(item[0]) + return instance_type, version, instance_number + + +def calc_price_performance(inst, avg): + region = read_config("cloud", "region") + cloud_type = read_config("cloud", "cloud_type") + os_type = read_config("test", "os_type") + cost_per_hour = None + try: + cost_per_hour = get_cloud_pricing( + inst, region, cloud_type.lower(), os_type) + price_perf = float(avg)/float(cost_per_hour) + except Exception as exc: + custom_logger.debug(str(exc)) + custom_logger.error("Error calculating value !") + return cost_per_hour, price_perf + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) def create_summary_phoronix_data(data, OS_RELEASE): - results = [] - processed_data = None - gmean_data = [] - SYSTEM_GEOMEAN = [] - end_index = 0 - start_index = 0 - system = "" - # Add summary data - for index, row in enumerate(data): - if row == [""]: - if processed_data: - SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - processed_data = [] - gmean_data = [] - system = "" - start_index = end_index + 1 - end_index = 0 - elif start_index: - system = row[0] - processed_data.append(system) - end_index = start_index + 1 - start_index = 0 - elif end_index: - gmean_data.append(float(row[1])) - SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - results.append([""]) - results.append(["SYSTEM_NAME", "GEOMEAN-" + str(OS_RELEASE)]) - sorted_data = sorted(SYSTEM_GEOMEAN, key=custom_key) - for item in sorted_data: - results.append(item) - return results + ret_results = [] + + results = list(filter(None, data)) + sort_data(results) + results = group_data(results) + for _, items in results: + mac_data = [["System name", "Geomean-" + OS_RELEASE]] + cost_data = [["Cost/Hr"]] + price_perf_data = [["Price-perf",f"Geomean/$-{OS_RELEASE}"]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + cost_per_hour, price_per_perf = [], [] + # Add summary data + for index, row in enumerate(sorted_data): + inst = row[1][0] + gmean_data=[] + for i in range(2,len(row)): + try: + gmean_data.append(float(row[i][1].strip())) + except Exception as exc: + gmean_data.append(0.0) + gdata = gmean(gmean_data) + try: + cph, pp = calc_price_performance(inst, gdata) + except Exception as exc: + custom_logger.error(str(exc)) + continue + + mac_data.append([inst, gdata]) + cost_data.append([inst, cph]) + price_perf_data.append([inst, pp]) + ret_results.append([""]) + ret_results.extend(mac_data) + ret_results.append([""]) + ret_results.extend(cost_data) + ret_results.append([""]) + ret_results.extend(price_perf_data) + return ret_results def extract_phoronix_data(path, system_name, OS_RELEASE): """""" results = [] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith("results.csv"): with open(path) as file: phoronix_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: return None except Exception as exc: @@ -85,4 +137,4 @@ def extract_phoronix_data(path, system_name, OS_RELEASE): results.append([""]) results.append([system_name]) results.extend(phoronix_results[1:]) - return results + return [results], summary_data \ No newline at end of file diff --git a/quisby/benchmarks/pyperf/compare.py b/quisby/benchmarks/pyperf/compare.py index e6610df..ff4e3c4 100644 --- a/quisby/benchmarks/pyperf/compare.py +++ b/quisby/benchmarks/pyperf/compare.py @@ -7,10 +7,32 @@ get_sheet, create_sheet, clear_sheet_data, clear_sheet_charts, ) -from quisby.util import merge_lists_alternately +from quisby.util import merge_lists_alternately,read_config +import re -def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=["SYSTEM_NAME"]): +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None + + +def compare_inst(item1, item2): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "localhost": + return True + elif cloud_type == "aws": + return item1.split(".")[0] == item2.split(".")[0] + elif cloud_type == "gcp": + return item1.split("-")[0], item2.split("-")[0] + elif cloud_type == "azure": + return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) + + +def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -27,13 +49,29 @@ def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=[" for value in list_1: for ele in list_2: # Check max throughput - if value[0][0] in table_name and ele[0][0] in table_name: - results.append([""]) - for item1 in value: - for item2 in ele: - if item1[0] == item2[0]: - results = merge_lists_alternately(results, item1, item2) - break + if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results = merge_lists_alternately(results, item1, item2) + + elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results.append(item1) + + elif value[1][0] == ele[1][0]: + if value[0][0] == ele[0][0]: + results.append([""]) + results.append(value[0]) + for item1, item2 in zip(value[1:], ele[1:]): + results = merge_lists_alternately(results, item1, item2) + try: create_sheet(spreadsheetId, test_name) @@ -56,4 +94,4 @@ def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=[" test_name = "pyperf" compare_pyperf_results(spreadsheets, "", test_name, - table_name=["SYSTEM_NAME"]) + table_name=["System name"]) \ No newline at end of file diff --git a/quisby/benchmarks/pyperf/graph.py b/quisby/benchmarks/pyperf/graph.py index 646b3a8..62ad112 100644 --- a/quisby/benchmarks/pyperf/graph.py +++ b/quisby/benchmarks/pyperf/graph.py @@ -96,6 +96,7 @@ def graph_pyperf_data(spreadsheetId, range, action): end_index = 0 sheetId = -1 diff_col = [3] + row_val = 1 data = read_sheet(spreadsheetId, range) if len(data) > 500: @@ -103,11 +104,21 @@ def graph_pyperf_data(spreadsheetId, range, action): for index, row in enumerate(data): for col in row: - if "GEOMEAN" in col: + if "System name" in col: start_index = index + if row_val == 1: + row_val = start_index + title = "%s : %s" % (range, "Geomean") + subtitle = "" + elif "Price-perf" in row: + start_index = index + if row_val == 1: + row_val = start_index + title = "%s : %s" % (range, "Price-Performance") + subtitle = "Geomean/$" if start_index: if not row: - end_index = index - 1 + end_index = index if index + 1 == len(data): end_index = index + 1 @@ -126,17 +137,35 @@ def graph_pyperf_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s" % (range, "GEOMEAN"), + "title": title, + "subtitle": graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ - {"position": "BOTTOM_AXIS", "title": ""}, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + "position": "BOTTOM_AXIS", + "title": "System"}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", - "title": "geomean", + "title": graph_data[0][1].lower(), }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -166,9 +195,12 @@ def graph_pyperf_data(spreadsheetId, range, action): "overlayPosition": { "anchorCell": { "sheetId": sheetId, - "rowIndex": GRAPH_ROW_INDEX, + "rowIndex": GRAPH_ROW_INDEX , "columnIndex": column_count + GRAPH_COL_INDEX, - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 } }, } diff --git a/quisby/benchmarks/pyperf/pyperf.py b/quisby/benchmarks/pyperf/pyperf.py index 78e49aa..843578a 100644 --- a/quisby/benchmarks/pyperf/pyperf.py +++ b/quisby/benchmarks/pyperf/pyperf.py @@ -1,9 +1,16 @@ +from itertools import groupby + from scipy.stats import gmean from quisby import custom_logger from quisby.util import read_config +from quisby.pricing.cloud_pricing import get_cloud_pricing import re +from quisby.util import process_instance, mk_int + +from quisby.sheet.sheet_util import append_to_sheet + def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) @@ -28,69 +35,109 @@ def custom_key(item): instance_number = int(item[0].split('-')[-1]) return instance_type, instance_number elif cloud_type == "azure": - instance_type, instance_number, version=extract_prefix_and_number(item[0]) - return (instance_type, instance_number) + instance_type, instance_number, version= extract_prefix_and_number(item[0]) + return instance_type, version, instance_number + + +def calc_price_performance(inst, avg): + region = read_config("cloud", "region") + cloud_type = read_config("cloud", "cloud_type") + os_type = read_config("test", "os_type") + cost_per_hour = None + try: + cost_per_hour = get_cloud_pricing( + inst, region, cloud_type.lower(), os_type) + price_perf = float(avg)/float(cost_per_hour) + except Exception as exc: + custom_logger.debug(str(exc)) + custom_logger.error("Error calculating value !") + return cost_per_hour, price_perf + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) def create_summary_pyperf_data(data, OS_RELEASE): - results = [] - processed_data = None - gmean_data = [] - SYSTEM_GEOMEAN = [] - end_index = 0 - start_index = 0 - system = "" - # Add summary data - for index, row in enumerate(data): - try: - if row == [""]: - if processed_data: - results.append(processed_data) - SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - processed_data = [] - gmean_data = [] - system = "" - start_index = end_index + 1 - end_index = 0 - elif start_index: - system = row[0] - processed_data.append(system) - end_index = start_index + 1 - start_index = 0 - elif end_index: - gmean_data.append(float(row[1])) - processed_data.append(row[0] + " :" + row[1]) - if float(row[1] == 0.0): - custom_logger.warning("Value for test: " + row[0] + " is 0.0 for machine " + system) - except Exception as exc: - custom_logger.error(str(exc)) - results.append(processed_data) - SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - results.append([""]) - results.append(["SYSTEM_NAME", "GEOMEAN-" + str(OS_RELEASE)]) - sorted_data = sorted(SYSTEM_GEOMEAN, key=custom_key) - for item in sorted_data: - results.append(item) - return results + ret_results = [] + + results = list(filter(None, data)) + sort_data(results) + results = group_data(results) + for _, items in results: + mac_data = [["System name", "Geomean-" + OS_RELEASE]] + cost_data = [["Cost/Hr"]] + price_perf_data = [["Price-perf",f"Geomean/$-{OS_RELEASE}"]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + cost_per_hour, price_per_perf = [], [] + # Add summary data + for index, row in enumerate(sorted_data): + inst = row[1][0] + gmean_data=[] + for i in range(2, len(row)): + try: + gmean_data.append(float(row[i][1].strip())) + except Exception as exc: + gmean_data.append(0.0) + gdata = gmean(gmean_data) + try: + cph, pp = calc_price_performance(inst, gdata) + except Exception as exc: + custom_logger.error(str(exc)) + continue + + mac_data.append([inst, gdata]) + cost_data.append([inst, cph]) + price_perf_data.append([inst, pp]) + ret_results.append([""]) + ret_results.extend(mac_data) + ret_results.append([""]) + ret_results.extend(cost_data) + ret_results.append([""]) + ret_results.extend(price_perf_data) + return ret_results def extract_pyperf_data(path, system_name, OS_RELEASE): """""" results = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file + summary_data = [] try: if path: with open(path) as file: pyperf_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://"+server+"/results/"+result_dir+"/"+sum_path]) else: return None except Exception as exc: custom_logger.error(str(exc)) return None - for index, data in enumerate(pyperf_results): pyperf_results[index] = data.strip("\n").split(":") results.append([""]) results.append([system_name]) - results.extend(pyperf_results[0:]) - return results + results.extend(pyperf_results[1:]) + return [results], summary_data \ No newline at end of file diff --git a/quisby/benchmarks/specjbb/comparison.py b/quisby/benchmarks/specjbb/comparison.py index d02cfd2..db847fc 100644 --- a/quisby/benchmarks/specjbb/comparison.py +++ b/quisby/benchmarks/specjbb/comparison.py @@ -16,8 +16,9 @@ def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) if match: prefix = match.group(1) - return prefix - return None + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None def compare_inst(item1, item2): diff --git a/quisby/benchmarks/specjbb/graph.py b/quisby/benchmarks/specjbb/graph.py index b17d0dd..3dc0e75 100644 --- a/quisby/benchmarks/specjbb/graph.py +++ b/quisby/benchmarks/specjbb/graph.py @@ -107,8 +107,17 @@ def graph_specjbb_data(spreadsheetId, range, action): for index, row in enumerate(data): try: - if "Peak" in row or "Peak/$eff" in row: + if "Peak" in row: start_index = index + title = "%s : %s" %(range,row[0]) + subtitle = "Throughput" + left_title = row[1].lower() + + if "Peak/$eff" in row: + start_index = index + title = "%s : %s" % (range, "Price-Performance") + subtitle = "%s" %(row[0]) + left_title = row[1].lower() if start_index: if not row: @@ -130,17 +139,36 @@ def graph_specjbb_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s" % (range, graph_data[0][0]), + "title": title, + "subtitle": subtitle+" : "+graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ - {"position": "BOTTOM_AXIS", "title": ""}, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + + "position": "BOTTOM_AXIS", + "title": "System"}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", - "title": "Throughput(bops)", + "title": left_title, }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -172,8 +200,12 @@ def graph_specjbb_data(spreadsheetId, range, action): "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, "columnIndex": column_count + GRAPH_COL_INDEX, - } - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 + }, + }, } } diff --git a/quisby/benchmarks/specjbb/specjbb.py b/quisby/benchmarks/specjbb/specjbb.py index 95f1bc2..d24bc6e 100644 --- a/quisby/benchmarks/specjbb/specjbb.py +++ b/quisby/benchmarks/specjbb/specjbb.py @@ -31,15 +31,26 @@ def custom_key(item): return instance_type, instance_number elif cloud_type == "azure": instance_type, instance_number, version= extract_prefix_and_number(item[1][0]) - return instance_type, instance_number + return instance_type,version, instance_number +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + def specjbb_sort_data_by_system_family(results): sorted_result = [] results.sort(key=lambda x: str(process_instance( x[1][0], "family", "version", "feature"))) + for _, items in groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")): sorted_result.append( sorted(list(items), key=lambda x: mk_int( @@ -61,21 +72,35 @@ def calc_peak_throughput_peak_efficiency(data): peak_efficiency = float(peak_throughput) / float(cost_per_hour) except Exception as exc: custom_logger.debug(str(exc)) - custom_logger.error("Error calculating value !") + custom_logger.error("Error calculating value for :" + data[1][0]) return peak_throughput, cost_per_hour, peak_efficiency + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + def create_summary_specjbb_data(specjbb_data, OS_RELEASE): """""" - results = [] - specjbb_data = specjbb_sort_data_by_system_family(specjbb_data) + results = [] + #specjbb_data = specjbb_sort_data_by_system_family(specjbb_data) + specjbb_data = list(filter(None, specjbb_data)) + sort_data(specjbb_data) + specjbb_data = group_data(specjbb_data) - for items in specjbb_data: + for _,items in specjbb_data: + items = list(items) peak_throughput, cost_per_hour, peak_efficiency = [], [], [] - sorted_data = sorted(items, key=custom_key) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) for item in sorted_data: - results += item + results.extend(item) try: pt, cph, pe = calc_peak_throughput_peak_efficiency(item) except Exception as exc: @@ -92,7 +117,7 @@ def create_summary_specjbb_data(specjbb_data, OS_RELEASE): results.append(["Cost/Hr"]) results += cost_per_hour results.append([""]) - results.append(["Peak/$eff", f"Price/perf-{OS_RELEASE}"]) + results.append(["Peak/$eff", f"Price-perf-{OS_RELEASE}"]) results += peak_efficiency return results @@ -101,17 +126,22 @@ def create_summary_specjbb_data(specjbb_data, OS_RELEASE): def extract_specjbb_data(path, system_name, OS_RELEASE): """""" results = [[""], [system_name]] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # File read try: if path.endswith(".csv"): with open(path) as csv_file: specjbb_results = list(csv.DictReader(csv_file, delimiter=":")) + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: return None except Exception as exc: custom_logger.error(str(exc)) - return None + return None, None results.append(["Warehouses", f"Thrput-{OS_RELEASE}"]) for data_dict in specjbb_results[1:]: @@ -120,4 +150,4 @@ def extract_specjbb_data(path, system_name, OS_RELEASE): else: results.append([data_dict["Warehouses"], data_dict["Bops"]]) - return results + return results, summary_data