From 391673a28fb25a8c9ae5bcf10e48eb50d3af40d7 Mon Sep 17 00:00:00 2001 From: Scott Mountenay Date: Mon, 10 Feb 2025 23:57:56 -0600 Subject: [PATCH 1/4] Add perf results report endpoint --- backend/ttnn_visualizer/csv_queries.py | 54 +++++++++++++++++++++++- backend/ttnn_visualizer/requirements.txt | 2 +- backend/ttnn_visualizer/views.py | 9 ++++ pyproject.toml | 3 +- 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/backend/ttnn_visualizer/csv_queries.py b/backend/ttnn_visualizer/csv_queries.py index 4b119575..3449dc97 100644 --- a/backend/ttnn_visualizer/csv_queries.py +++ b/backend/ttnn_visualizer/csv_queries.py @@ -1,12 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # # SPDX-FileCopyrightText: © 2024 Tenstorrent Inc. - +import csv import os +import tempfile +from io import StringIO from pathlib import Path from typing import List, Dict, Union, Optional import pandas as pd +from tt_perf_report import perf_report from ttnn_visualizer.models import TabSession from ttnn_visualizer.ssh_client import get_client @@ -449,6 +452,28 @@ class OpsPerformanceQueries: "CompileProgram_TT_HOST_FUNC [ns]", "HWCommandQueue_write_buffer_TT_HOST_FUNC [ns]", ] + REPORT_COLUMNS = [ + "id", + "total_percent", + "bound", + "op_code", + "device_time", + "op_to_op_gap", + "cores", + "dram", + "dram_percent", + "flops", + "flops_percent", + "math_fidelity", + "output_datatype", + "input_0_datatype", + "input_1_datatype", + "dram_sharded", + "input_0_Memory", + "inner_dim_block_size", + "output_subblock_h", + "output_subblock_w" + ] def __init__(self, session: TabSession): """ @@ -512,6 +537,33 @@ def get_raw_csv(session): path = OpsPerformanceQueries.get_remote_ops_perf_file_path(session) return read_remote_file(session.remote_connection, path) + @classmethod + def generate_report(cls, session): + raw_csv = OpsPerformanceQueries.get_raw_csv(session) + csv_file = StringIO(raw_csv) + signpost = None + ignore_signposts = None + min_percentage = 0.5 + id_range = None + csv_output_file = tempfile.mktemp(suffix=".csv") + no_advice = False + perf_report.generate_perf_report( + csv_file, signpost, ignore_signposts, min_percentage, id_range, csv_output_file, no_advice) + + report = [] + + with open(csv_output_file, newline="") as csvfile: + reader = csv.reader(csvfile, delimiter=",") + next(reader, None) + for row in reader: + report.append({ + column: row[index] for index, column in enumerate(cls.REPORT_COLUMNS) + }) + + os.unlink(csv_output_file) + + return report + def __exit__(self, exc_type, exc_val, exc_tb): """ Clean up resources when exiting the context. diff --git a/backend/ttnn_visualizer/requirements.txt b/backend/ttnn_visualizer/requirements.txt index c263df9e..29965c43 100644 --- a/backend/ttnn_visualizer/requirements.txt +++ b/backend/ttnn_visualizer/requirements.txt @@ -1,4 +1,3 @@ - gunicorn~=22.0.0 uvicorn==0.30.1 paramiko~=3.4.0 @@ -17,6 +16,7 @@ wheel build PyYAML==6.0.2 python-dotenv==1.0.1 +tt-perf-report==1.0.0 # Dev dependencies mypy diff --git a/backend/ttnn_visualizer/views.py b/backend/ttnn_visualizer/views.py index 9b4d69ee..0bf58bc2 100644 --- a/backend/ttnn_visualizer/views.py +++ b/backend/ttnn_visualizer/views.py @@ -387,6 +387,15 @@ def get_profiler_perf_results_data_raw(session: TabSession): ) +@api.route("/profiler/perf-results/report", methods=["GET"]) +@with_session +def get_profiler_perf_results_report(session: TabSession): + if not session.profiler_path: + return Response(status=HTTPStatus.NOT_FOUND) + report = OpsPerformanceQueries.generate_report(session) + return jsonify(report), 200 + + @api.route("/profiler/device-log/raw", methods=["GET"]) @with_session def get_profiler_data_raw(session: TabSession): diff --git a/pyproject.toml b/pyproject.toml index 98c47994..15dd126e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,8 @@ dependencies = [ "flask-socketio==5.4.1", "flask-sqlalchemy==3.1.1", "PyYAML==6.0.2", - "python-dotenv==1.0.1" + "python-dotenv==1.0.1", + "tt-perf-report==1.0.0" ] classifiers = [ From e8079771affe0fbfbcab295ad52952778a4a8c63 Mon Sep 17 00:00:00 2001 From: Scott Mountenay Date: Tue, 11 Feb 2025 11:47:05 -0600 Subject: [PATCH 2/4] Refactor report api into its own class --- backend/ttnn_visualizer/csv_queries.py | 109 ++++++++++++++----------- backend/ttnn_visualizer/views.py | 4 +- 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/backend/ttnn_visualizer/csv_queries.py b/backend/ttnn_visualizer/csv_queries.py index 3449dc97..c1890e24 100644 --- a/backend/ttnn_visualizer/csv_queries.py +++ b/backend/ttnn_visualizer/csv_queries.py @@ -452,28 +452,6 @@ class OpsPerformanceQueries: "CompileProgram_TT_HOST_FUNC [ns]", "HWCommandQueue_write_buffer_TT_HOST_FUNC [ns]", ] - REPORT_COLUMNS = [ - "id", - "total_percent", - "bound", - "op_code", - "device_time", - "op_to_op_gap", - "cores", - "dram", - "dram_percent", - "flops", - "flops_percent", - "math_fidelity", - "output_datatype", - "input_0_datatype", - "input_1_datatype", - "dram_sharded", - "input_0_Memory", - "inner_dim_block_size", - "output_subblock_h", - "output_subblock_w" - ] def __init__(self, session: TabSession): """ @@ -537,33 +515,6 @@ def get_raw_csv(session): path = OpsPerformanceQueries.get_remote_ops_perf_file_path(session) return read_remote_file(session.remote_connection, path) - @classmethod - def generate_report(cls, session): - raw_csv = OpsPerformanceQueries.get_raw_csv(session) - csv_file = StringIO(raw_csv) - signpost = None - ignore_signposts = None - min_percentage = 0.5 - id_range = None - csv_output_file = tempfile.mktemp(suffix=".csv") - no_advice = False - perf_report.generate_perf_report( - csv_file, signpost, ignore_signposts, min_percentage, id_range, csv_output_file, no_advice) - - report = [] - - with open(csv_output_file, newline="") as csvfile: - reader = csv.reader(csvfile, delimiter=",") - next(reader, None) - for row in reader: - report.append({ - column: row[index] for index, column in enumerate(cls.REPORT_COLUMNS) - }) - - os.unlink(csv_output_file) - - return report - def __exit__(self, exc_type, exc_val, exc_tb): """ Clean up resources when exiting the context. @@ -590,3 +541,63 @@ def get_all_entries( return self.runner.execute_query( columns=self.PERF_RESULTS_COLUMNS, as_dict=as_dict, limit=limit ) + + +class OpsPerformanceReportQueries: + REPORT_COLUMNS = [ + "id", + "total_percent", + "bound", + "op_code", + "device_time", + "op_to_op_gap", + "cores", + "dram", + "dram_percent", + "flops", + "flops_percent", + "math_fidelity", + "output_datatype", + "input_0_datatype", + "input_1_datatype", + "dram_sharded", + "input_0_Memory", + "inner_dim_block_size", + "output_subblock_h", + "output_subblock_w" + ] + + DEFAULT_SIGNPOST = None + DEFAULT_IGNORE_SIGNPOSTS = None + DEFAULT_MIN_PERCENTAGE = 0.5 + DEFAULT_ID_RANGE = None + DEFAULT_NO_ADVICE = False + + @classmethod + def generate_report(cls, session): + raw_csv = OpsPerformanceQueries.get_raw_csv(session) + csv_file = StringIO(raw_csv) + csv_output_file = tempfile.mktemp(suffix=".csv") + perf_report.generate_perf_report( + csv_file, + cls.DEFAULT_SIGNPOST, + cls.DEFAULT_IGNORE_SIGNPOSTS, + cls.DEFAULT_MIN_PERCENTAGE, + cls.DEFAULT_ID_RANGE, + csv_output_file, + cls.DEFAULT_NO_ADVICE, + ) + + report = [] + + with open(csv_output_file, newline="") as csvfile: + reader = csv.reader(csvfile, delimiter=",") + next(reader, None) + for row in reader: + report.append({ + column: row[index] for index, column in enumerate(cls.REPORT_COLUMNS) + }) + + os.unlink(csv_output_file) + + return report diff --git a/backend/ttnn_visualizer/views.py b/backend/ttnn_visualizer/views.py index 0bf58bc2..ac25c665 100644 --- a/backend/ttnn_visualizer/views.py +++ b/backend/ttnn_visualizer/views.py @@ -14,7 +14,7 @@ from flask import Blueprint, Response, jsonify from flask import request, current_app -from ttnn_visualizer.csv_queries import DeviceLogProfilerQueries, OpsPerformanceQueries +from ttnn_visualizer.csv_queries import DeviceLogProfilerQueries, OpsPerformanceQueries, OpsPerformanceReportQueries from ttnn_visualizer.decorators import with_session from ttnn_visualizer.enums import ConnectionTestStates from ttnn_visualizer.exceptions import RemoteConnectionException @@ -392,7 +392,7 @@ def get_profiler_perf_results_data_raw(session: TabSession): def get_profiler_perf_results_report(session: TabSession): if not session.profiler_path: return Response(status=HTTPStatus.NOT_FOUND) - report = OpsPerformanceQueries.generate_report(session) + report = OpsPerformanceReportQueries.generate_report(session) return jsonify(report), 200 From c982a22a895cdf16d421ebac282313b26e9b55c6 Mon Sep 17 00:00:00 2001 From: Scott Mountenay Date: Tue, 11 Feb 2025 13:17:30 -0600 Subject: [PATCH 3/4] Add exception handling around invalid report csv files --- backend/ttnn_visualizer/csv_queries.py | 22 +++++++++++++--------- backend/ttnn_visualizer/exceptions.py | 4 ++++ backend/ttnn_visualizer/views.py | 8 +++++++- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/ttnn_visualizer/csv_queries.py b/backend/ttnn_visualizer/csv_queries.py index c1890e24..90a8069e 100644 --- a/backend/ttnn_visualizer/csv_queries.py +++ b/backend/ttnn_visualizer/csv_queries.py @@ -11,6 +11,7 @@ import pandas as pd from tt_perf_report import perf_report +from ttnn_visualizer.exceptions import DataFormatError from ttnn_visualizer.models import TabSession from ttnn_visualizer.ssh_client import get_client @@ -590,14 +591,17 @@ def generate_report(cls, session): report = [] - with open(csv_output_file, newline="") as csvfile: - reader = csv.reader(csvfile, delimiter=",") - next(reader, None) - for row in reader: - report.append({ - column: row[index] for index, column in enumerate(cls.REPORT_COLUMNS) - }) - - os.unlink(csv_output_file) + try: + with open(csv_output_file, newline="") as csvfile: + reader = csv.reader(csvfile, delimiter=",") + next(reader, None) + for row in reader: + report.append({ + column: row[index] for index, column in enumerate(cls.REPORT_COLUMNS) + }) + except csv.Error as e: + raise DataFormatError() from e + finally: + os.unlink(csv_output_file) return report diff --git a/backend/ttnn_visualizer/exceptions.py b/backend/ttnn_visualizer/exceptions.py index 03d53b74..066cd2bc 100644 --- a/backend/ttnn_visualizer/exceptions.py +++ b/backend/ttnn_visualizer/exceptions.py @@ -34,3 +34,7 @@ def __init__(self, message, status): class DatabaseFileNotFoundException(Exception): pass + + +class DataFormatError(Exception): + pass diff --git a/backend/ttnn_visualizer/views.py b/backend/ttnn_visualizer/views.py index ac25c665..39365b2e 100644 --- a/backend/ttnn_visualizer/views.py +++ b/backend/ttnn_visualizer/views.py @@ -16,6 +16,7 @@ from ttnn_visualizer.csv_queries import DeviceLogProfilerQueries, OpsPerformanceQueries, OpsPerformanceReportQueries from ttnn_visualizer.decorators import with_session +from ttnn_visualizer.exceptions import DataFormatError from ttnn_visualizer.enums import ConnectionTestStates from ttnn_visualizer.exceptions import RemoteConnectionException from ttnn_visualizer.file_uploads import ( @@ -392,7 +393,12 @@ def get_profiler_perf_results_data_raw(session: TabSession): def get_profiler_perf_results_report(session: TabSession): if not session.profiler_path: return Response(status=HTTPStatus.NOT_FOUND) - report = OpsPerformanceReportQueries.generate_report(session) + + try: + report = OpsPerformanceReportQueries.generate_report(session) + except DataFormatError: + return Response(status=HTTPStatus.UNPROCESSABLE_ENTITY) + return jsonify(report), 200 From 0584e2779ed624cccc09174e83b094e36958cfa8 Mon Sep 17 00:00:00 2001 From: Scott Mountenay Date: Tue, 11 Feb 2025 13:29:53 -0600 Subject: [PATCH 4/4] Fix capitalization of input_0_memory field --- backend/ttnn_visualizer/csv_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ttnn_visualizer/csv_queries.py b/backend/ttnn_visualizer/csv_queries.py index 90a8069e..542b593d 100644 --- a/backend/ttnn_visualizer/csv_queries.py +++ b/backend/ttnn_visualizer/csv_queries.py @@ -562,7 +562,7 @@ class OpsPerformanceReportQueries: "input_0_datatype", "input_1_datatype", "dram_sharded", - "input_0_Memory", + "input_0_memory", "inner_dim_block_size", "output_subblock_h", "output_subblock_w"