From fdc8caa774632dff9e7df0ea8b917ba37f396a89 Mon Sep 17 00:00:00 2001 From: Murilo Geraldini Date: Wed, 5 Mar 2025 13:22:01 -0300 Subject: [PATCH] fix: update tree details query to schema v5 - Removed "waived" field that's not being used and will be removed in the next version of the schema - Removed other unused fields (e.g. "builds.checkout_id") - Updated the `get_current_row_data` function - Moved `get_tree_details_data` to the queries directory Part of #1022 --- backend/kernelCI_app/helpers/treeDetails.py | 182 ++++-------------- backend/kernelCI_app/models.py | 1 - backend/kernelCI_app/queries/tree.py | 108 +++++++++++ .../views/treeDetailsBootsView.py | 2 +- .../views/treeDetailsBuildsView.py | 2 +- .../views/treeDetailsSummaryView.py | 2 +- .../views/treeDetailsTestsView.py | 2 +- backend/kernelCI_app/views/treeDetailsView.py | 2 +- 8 files changed, 149 insertions(+), 152 deletions(-) create mode 100644 backend/kernelCI_app/queries/tree.py diff --git a/backend/kernelCI_app/helpers/treeDetails.py b/backend/kernelCI_app/helpers/treeDetails.py index 39f169ba1..a220e02a8 100644 --- a/backend/kernelCI_app/helpers/treeDetails.py +++ b/backend/kernelCI_app/helpers/treeDetails.py @@ -19,9 +19,7 @@ build_misc_value_or_default, env_misc_value_or_default, ) -from kernelCI_app.cache import get_query_cache, set_query_cache from kernelCI_app.utils import is_boot -from django.db import connection class CheckoutWhereClauses(TypedDict): @@ -44,153 +42,45 @@ def create_checkouts_where_clauses( return {"git_url_clause": git_url_clause, "git_branch_clause": git_branch_clause} -def get_tree_details_data(request, commit_hash): - cache_key = "treeDetails" - - origin_param = request.GET.get("origin") - git_url_param = request.GET.get("git_url") - git_branch_param = request.GET.get("git_branch") - - params = { - "commit_hash": commit_hash, - "origin_param": origin_param, - "git_url_param": git_url_param, - "git_branch_param": git_branch_param, - } - - rows = get_query_cache(cache_key, params) - if rows is None: - checkout_clauses = create_checkouts_where_clauses( - git_url=git_url_param, git_branch=git_branch_param - ) - - git_url_clause = checkout_clauses.get("git_url_clause") - git_branch_clause = checkout_clauses.get("git_branch_clause") - - query = f""" - SELECT - tests.build_id AS tests_build_id, - tests.id AS tests_id, - tests.origin AS tests_origin, - tests.environment_comment AS tests_environment_comment, - tests.environment_misc AS tests_environment_misc, - tests.path AS tests_path, - tests.comment AS tests_comment, - tests.log_url AS tests_log_url, - tests.status AS tests_status, - tests.waived AS tests_waived, - tests.start_time AS tests_start_time, - tests.duration AS tests_duration, - tests.number_value AS tests_number_value, - tests.misc AS tests_misc, - tests.environment_compatible AS tests_environment_compatible, - builds_filter.*, - incidents.id AS incidents_id, - incidents.test_id AS incidents_test_id, - incidents.present AS incidents_present, - issues.id AS issues_id, - issues.version AS issues_version, - issues.comment AS issues_comment, - issues.report_url AS issues_report_url - FROM - ( - SELECT - builds.checkout_id AS builds_checkout_id, - builds.id AS builds_id, - builds.comment AS builds_comment, - builds.start_time AS builds_start_time, - builds.duration AS builds_duration, - builds.architecture AS builds_architecture, - builds.command AS builds_command, - builds.compiler AS builds_compiler, - builds.config_name AS builds_config_name, - builds.config_url AS builds_config_url, - builds.log_url AS builds_log_url, - builds.valid AS builds_valid, - builds.misc AS builds_misc, - tree_head.* - FROM - ( - SELECT - checkouts.id AS checkout_id, - checkouts.git_repository_url AS checkouts_git_repository_url, - checkouts.git_repository_branch AS checkouts_git_repository_branch, - checkouts.git_commit_tags - FROM - checkouts - WHERE - checkouts.git_commit_hash = %(commit_hash)s AND - {git_url_clause} AND - {git_branch_clause} AND - checkouts.origin = %(origin_param)s - ) AS tree_head - INNER JOIN builds - ON tree_head.checkout_id = builds.checkout_id - WHERE - builds.origin = %(origin_param)s - ) AS builds_filter - LEFT JOIN tests - ON builds_filter.builds_id = tests.build_id - LEFT JOIN incidents - ON tests.id = incidents.test_id OR - builds_filter.builds_id = incidents.build_id - LEFT JOIN issues - ON incidents.issue_id = issues.id - AND incidents.issue_version = issues.version - WHERE - tests.origin = %(origin_param)s OR - tests.origin IS NULL - ORDER BY - issues."_timestamp" DESC - """ - with connection.cursor() as cursor: - cursor.execute(query, params) - rows = cursor.fetchall() - set_query_cache(cache_key, params, rows) - - return rows - - def get_current_row_data(current_row: dict) -> dict: - tmp_test_env_comp_key = 14 + tmp_test_env_comp_key = 12 current_row_data = { - "test_id": current_row[1], - "test_origin": current_row[2], - "test_environment_comment": current_row[3], - "test_environment_misc": current_row[4], - "test_path": current_row[5], - "test_comment": current_row[6], - "test_log_url": current_row[7], - "test_status": current_row[8], - "test_waived": current_row[9], - "test_start_time": current_row[10], - "test_duration": current_row[11], - "test_number_value": current_row[12], - "test_misc": current_row[13], + "test_id": current_row[0], + "test_origin": current_row[1], + "test_environment_comment": current_row[2], + "test_environment_misc": current_row[3], + "test_path": current_row[4], + "test_comment": current_row[5], + "test_log_url": current_row[6], + "test_status": current_row[7], + "test_start_time": current_row[8], + "test_duration": current_row[9], + "test_number_value": current_row[10], + "test_misc": current_row[11], "test_environment_compatible": current_row[tmp_test_env_comp_key], - "build_id": current_row[16], - "build_comment": current_row[17], - "build_start_time": current_row[18], - "build_duration": current_row[19], - "build_architecture": current_row[20], - "build_command": current_row[21], - "build_compiler": current_row[22], - "build_config_name": current_row[23], - "build_config_url": current_row[24], - "build_log_url": current_row[25], - "build_valid": current_row[26], - "build_misc": current_row[27], - "checkout_id": current_row[28], - "checkout_git_repository_url": current_row[29], - "checkout_git_repository_branch": current_row[30], - "checkout_git_commit_tags": current_row[31], - "incident_id": current_row[32], - "incident_test_id": current_row[33], - "incident_present": current_row[34], - "issue_id": current_row[35], - "issue_version": current_row[36], - "issue_comment": current_row[37], - "issue_report_url": current_row[38], + "build_id": current_row[13], + "build_comment": current_row[14], + "build_start_time": current_row[15], + "build_duration": current_row[16], + "build_architecture": current_row[17], + "build_command": current_row[18], + "build_compiler": current_row[19], + "build_config_name": current_row[20], + "build_config_url": current_row[21], + "build_log_url": current_row[22], + "build_valid": current_row[23], + "build_misc": current_row[24], + "checkout_id": current_row[25], + "checkout_git_repository_url": current_row[26], + "checkout_git_repository_branch": current_row[27], + "checkout_git_commit_tags": current_row[28], + "incident_id": current_row[29], + "incident_test_id": current_row[30], + "incident_present": current_row[31], + "issue_id": current_row[32], + "issue_version": current_row[33], + "issue_comment": current_row[34], + "issue_report_url": current_row[35], } environment_misc = handle_environment_misc( diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index 7e7998723..d6b301bea 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -95,7 +95,6 @@ class Tests(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) status = models.TextField(blank=True, null=True) # This field type is a guess. - waived = models.BooleanField(blank=True, null=True) start_time = models.DateTimeField(blank=True, null=True) duration = models.FloatField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py new file mode 100644 index 000000000..7b84b04c2 --- /dev/null +++ b/backend/kernelCI_app/queries/tree.py @@ -0,0 +1,108 @@ +from django.db import connection + +from kernelCI_app.cache import get_query_cache, set_query_cache +from kernelCI_app.helpers.treeDetails import create_checkouts_where_clauses + + +def get_tree_details_data(request, commit_hash: str) -> list[tuple]: + cache_key = "treeDetails" + + origin_param = request.GET.get("origin") + git_url_param = request.GET.get("git_url") + git_branch_param = request.GET.get("git_branch") + + params = { + "commit_hash": commit_hash, + "origin_param": origin_param, + "git_url_param": git_url_param, + "git_branch_param": git_branch_param, + } + + rows = get_query_cache(cache_key, params) + if rows is None: + checkout_clauses = create_checkouts_where_clauses( + git_url=git_url_param, git_branch=git_branch_param + ) + + git_url_clause = checkout_clauses.get("git_url_clause") + git_branch_clause = checkout_clauses.get("git_branch_clause") + + query = f""" + SELECT + tests.id AS tests_id, + tests.origin AS tests_origin, + tests.environment_comment AS tests_environment_comment, + tests.environment_misc AS tests_environment_misc, + tests.path AS tests_path, + tests.comment AS tests_comment, + tests.log_url AS tests_log_url, + tests.status AS tests_status, + tests.start_time AS tests_start_time, + tests.duration AS tests_duration, + tests.number_value AS tests_number_value, + tests.misc AS tests_misc, + tests.environment_compatible AS tests_environment_compatible, + builds_filter.*, + incidents.id AS incidents_id, + incidents.test_id AS incidents_test_id, + incidents.present AS incidents_present, + issues.id AS issues_id, + issues.version AS issues_version, + issues.comment AS issues_comment, + issues.report_url AS issues_report_url + FROM + ( + SELECT + builds.id AS builds_id, + builds.comment AS builds_comment, + builds.start_time AS builds_start_time, + builds.duration AS builds_duration, + builds.architecture AS builds_architecture, + builds.command AS builds_command, + builds.compiler AS builds_compiler, + builds.config_name AS builds_config_name, + builds.config_url AS builds_config_url, + builds.log_url AS builds_log_url, + builds.valid AS builds_valid, + builds.misc AS builds_misc, + tree_head.* + FROM + ( + SELECT + checkouts.id AS checkout_id, + checkouts.git_repository_url AS checkouts_git_repository_url, + checkouts.git_repository_branch AS checkouts_git_repository_branch, + checkouts.git_commit_tags + FROM + checkouts + WHERE + checkouts.git_commit_hash = %(commit_hash)s AND + {git_url_clause} AND + {git_branch_clause} AND + checkouts.origin = %(origin_param)s + ) AS tree_head + INNER JOIN builds + ON tree_head.checkout_id = builds.checkout_id + WHERE + builds.origin = %(origin_param)s + ) AS builds_filter + LEFT JOIN tests + ON builds_filter.builds_id = tests.build_id + LEFT JOIN incidents + ON tests.id = incidents.test_id OR + builds_filter.builds_id = incidents.build_id + LEFT JOIN issues + ON incidents.issue_id = issues.id + AND incidents.issue_version = issues.version + WHERE + tests.origin = %(origin_param)s OR + tests.origin IS NULL + ORDER BY + issues."_timestamp" DESC + """ + with connection.cursor() as cursor: + cursor.execute(query, params) + rows = cursor.fetchall() + set_query_cache(cache_key, params, rows) + + return rows diff --git a/backend/kernelCI_app/views/treeDetailsBootsView.py b/backend/kernelCI_app/views/treeDetailsBootsView.py index 91db86d34..b086e3ad6 100644 --- a/backend/kernelCI_app/views/treeDetailsBootsView.py +++ b/backend/kernelCI_app/views/treeDetailsBootsView.py @@ -14,9 +14,9 @@ decide_if_is_boot_filtered_out, decide_if_is_full_row_filtered_out, get_current_row_data, - get_tree_details_data, is_test_boots_test, ) +from kernelCI_app.queries.tree import get_tree_details_data from kernelCI_app.typeModels.treeDetails import ( TreeQueryParameters, ) diff --git a/backend/kernelCI_app/views/treeDetailsBuildsView.py b/backend/kernelCI_app/views/treeDetailsBuildsView.py index d925b8386..5244e32aa 100644 --- a/backend/kernelCI_app/views/treeDetailsBuildsView.py +++ b/backend/kernelCI_app/views/treeDetailsBuildsView.py @@ -13,8 +13,8 @@ decide_if_is_full_row_filtered_out, get_build, get_current_row_data, - get_tree_details_data, ) +from kernelCI_app.queries.tree import get_tree_details_data from kernelCI_app.typeModels.treeDetails import ( TreeDetailsBuildsResponse, TreeQueryParameters, diff --git a/backend/kernelCI_app/views/treeDetailsSummaryView.py b/backend/kernelCI_app/views/treeDetailsSummaryView.py index 3b7858386..36b35e363 100644 --- a/backend/kernelCI_app/views/treeDetailsSummaryView.py +++ b/backend/kernelCI_app/views/treeDetailsSummaryView.py @@ -11,7 +11,6 @@ decide_if_is_test_filtered_out, get_build, get_current_row_data, - get_tree_details_data, process_boots_issue, process_tree_url, is_test_boots_test, @@ -21,6 +20,7 @@ process_tests_issue, process_filters, ) +from kernelCI_app.queries.tree import get_tree_details_data from kernelCI_app.typeModels.commonDetails import ( BuildSummary, DetailsFilters, diff --git a/backend/kernelCI_app/views/treeDetailsTestsView.py b/backend/kernelCI_app/views/treeDetailsTestsView.py index ad729e3db..a9ed94571 100644 --- a/backend/kernelCI_app/views/treeDetailsTestsView.py +++ b/backend/kernelCI_app/views/treeDetailsTestsView.py @@ -13,9 +13,9 @@ decide_if_is_full_row_filtered_out, decide_if_is_test_filtered_out, get_current_row_data, - get_tree_details_data, is_test_boots_test, ) +from kernelCI_app.queries.tree import get_tree_details_data from kernelCI_app.typeModels.treeDetails import ( TreeQueryParameters, ) diff --git a/backend/kernelCI_app/views/treeDetailsView.py b/backend/kernelCI_app/views/treeDetailsView.py index 3ffe966eb..39def35ff 100644 --- a/backend/kernelCI_app/views/treeDetailsView.py +++ b/backend/kernelCI_app/views/treeDetailsView.py @@ -17,7 +17,6 @@ decide_if_is_test_filtered_out, get_build, get_current_row_data, - get_tree_details_data, process_boots_issue, process_tree_url, is_test_boots_test, @@ -27,6 +26,7 @@ process_tests_issue, process_filters, ) +from kernelCI_app.queries.tree import get_tree_details_data from kernelCI_app.typeModels.commonDetails import ( BuildSummary, DetailsFilters,