New experiment on PUI suppression, worked v1.

src/suppression_study/evolution/AccidentalSuppressionFinder.py

@@ -7,8 +7,10 @@
 import ast
 import os
 from os.path import join, exists
+import subprocess
 from typing import List
 from suppression_study.evolution.ExtractHistory import read_histories_from_json
+from suppression_study.evolution.MapWarningLines import MapWarningLines
 from suppression_study.utils.FunctionsCommon import get_commit_list
 from suppression_study.warnings.WarningSuppressionMapper import main as compute_warning_suppression_mapping
 from suppression_study.warnings.WarningSuppressionUtil import read_mapping_from_csv
@@ -111,6 +113,7 @@ def get_suppression_warning_pairs(repo_dir, commit, relevant_files, results_dir,
 def check_for_accidental_suppressions(repo_dir, history, relevant_commits, relevant_files, results_dir, is_file_specific):
     accidentally_suppressed_warnings = []
     previous_commit = None
+    previous_suppression = None
     warnings_suppressed_at_previous_commit = None
 
     add_event = history[0]
@@ -158,24 +161,54 @@ def check_for_accidental_suppressions(repo_dir, history, relevant_commits, relev
                 # if a new warning shows up that wasn't suppressed by this suppression
                 # at the previous commit, create an AccidentallySuppressedWarning
                 if warnings_suppressed_at_previous_commit is not None:
+                    to_append = False
                     if suppression is not None:
-                        if len(warnings_suppressed_at_commit) > len(warnings_suppressed_at_previous_commit):
+                        refined_warnings = []
+                        num_new_warning = 0
+                        current_len = len(warnings_suppressed_at_commit)
+                        previous_len = len(warnings_suppressed_at_previous_commit)
+                        if previous_len > current_len:
+                            pass
+                        elif previous_len == current_len and warnings_suppressed_at_previous_commit != warnings_suppressed_at_commit:
+                            backup_warnings_suppressed_at_previous_commit = [w for w in warnings_suppressed_at_previous_commit]
+                            backup_warnings_suppressed_at_commit = [w for w in warnings_suppressed_at_commit]
+                            # the warnings may different, run diff to check the line number maps
+                            refined_warnings, num_new_warning = MapWarningLines(repo_dir, previous_commit, commit, \
+                                    backup_warnings_suppressed_at_previous_commit, backup_warnings_suppressed_at_commit ).check_warning_mapping()
+                            if refined_warnings == None:
+                                pass # ignore the cases
+                            else:
+                                if num_new_warning > 0:
+                                    to_append == True
+                        elif previous_len < current_len:
+                            to_append = True
+                            num_new_warning = current_len - previous_len
+
+                        if to_append == True:
                             # there's a new warning suppressed by this suppression
-                            # TODO Check if the suppression is useless.
+                            summary = f"from {previous_len} to {current_len}, new warnings: {num_new_warning}"
+                            warnings_suppressed_at_commit_v = warnings_suppressed_at_commit
+                            if refined_warnings:
+                                warnings_suppressed_at_commit_v = refined_warnings
                             accidentally_suppressed_warnings.append(
-                                AccidentallySuppressedWarning(previous_commit,
+                                AccidentallySuppressedWarning(summary,
+                                                            previous_commit,
                                                             commit,
+                                                            previous_suppression,
                                                             suppression,              
                                                             warnings_suppressed_at_previous_commit,
-                                                            warnings_suppressed_at_commit))
+                                                            warnings_suppressed_at_commit_v))
                 previous_commit = commit
+                previous_suppression = suppression
                 warnings_suppressed_at_previous_commit = warnings_suppressed_at_commit
             else:
                 warnings_suppressed_at_previous_commit = None
                 previous_commit = None
+                previous_suppression = None
         else:
             warnings_suppressed_at_previous_commit = None
             previous_commit = None
+            previous_suppression = None
 
     return accidentally_suppressed_warnings
 

src/suppression_study/evolution/AccidentallySuppressedWarning.py

@@ -2,9 +2,11 @@
 
 
 class AccidentallySuppressedWarning:
-    def __init__(self, previous_commit, commit, suppression, previous_warnings, warnings):
+    def __init__(self, summary, previous_commit, commit, previous_suppression, suppression, previous_warnings, warnings):
+        self.summary = summary
         self.previous_commit = previous_commit
         self.commit = commit
+        self.previous_suppression = previous_suppression
         self.suppression = suppression
         self.previous_warnings = previous_warnings
         self.warnings = warnings
@@ -15,6 +17,8 @@ def __lt__(self, other):
             return self.previous_commit < other.previous_commit
         elif self.commit != other.commit:
             return self.commit < other.commit
+        elif self.previous_suppression != other.previous_suppression:
+            return self.previous_suppression < other.previous_suppression
         elif self.suppression != other.suppression:
             return self.suppression < other.suppression
         elif self.previous_warnings != other.previous_warnings:
@@ -24,8 +28,14 @@ def __lt__(self, other):
 
     def to_dict(self):
         d = {
+            "summary": self.summary,
             "previous_commit": self.previous_commit,
             "commit": self.commit,
+            "previous_suppression": {
+                "path": self.previous_suppression.path,
+                "text": self.previous_suppression.text,
+                "line": self.previous_suppression.line
+            },
             "suppression": {
                 "path": self.suppression.path,
                 "text": self.suppression.text,

src/suppression_study/evolution/MapWarningLines.py

@@ -0,0 +1,113 @@
+import subprocess
+from suppression_study.warnings.Warning import Warning
+
+
+class MapWarningLines():
+    def __init__(self, repo_dir, pre_c, c, pre_warnings, warnings):
+        self.repo_dir = repo_dir
+        self.pre_c = pre_c
+        self.c = c
+        self.pre_warnings = pre_warnings
+        self.warnings = warnings
+
+        self.refined_warnings = []
+        self.mapped_warning_lines = []
+
+    def check_warning_mapping(self):
+        pre_file = self.pre_warnings[0].path
+        file = self.warnings[0].path
+
+        for w in self.pre_warnings:
+            self.mapped_warning_lines.append(w.line)
+
+        commit_diff_command = f"git diff --word-diff --unified=0 {self.pre_c}:{pre_file} {self.c}:{file}"
+        diff_result = subprocess.run(commit_diff_command, cwd=self.repo_dir, shell=True,
+            stdout=subprocess.PIPE, universal_newlines=True)
+        diff_contents = diff_result.stdout
+        diffs = diff_contents.split("\n")
+
+        for diff_line in diffs:
+            diff_line = diff_line.strip()
+            if diff_line.startswith("@@"):
+                tmp = diff_line.split(" ")
+                base_hunk_range, base_step = get_diff_reported_range(tmp[1])
+                for i, w, warning_line in zip(range(len(self.pre_warnings)), self.pre_warnings, self.mapped_warning_lines):
+                    if warning_line < base_hunk_range.start:
+                        self.map_helper(w, warning_line)
+                    else:
+                        target_hunk_range, target_step = get_diff_reported_range(tmp[2], False)
+                        if warning_line in list(base_hunk_range):
+                            ref_delta = warning_line - base_hunk_range.start
+                            may_mapped_line = target_hunk_range.start + ref_delta
+                            self.map_helper(w, may_mapped_line)
+                        else:
+                            move_steps = target_step - base_step
+                            self.mapped_warning_lines[i] += move_steps
+
+                if not self.mapped_warning_lines:
+                    break
+
+        if self.mapped_warning_lines:
+            # map the warning after the last changed hunk
+            for w, warning_line in zip(self.pre_warnings, self.mapped_warning_lines):
+                self.map_helper(w, warning_line)
+            if self.mapped_warning_lines:
+                print(f"Inaccurate mapping happens. Current commit: {self.c}, file path: {file}")
+                return None, False
+
+        self.refined_warnings.append(None) # A symbol to start adding new warnings 
+        is_new_warning = []
+        for w in self.warnings:
+            if w not in self.refined_warnings:
+                self.refined_warnings.append(w)   
+                is_new_warning.append(True)
+        # print(f"{len(is_new_warning)}: is_new_warning")
+        return self.refined_warnings, len(is_new_warning) # the number of new warnings
+
+    def map_helper(self, warning, warning_line):
+        updated_w = Warning(warning.path, warning.kind, warning_line)
+        for w in self.warnings:
+            if updated_w == w: # old warning
+                self.refined_warnings.append(updated_w)
+                self.warnings.remove(updated_w)
+                break
+            self.refined_warnings.append(None) # mean the self.warnings fixed/disappeared
+        # do not check the mapped line for it anymore
+        try:
+            self.mapped_warning_lines.remove(warning_line)
+        except:
+            print(f"{warning_line} and {w.line}")
+        self.pre_warnings.remove(warning)
+
+
+def get_diff_reported_range(meta_range, base=True):
+    '''
+    Get range from diff results:
+    Input: 23,4  or  23
+    Return: 
+        * a range [ ) : end is not covered
+        * step, 4 and 0 in the input examples, respectively
+        * start+step, that is the line number of last line in base hunk.
+    '''
+
+    start = None
+    step = None
+    end = None
+    reported_range = None
+
+    sep = "+"
+    if base == True:
+        sep = "-"
+
+    if "," in meta_range:
+        tmp = meta_range.lstrip(sep).split(",")
+        start = int(tmp[0])
+        step = int(tmp[1]) 
+    else:
+        start = int(meta_range.lstrip(sep))
+        step = 1
+    end = start + step
+
+    reported_range = range(start, end) # [x, y)
+
+    return reported_range, step

src/suppression_study/experiments/InspectSuppressionRelatedCommits.py

@@ -54,7 +54,7 @@ def _prepare_inspection_file(self, repo_dir_to_commit_urls, name):
         my_random = random.Random(42)
         my_random.shuffle(all_commit_urls)
 
-        inspection_list = [{"url:": u, "comment": ""} for u in all_commit_urls]
+        inspection_list = [{"url": u, "comment": ""} for u in all_commit_urls]
 
         target_file = join("data", "results",
                            f"inspection_{name}_commits.json")