review plugin filter order

data/templates/plugins/openshift-conformance-replay.yaml

@@ -29,13 +29,13 @@ podSpec:
             --certificate-authority="${SA_CA_PATH}";
       env:
         - name: KUBECONFIG
-          value: "/tmp/shared/kubeconfig"
+          value: /tmp/shared/kubeconfig
         - name: KUBE_API_URL
           value: "https://172.30.0.1:443"
         - name: SA_TOKEN_PATH
-          value: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+          value: /var/run/secrets/kubernetes.io/serviceaccount/token
         - name: SA_CA_PATH
-          value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+          value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
       volumeMounts:
         - mountPath: /tmp/shared
           name: shared
@@ -51,11 +51,11 @@ podSpec:
         - name: KUBECONFIG
           value: /tmp/shared/kubeconfig
         - name: PLUGIN_NAME
-          value: "openshift-tests-replay"
+          value: openshift-tests-replay
         - name: DEFAULT_SUITE_NAME
-          value: "all"
+          value: all
         - name: OT_RUN_COMMAND
-          value: "run"
+          value: run
 
 sonobuoy-config:
   driver: Job
@@ -64,9 +64,9 @@ sonobuoy-config:
   description: |
     OPCT plugin to collect e2e failures from previous executions and
     schedule a new execution running in serial mode with openshift-tests.
-  source-url:
-    "https://github.com/redhat-openshift-ecosystem/provider-certification-tool/\
-    blob/main/manifests/openshift-conformance-validated.yaml"
+  source-url: |
+    https://github.com/redhat-openshift-ecosystem/provider-certification-tool/\
+    blob/main/manifests/openshift-conformance-validated.yaml
   skipCleanup: true
 spec:
   name: plugin
@@ -85,7 +85,7 @@ spec:
     - name: KUBECONFIG
       value: /tmp/shared/kubeconfig
     - name: PLUGIN_NAME
-      value: "openshift-tests-replay"
+      value: openshift-tests-replay
     - name: PLUGIN_ID
       value: "80"
     - name: ENV_NODE_NAME

docs/opct/report.md

@@ -1 +1,21 @@
-# opct report
+# opct report
+
+## Usage
+
+
+## Examples
+
+### Running the default conformance workflow
+
+
+
+### Development
+
+```sh
+VERSION=v0.0.0-devel-d4745f8
+opct-devel destroy; opct-devel run -w --devel-limit-tests=10 --log-level=debug \
+--plugins-image=quay.io/opct/plugin-openshift-tests:${VERSION} \
+--collector-image=quay.io/opct/plugin-artifacts-collector:${VERSION} \
+--must-gather-monitoring-image=quay.io/opct/must-gather-monitoring:${VERSION};
+opct-devel retrieve
+```

internal/opct/plugin/plugin.go

@@ -81,7 +81,7 @@ type OPCTPluginSummary struct {
 	// It should not be used to exclude failures from the report of e2e included in suite,
 	// but to remove known flake/failures that is not relevant to the pipeline.
 	// Example: '[sig-arch] External binary usage'
-	Filter5KnownFailures  []string
+	// Filter5KnownFailures  []string
 	FailedFilter5         []string
 	FailedExcludedFilter5 []string
 
@@ -113,4 +113,70 @@ func (ps *OPCTPluginSummary) GetErrorCounters() *archive.ErrorCounter {
 	return ps.calculateErrorCounter()
 }
 
-//
+const (
+	// FilterNameSuiteOnly is the filter to remove failures of tests not included in the suite.
+	FilterNameSuiteOnly = "suite-only"
+
+	// FilterID1 is the filter to exclude known failures from the OPCT CI.
+	FilterNameKF = "known-failures"
+
+	FilterNameBaseline = "baseline"
+
+	FilterNameFlaky = "flaky"
+
+	FilterNameReplay = "replay"
+
+	FilterNameFinalCopy = "copy"
+)
+
+// TODO: move to a chain stack to allow multiple filters.
+func (ps *OPCTPluginSummary) GetFailuresByFilterID(filterID string) ([]string, []string) {
+	switch filterID {
+	case FilterNameSuiteOnly:
+		return ps.FailedFilter1, ps.FailedExcludedFilter1
+	case FilterNameBaseline:
+		return ps.FailedFilter2, ps.FailedExcludedFilter2
+	case FilterNameKF:
+		return ps.FailedFilter5, ps.FailedExcludedFilter5
+	case FilterNameReplay:
+		return ps.FailedFilter6, ps.FailedExcludedFilter6
+	}
+	return nil, nil
+}
+
+func (ps *OPCTPluginSummary) SetFailuresByFilterID(filterID string, failures []string, excluded []string) {
+	switch filterID {
+	case FilterNameSuiteOnly:
+		ps.FailedFilter1 = failures
+		ps.FailedExcludedFilter1 = excluded
+		return
+	case FilterNameBaseline:
+		ps.FailedFilter2 = failures
+		ps.FailedExcludedFilter2 = excluded
+		return
+	case FilterNameKF:
+		ps.FailedFilter5 = failures
+		ps.FailedExcludedFilter5 = excluded
+		return
+	case FilterNameReplay:
+		ps.FailedFilter6 = failures
+		ps.FailedExcludedFilter6 = excluded
+		return
+	}
+}
+
+func (ps *OPCTPluginSummary) GetPreviousFailuresByFilterID(filterID string) []string {
+	switch filterID {
+	case FilterNameSuiteOnly:
+		return nil
+	case FilterNameKF:
+		return ps.FailedFilter1 // SuiteOnly
+	case FilterNameReplay:
+		return ps.FailedFilter5 // KnownFailures
+	case FilterNameBaseline:
+		return ps.FailedFilter6 // Replay
+	case FilterNameFinalCopy:
+		return ps.FailedFilter4 // BaselineAPI
+	}
+	return nil
+}