Skip to content

Commit

Permalink
add etcd checks ; cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@mtulio
mtulio committed Sep 25, 2023
1 parent 6bc5fc8 commit 413e5dd
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 49 deletions.
2 changes: 1 addition & 1 deletion data/templates/report/report.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,6 @@
},
getTotalCounter(errors=[]) {
for (let e in errors) {
console.log(e)
if (e == "total") {
return errors[e]
}
Expand Down Expand Up @@ -358,6 +357,7 @@
dataP.push(this.getPluginStat("10-openshift-kube-conformance")[0])
dataP.push(this.getPluginStat("20-openshift-conformance-validated")[0])
dataP.push(this.getPluginStat("99-openshift-artifacts-collector")[0])
dataP.push({"Plugin Name": "Execution Time", "Time": this.report.summary.runtime.executionTime})
const pluginHeaders = new Map(Object.entries(dataP[0]));
let tbPluginsRes = {
data: dataP,
Expand Down
16 changes: 0 additions & 16 deletions hack/verify-codegen.sh
View file

This file was deleted.

16 changes: 11 additions & 5 deletions internal/opct/archive/runtime.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
package archive

type RuntimeInfoItem struct {
Name string `json:"name"`
Value string `json:"value"`
// Name holds the name of the item/attribute.
Name string `json:"name"`
// Value holds the value of the item/attribute.
Value string `json:"value"`
// Config is a optional field containing extra config information from the item.
Config string `json:"config,omitempty"`
Time string `json:"time,omitempty"`
Total string `json:"total,omitempty"`
Delta string `json:"delta,omitempty"`
// Time is a optional field with the timestamp in the datetime format.
Time string `json:"time,omitempty"`
// Total is a optional field with the calculation of total time the item take.
Total string `json:"total,omitempty"`
// Detal is a optional field with the calculation of difference between two timers.
Delta string `json:"delta,omitempty"`
}
105 changes: 84 additions & 21 deletions internal/opct/report/checks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ package report
import (
"fmt"
"os"
"strconv"
"strings"

"github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin"
log "github.com/sirupsen/logrus"
Expand All @@ -31,8 +33,9 @@ type CheckSummary struct {
func NewCheckSummary(re *Report) *CheckSummary {

baseURL := defaultBaseURL
// mkdocs serve
// export OPCT_DEV_BASE_URL_DOC="http://127.0.0.1:8000/provider-certification-tool"
// Developer environment:
// $ mkdocs serve
// $ export OPCT_DEV_BASE_URL_DOC="http://127.0.0.1:8000/provider-certification-tool"
localDevBaseURL := os.Getenv("OPCT_DEV_BASE_URL_DOC")
if localDevBaseURL != "" {
baseURL = localDevBaseURL
Expand Down Expand Up @@ -164,7 +167,6 @@ func NewCheckSummary(re *Report) *CheckSummary {
if _, ok := re.Provider.Plugins[plugin.PluginNameKubernetesConformance]; !ok {
return CheckResultFail
}
fmt.Println(len(re.Provider.Plugins[plugin.PluginNameKubernetesConformance].TestsFailedPrio))
if len(re.Provider.Plugins[plugin.PluginNameKubernetesConformance].TestsFailedPrio) > 0 {
return CheckResultFail
}
Expand All @@ -177,8 +179,8 @@ func NewCheckSummary(re *Report) *CheckSummary {
// if _, ok := re.Provider.Plugins[PluginNameOpenShiftConformance]; !ok {
// return CheckResultFail
// }
// // "Acceptable" are relative, we are trying to observe the baselines and set
// // a "good" value considering a healthy cluster.
// // "Acceptance" are relative, the baselines is observed to set
// // an "accepted" value considering a healthy cluster in known provider/installation.
// plugin := re.Provider.Plugins[PluginNameOpenShiftConformance]
// if re.Provider.ClusterHealth.PodHealthTotal != 0 {
// return CheckResultFail
Expand All @@ -193,8 +195,8 @@ func NewCheckSummary(re *Report) *CheckSummary {
if _, ok := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance]; !ok {
return CheckResultFail
}
// "Acceptable" are relative, we are trying to observe the baselines and set
// a "good" value considering a healthy cluster.
// "Acceptance" are relative, the baselines is observed to set
// an "accepted" value considering a healthy cluster in known provider/installation.
plugin := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance]
perc := (float64(plugin.Stat.Failed) / float64(plugin.Stat.Total)) * 100
if perc > 1.5 {
Expand All @@ -210,8 +212,8 @@ func NewCheckSummary(re *Report) *CheckSummary {
if _, ok := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance]; !ok {
return CheckResultFail
}
// "Acceptable" are relative, we are trying to observe the baselines and set
// a "good" value considering a healthy cluster.
// "Acceptance" are relative, the baselines is observed to set
// an "accepted" value considering a healthy cluster in known provider/installation.
plugin := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance]
perc := (float64(plugin.Stat.FilterFailedPrio) / float64(plugin.Stat.Total)) * 100
if perc > 0.5 {
Expand All @@ -231,8 +233,8 @@ func NewCheckSummary(re *Report) *CheckSummary {
if _, ok := cnt["total"]; !ok {
return CheckResultFail
}
// "Acceptable" are relative, we are trying to observe the baselines and set
// a "good" value considering a healthy cluster.
// "Acceptance" are relative, the baselines is observed to set
// an "accepted" value considering a healthy cluster in known provider/installation.
total := cnt["total"]
if total > 150 {
return CheckResultFail
Expand All @@ -256,8 +258,8 @@ func NewCheckSummary(re *Report) *CheckSummary {
log.Debugf("Check Failed: OPCT-007: ErrorCounters[\"total\"]")
return CheckResultFail
}
// "Acceptable" are relative, we are trying to observe the baselines and set
// a "good" value considering a healthy cluster.
// "Acceptance" are relative, the baselines is observed to set
// an "accepted" value considering a healthy cluster in known provider/installation.
total := re.Provider.MustGatherInfo.ErrorCounters["total"]
if total > 30000 {
log.Debugf("Check Failed: OPCT-007: want[<=25000] got[%d]", total)
Expand Down Expand Up @@ -303,19 +305,80 @@ func NewCheckSummary(re *Report) *CheckSummary {
Name: "[TODO] etcd fio must accept the tests (TODO)",
Test: AcceptanceCheckFail,
})
checkSum.Checks = append(checkSum.Checks, &Check{
Name: "[TODO] etcd slow requests: average must be under 500ms",
Test: ExampleAcceptanceCheckPass,
})
checkSum.Checks = append(checkSum.Checks, &Check{
Name: "[TODO] etcd slow requests: p99 must be lower than 900ms",
Test: AcceptanceCheckFail,
})
checkSum.Checks = append(checkSum.Checks, &Check{
Name: "[TODO] etcd slow requests: must nto have requests taking more than 1s",
Test: ExampleAcceptanceCheckPass,
})
*/
checkSum.Checks = append(checkSum.Checks, &Check{
ID: "OPCT-010",
Name: "etcd logs: slow requests: average should be under 500ms",
Test: func() CheckResult {
prefix := "Check OPCT-010 Failed"
if re.Provider.MustGatherInfo == nil {
log.Debugf("%s: unable to read must-gather information.", prefix)
return CheckResultFail
}
if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"] == nil {
log.Debugf("%s: unable to read statistics from parsed etcd logs.", prefix)
return CheckResultFail
}
if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMean == "" {
log.Debugf("%s: unable to get p50/mean statistics from parsed data: %v", prefix, re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"])
return CheckResultFail
}
values := strings.Split(re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMean, " ")
if values[0] == "" {
log.Debugf("%s: unable to get parse p50/mean: %v", prefix, values)
return CheckResultFail
}
value, err := strconv.ParseFloat(values[0], 64)
if err != nil {
log.Debugf("%s: unable to convert p50/mean to float: %v", prefix, err)
return CheckResultFail
}
if value >= 500 {
log.Debugf("%s acceptance criteria: want=[%v] got=[%v]", prefix, "<500", value)
return CheckResultFail
}
return CheckResultPass
},
})
checkSum.Checks = append(checkSum.Checks, &Check{
ID: "OPCT-011",
Name: "etcd logs: slow requests: maximum should be under 1500ms",
Test: func() CheckResult {
prefix := "Check OPCT-011 Failed"
if re.Provider.MustGatherInfo == nil {
log.Debugf("%s: unable to read must-gather information.", prefix)
return CheckResultFail
}
if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"] == nil {
log.Debugf("%s: unable to read statistics from parsed etcd logs.", prefix)
return CheckResultFail
}
if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMax == "" {
log.Debugf("%s: unable to get max statistics from parsed data: %v", prefix, re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"])
return CheckResultFail
}
values := strings.Split(re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMax, " ")
if values[0] == "" {
log.Debugf("%s: unable to get parse max: %v", prefix, values)
return CheckResultFail
}
value, err := strconv.ParseFloat(values[0], 64)
if err != nil {
log.Debugf("%s: unable to convert max to float: %v", prefix, err)
return CheckResultFail
}
if value >= 1000 {
log.Debugf("%s acceptance criteria: want=[%v] got=[%v]", prefix, "<1000", value)
return CheckResultFail
}
return CheckResultPass
},
})
// TODO(network): podConnectivityChecks must not have outages

// Create docs reference when ID is set
for c := range checkSum.Checks {
Expand Down
8 changes: 6 additions & 2 deletions internal/opct/report/report.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,9 @@ type ReportSummary struct {
}

type ReportSummaryRuntime struct {
Timers metrics.Timers `json:"timers,omitempty"`
Plugins map[string]string `json:"plugins,omitempty"`
Timers metrics.Timers `json:"timers,omitempty"`
Plugins map[string]string `json:"plugins,omitempty"`
ExecutionTime string `json:"executionTime,omitempty"`
}

type ReportSummaryTests struct {
Expand Down Expand Up @@ -324,6 +325,9 @@ func (re *Report) populateSource(rs *summary.ResultSummary) error {
arr := strings.Split(e.Name, "plugin finished ")
re.Summary.Runtime.Plugins[arr[len(arr)-1]] = e.Delta
}
if strings.HasPrefix(e.Name, "server finished") {
re.Summary.Runtime.ExecutionTime = e.Total
}
}
}
if rs.Sonobuoy != nil && rs.Sonobuoy.MetaConfig != nil {
Expand Down
5 changes: 1 addition & 4 deletions internal/opct/summary/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,6 @@ func (rs *ResultSummary) Populate() error {
switch pluginName {
case plugin.PluginNameKubernetesConformance, plugin.PluginNameOpenShiftConformance:
rs.isConformance = true
// default:
// log.Debugf("Ignoring Plugin %s", plugin)
// continue
}
log.Debugf("Processing results/Populating/Processing Plugin/%s", pluginName)
err := rs.processPlugin(pluginName)
Expand All @@ -95,7 +92,7 @@ func (rs *ResultSummary) Populate() error {
}
}

// TODO: review the fd usage for tarbal and file
log.Info("Processing results...")
cleanup, err = rs.openReader()
defer cleanup()
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions internal/openshift/ci/sippy/sippy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ func (a *SippyAPI) QueryTests(in *SippyTestsRequestInput) (*SippyTestsRequestOut

}

if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, fmt.Errorf("invalid status code: %d", res.StatusCode)
}

sippyResponse := SippyTestsRequestOutput{}
if err := json.Unmarshal([]byte(body), &sippyResponse); err != nil {
return nil, fmt.Errorf("couldn't unmarshal response body: %+v \nBody: %s", string(body), err)
Expand Down

0 comments on commit 413e5dd

Please sign in to comment.