From 5cef5acacd8001fd45576ad7ce77ff1730943a84 Mon Sep 17 00:00:00 2001 From: Olivier Cazade Date: Fri, 26 Jan 2024 09:46:02 +0000 Subject: [PATCH] NETOBSERV-1203 Add field to enable zone (#537) * Added zone field in CRD * Add FLP configuration to enable zone tagging * FLP dependency update * Fixing tests * Bundle update --- .../v1alpha1/flowcollector_webhook.go | 1 + .../v1alpha1/zz_generated.conversion.go | 1 + .../v1beta1/flowcollector_webhook.go | 1 + .../v1beta1/zz_generated.conversion.go | 1 + .../v1beta2/flowcollector_types.go | 5 ++++ .../v1beta2/zz_generated.deepcopy.go | 5 ++++ .../flows.netobserv.io_flowcollectors.yaml | 5 ++++ ...observ-operator.clusterserviceversion.yaml | 4 ++++ .../flows.netobserv.io_flowcollectors.yaml | 5 ++++ .../samples/flows_v1beta2_flowcollector.yaml | 1 + .../config/static-frontend-config.yaml | 24 +++++++++++++++++++ .../flowcollector_controller_iso_test.go | 1 + controllers/flp/flp_pipeline_builder.go | 11 +++++++++ docs/FlowCollector.md | 9 +++++++ go.mod | 2 +- go.sum | 2 ++ ...ned.flows.netobserv.io_flowcollectors.yaml | 4 ++++ hack/crd2csvSpecDesc.sh | 2 +- .../pkg/api/transform_network.go | 5 ++++ vendor/modules.txt | 2 +- 20 files changed, 88 insertions(+), 3 deletions(-) diff --git a/apis/flowcollector/v1alpha1/flowcollector_webhook.go b/apis/flowcollector/v1alpha1/flowcollector_webhook.go index 253d52ae4..924f4e9dc 100644 --- a/apis/flowcollector/v1alpha1/flowcollector_webhook.go +++ b/apis/flowcollector/v1alpha1/flowcollector_webhook.go @@ -68,6 +68,7 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error { if restored.Spec.Processor.ClusterName != "" { dst.Spec.Processor.ClusterName = restored.Spec.Processor.ClusterName } + dst.Spec.Processor.AddZone = restored.Spec.Processor.AddZone if restored.Spec.Processor.MultiClusterDeployment != nil { dst.Spec.Processor.MultiClusterDeployment = restored.Spec.Processor.MultiClusterDeployment } diff --git a/apis/flowcollector/v1alpha1/zz_generated.conversion.go b/apis/flowcollector/v1alpha1/zz_generated.conversion.go index 2ab96ff1a..899103749 100644 --- a/apis/flowcollector/v1alpha1/zz_generated.conversion.go +++ b/apis/flowcollector/v1alpha1/zz_generated.conversion.go @@ -649,6 +649,7 @@ func autoConvert_v1beta2_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in *v1bet // WARNING: in.LogTypes requires manual conversion: does not exist in peer-type // WARNING: in.ClusterName requires manual conversion: does not exist in peer-type // WARNING: in.MultiClusterDeployment requires manual conversion: does not exist in peer-type + // WARNING: in.AddZone requires manual conversion: does not exist in peer-type // WARNING: in.Advanced requires manual conversion: does not exist in peer-type return nil } diff --git a/apis/flowcollector/v1beta1/flowcollector_webhook.go b/apis/flowcollector/v1beta1/flowcollector_webhook.go index 585c08c02..e43bfd4e3 100644 --- a/apis/flowcollector/v1beta1/flowcollector_webhook.go +++ b/apis/flowcollector/v1beta1/flowcollector_webhook.go @@ -51,6 +51,7 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Loki.Monolithic = restored.Spec.Loki.Monolithic dst.Spec.Loki.Microservices = restored.Spec.Loki.Microservices dst.Spec.Loki.Manual = restored.Spec.Loki.Manual + dst.Spec.Processor.AddZone = restored.Spec.Processor.AddZone return nil } diff --git a/apis/flowcollector/v1beta1/zz_generated.conversion.go b/apis/flowcollector/v1beta1/zz_generated.conversion.go index 95e8a0ac1..33b6f851d 100644 --- a/apis/flowcollector/v1beta1/zz_generated.conversion.go +++ b/apis/flowcollector/v1beta1/zz_generated.conversion.go @@ -637,6 +637,7 @@ func autoConvert_v1beta2_FlowCollectorFLP_To_v1beta1_FlowCollectorFLP(in *v1beta out.LogTypes = (*string)(unsafe.Pointer(in.LogTypes)) out.ClusterName = in.ClusterName out.MultiClusterDeployment = (*bool)(unsafe.Pointer(in.MultiClusterDeployment)) + // WARNING: in.AddZone requires manual conversion: does not exist in peer-type // WARNING: in.Advanced requires manual conversion: does not exist in peer-type return nil } diff --git a/apis/flowcollector/v1beta2/flowcollector_types.go b/apis/flowcollector/v1beta2/flowcollector_types.go index 85f97e791..0b39b5909 100644 --- a/apis/flowcollector/v1beta2/flowcollector_types.go +++ b/apis/flowcollector/v1beta2/flowcollector_types.go @@ -445,6 +445,11 @@ type FlowCollectorFLP struct { // Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data MultiClusterDeployment *bool `json:"multiClusterDeployment,omitempty"` + //+kubebuilder:default:=false + //+optional + // `addZone` when set to `true`, the source and destination of flow will their zone added to the flow + AddZone *bool `json:"addZone,omitempty"` + // `advanced` allows setting some aspects of the internal configuration of the flow processor. // This section is aimed mostly for debugging and fine-grained performance optimizations, // such as `GOGC` and `GOMAXPROCS` env vars. Users setting its values do it at their own risk. diff --git a/apis/flowcollector/v1beta2/zz_generated.deepcopy.go b/apis/flowcollector/v1beta2/zz_generated.deepcopy.go index 59b1efb3b..d659b863a 100644 --- a/apis/flowcollector/v1beta2/zz_generated.deepcopy.go +++ b/apis/flowcollector/v1beta2/zz_generated.deepcopy.go @@ -467,6 +467,11 @@ func (in *FlowCollectorFLP) DeepCopyInto(out *FlowCollectorFLP) { *out = new(bool) **out = **in } + if in.AddZone != nil { + in, out := &in.AddZone, &out.AddZone + *out = new(bool) + **out = **in + } if in.Advanced != nil { in, out := &in.Advanced, &out.Advanced *out = new(AdvancedProcessorConfig) diff --git a/bundle/manifests/flows.netobserv.io_flowcollectors.yaml b/bundle/manifests/flows.netobserv.io_flowcollectors.yaml index 73bf5e1e8..0cb4278b5 100644 --- a/bundle/manifests/flows.netobserv.io_flowcollectors.yaml +++ b/bundle/manifests/flows.netobserv.io_flowcollectors.yaml @@ -7102,6 +7102,11 @@ spec: and forwards them to the Loki persistence layer and/or any available exporter.' properties: + addZone: + default: false + description: '`addZone` when set to `true`, the source and destination + of flow will their zone added to the flow' + type: boolean advanced: description: '`advanced` allows setting some aspects of the internal configuration of the flow processor. This section is aimed mostly diff --git a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml index 54945ebfe..242ff007f 100644 --- a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml +++ b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml @@ -736,8 +736,12 @@ spec: path: loki.monolithic.tenantID - displayName: Url path: loki.monolithic.url + - displayName: Read timeout + path: loki.readTimeout - displayName: Namespace path: namespace + - displayName: Add zone + path: processor.addZone - displayName: Log types path: processor.logTypes - displayName: Disable alerts diff --git a/config/crd/bases/flows.netobserv.io_flowcollectors.yaml b/config/crd/bases/flows.netobserv.io_flowcollectors.yaml index 1d56dbee8..24d892e97 100644 --- a/config/crd/bases/flows.netobserv.io_flowcollectors.yaml +++ b/config/crd/bases/flows.netobserv.io_flowcollectors.yaml @@ -7088,6 +7088,11 @@ spec: and forwards them to the Loki persistence layer and/or any available exporter.' properties: + addZone: + default: false + description: '`addZone` when set to `true`, the source and destination + of flow will their zone added to the flow' + type: boolean advanced: description: '`advanced` allows setting some aspects of the internal configuration of the flow processor. This section is aimed mostly diff --git a/config/samples/flows_v1beta2_flowcollector.yaml b/config/samples/flows_v1beta2_flowcollector.yaml index 085537e2e..22200224b 100644 --- a/config/samples/flows_v1beta2_flowcollector.yaml +++ b/config/samples/flows_v1beta2_flowcollector.yaml @@ -50,6 +50,7 @@ spec: logTypes: Flows # Append a unique cluster name to each record # clusterName: + # addZone: true metrics: server: port: 9102 diff --git a/controllers/consoleplugin/config/static-frontend-config.yaml b/controllers/consoleplugin/config/static-frontend-config.yaml index 741c246a5..9a35b5660 100644 --- a/controllers/consoleplugin/config/static-frontend-config.yaml +++ b/controllers/consoleplugin/config/static-frontend-config.yaml @@ -141,6 +141,13 @@ columns: calculated: getConcatenatedValue(SrcAddr,SrcPort) default: false width: 15 + - id: SrcZone + group: Source + name: Zone + field: SrcK8S_Zone + filter: src_zone + default: false + width: 15 - id: DstK8S_Name group: Destination name: Name @@ -253,6 +260,13 @@ columns: calculated: getConcatenatedValue(DstAddr,DstPort) default: false width: 15 + - id: DstZone + group: Destination + name: Zone + field: DstK8S_Zone + filter: dst_zone + default: false + width: 15 - id: K8S_Name name: Names calculated: getSrcOrDstValue(SrcK8S_Name,DstK8S_Name) @@ -556,6 +570,16 @@ filters: - Starting text like cluster, "cluster-*" - Ending text like "*-registry" - Pattern like "cluster-*-registry", "c*-*-r*y", -i*e- + - id: src_zone + name: Zone Name + component: text + category: source + hint: Specify a single zone. + - id: dst_zone + name: Zone Name + component: text + category: destination + hint: Specify a single zone. - id: src_resource name: Resource component: autocomplete diff --git a/controllers/flowcollector_controller_iso_test.go b/controllers/flowcollector_controller_iso_test.go index 553ae215e..598a5f84f 100644 --- a/controllers/flowcollector_controller_iso_test.go +++ b/controllers/flowcollector_controller_iso_test.go @@ -67,6 +67,7 @@ func flowCollectorIsoSpecs() { KafkaConsumerBatchSize: int(zero), MultiClusterDeployment: ptr.To(true), ClusterName: "testCluster", + AddZone: ptr.To(false), Advanced: &flowslatest.AdvancedProcessorConfig{ Port: ptr.To(int32(12345)), HealthPort: ptr.To(int32(12346)), diff --git a/controllers/flp/flp_pipeline_builder.go b/controllers/flp/flp_pipeline_builder.go index dce915aba..566029a39 100644 --- a/controllers/flp/flp_pipeline_builder.go +++ b/controllers/flp/flp_pipeline_builder.go @@ -52,16 +52,27 @@ func (b *PipelineBuilder) AddProcessorStages() error { lastStage = b.addTransformFilter(lastStage) lastStage = b.addConnectionTracking(lastStage) + addZone := false + if b.desired.Processor.AddZone != nil { + addZone = *b.desired.Processor.AddZone + } + // enrich stage (transform) configuration enrichedStage := lastStage.TransformNetwork("enrich", api.TransformNetwork{ Rules: api.NetworkTransformRules{{ Input: "SrcAddr", Output: "SrcK8S", Type: api.AddKubernetesRuleType, + Kubernetes: &api.K8sRule{ + AddZone: addZone, + }, }, { Input: "DstAddr", Output: "DstK8S", Type: api.AddKubernetesRuleType, + Kubernetes: &api.K8sRule{ + AddZone: addZone, + }, }, { Type: api.ReinterpretDirectionRuleType, }, { diff --git a/docs/FlowCollector.md b/docs/FlowCollector.md index b8d3b43aa..7b8af1afd 100644 --- a/docs/FlowCollector.md +++ b/docs/FlowCollector.md @@ -12657,6 +12657,15 @@ TLS client configuration for Loki URL. + addZone + boolean + + `addZone` when set to `true`, the source and destination of flow will their zone added to the flow
+
+ Default: false
+ + false + advanced object diff --git a/go.mod b/go.mod index 4886124f4..43ac000fa 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/go-logr/logr v1.4.1 github.com/mitchellh/mapstructure v1.5.0 - github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240123101451-40b694850494 + github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 github.com/onsi/ginkgo/v2 v2.15.0 github.com/onsi/gomega v1.30.0 github.com/openshift/api v0.0.0-20220112145620-704957ce4980 diff --git a/go.sum b/go.sum index f33de7c9a..256d1cd56 100644 --- a/go.sum +++ b/go.sum @@ -156,6 +156,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240123101451-40b694850494 h1:r0yWwNs/r0J6kRO4nnbGxUwITVdHAkWSXFr1+6gzIqs= github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240123101451-40b694850494/go.mod h1:Hfcqa5PFrELNp1B4M9e2+rG3226uOiKMi8KEOYBY0ow= +github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 h1:vMJ6tEETHl81Lbzmvs7z7I7PgvyC6WEzlJym2zSWMpQ= +github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934/go.mod h1:Hfcqa5PFrELNp1B4M9e2+rG3226uOiKMi8KEOYBY0ow= github.com/netobserv/prometheus-common v0.46.0-netobserv h1:gpnbXZtSWBauwlSm0aC//mAwAUIvblW6xqNV4mnY2mE= github.com/netobserv/prometheus-common v0.46.0-netobserv/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= diff --git a/hack/cloned.flows.netobserv.io_flowcollectors.yaml b/hack/cloned.flows.netobserv.io_flowcollectors.yaml index 5e74a1260..b5b0fa4ad 100644 --- a/hack/cloned.flows.netobserv.io_flowcollectors.yaml +++ b/hack/cloned.flows.netobserv.io_flowcollectors.yaml @@ -4933,6 +4933,10 @@ spec: processor: description: '`processor` defines the settings of the component that receives the flows from the agent, enriches them, generates metrics, and forwards them to the Loki persistence layer and/or any available exporter.' properties: + addZone: + default: false + description: '`addZone` when set to `true`, the source and destination of flow will their zone added to the flow' + type: boolean advanced: description: '`advanced` allows setting some aspects of the internal configuration of the flow processor. This section is aimed mostly for debugging and fine-grained performance optimizations, such as `GOGC` and `GOMAXPROCS` env vars. Users setting its values do it at their own risk.' properties: diff --git a/hack/crd2csvSpecDesc.sh b/hack/crd2csvSpecDesc.sh index c2a983436..dae94bf6e 100755 --- a/hack/crd2csvSpecDesc.sh +++ b/hack/crd2csvSpecDesc.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash version="$1" diff --git a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go index e27866de1..28167bbbd 100644 --- a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go +++ b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go @@ -69,6 +69,7 @@ type NetworkTransformRule struct { Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"` Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"` KubernetesInfra *K8sInfraRule `yaml:"kubernetes_infra,omitempty" json:"kubernetes_infra,omitempty" doc:"Kubernetes infra rule specific configuration"` + Kubernetes *K8sRule `yaml:"kubernetes,omitempty" json:"kubernetes,omitempty" doc:"Kubernetes rule specific configuration"` } type K8sInfraRule struct { @@ -77,6 +78,10 @@ type K8sInfraRule struct { InfraPrefix string `yaml:"infra_prefixes,omitempty" json:"infra_prefixes,omitempty" doc:"Namespace prefixes that will be tagged as infra"` } +type K8sRule struct { + AddZone bool `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"If true the rule will add the zone"` +} + type NetworkTransformDirectionInfo struct { ReporterIPField string `yaml:"reporterIPField,omitempty" json:"reporterIPField,omitempty" doc:"field providing the reporter (agent) host IP"` SrcHostField string `yaml:"srcHostField,omitempty" json:"srcHostField,omitempty" doc:"source host field"` diff --git a/vendor/modules.txt b/vendor/modules.txt index 06891e2df..0a26ae134 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -111,7 +111,7 @@ github.com/munnerz/goautoneg # github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f ## explicit github.com/mwitkow/go-conntrack -# github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240123101451-40b694850494 +# github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 ## explicit; go 1.20 github.com/netobserv/flowlogs-pipeline/pkg/api github.com/netobserv/flowlogs-pipeline/pkg/config