From 18cac7f941e8dd885fe1e44a59c8a8484fb88070 Mon Sep 17 00:00:00 2001 From: "bo.jiang" Date: Fri, 17 Jan 2025 18:02:56 +0800 Subject: [PATCH] add security group rules in cluster status Signed-off-by: bo.jiang --- api/v1alpha1/huaweicloudcluster_types.go | 1 + api/v1alpha1/network_types.go | 80 +++++++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 58 +++++++++++++ ....cluster.x-k8s.io_huaweicloudclusters.yaml | 85 +++++++++++++++++++ .../huaweicloudcluster_controller.go | 19 ++++- pkg/scope/cluster.go | 10 +++ pkg/services/network/network.go | 4 - pkg/services/network/subnet.go | 33 ++++--- pkg/services/network/vpc.go | 45 ++++------ pkg/services/securitygroup/securitygroups.go | 32 ++++++- pkg/services/securitygroup/service.go | 5 +- 11 files changed, 321 insertions(+), 51 deletions(-) diff --git a/api/v1alpha1/huaweicloudcluster_types.go b/api/v1alpha1/huaweicloudcluster_types.go index 1854cf5..985a4de 100644 --- a/api/v1alpha1/huaweicloudcluster_types.go +++ b/api/v1alpha1/huaweicloudcluster_types.go @@ -54,6 +54,7 @@ type HuaweiCloudClusterStatus struct { // +kubebuilder:default=false Ready bool `json:"ready"` + Network NetworkStatus `json:"networkStatus,omitempty"` Conditions clusterv1.Conditions `json:"conditions,omitempty"` } diff --git a/api/v1alpha1/network_types.go b/api/v1alpha1/network_types.go index 1760f73..b6feac0 100644 --- a/api/v1alpha1/network_types.go +++ b/api/v1alpha1/network_types.go @@ -16,6 +16,8 @@ limitations under the License. package v1alpha1 +import "fmt" + // NetworkSpect encapsulates the configuration options for HuaweiCloud network. type NetworkSpec struct { // VPC configuration. @@ -63,3 +65,81 @@ type SubnetSpec struct { // NeutronSubnetId is the identifier of the subnet (OpenStack Neutron interface). NeutronSubnetId string `json:"neutron_subnet_id"` } + +// SecurityGroupRole defines the unique role of a security group. +// +kubebuilder:validation:Enum=bastion;node;controlplane;apiserver-lb;lb;node-eks-additional +type SecurityGroupRole string + +var ( + // SecurityGroupNode defines a Kubernetes workload node role. + SecurityGroupNode = SecurityGroupRole("node") + + // SecurityGroupControlPlane defines a Kubernetes control plane node role. + SecurityGroupControlPlane = SecurityGroupRole("controlplane") + + // SecurityGroupAPIServerLB defines a Kubernetes API Server Load Balancer role. + SecurityGroupAPIServerLB = SecurityGroupRole("apiserver-lb") + + // SecurityGroupLB defines a container for the cloud provider to inject its load balancer ingress rules. + SecurityGroupLB = SecurityGroupRole("lb") +) + +// SecurityGroupRule +type SecurityGroupRule struct { + // ID is the unique identifier of the security group rule. + Id string `json:"id"` + + // Description is the description of the security group rule. + Description string `json:"description"` + + // SecurityGroupId is the security group id. + SecurityGroupId string `json:"security_group_id"` + + // Direction is the direction of the security group rule. Accepted values are "ingress" and "egress". + Direction string `json:"direction"` + + // Ethertype is the IP protocol type. The value can be IPv4 or IPv6. + Ethertype string `json:"ethertype"` + + // Protocol is the protocol for the security group rule. + Protocol string `json:"protocol"` + + // PortRangeMin is the start of port range. + PortRangeMin int32 `json:"port_range_min"` + + // PortRangeMax is the end of port range. + PortRangeMax int32 `json:"port_range_max"` + + // RemoteIpPrefix is the CIDR block to allow access from. + RemoteIpPrefix string `json:"remote_ip_prefix"` + + // RemoteGroupId is the remote security group id. + RemoteGroupId string `json:"remote_group_id"` + + // RemoteAddressGroupId is the remote address group id. + RemoteAddressGroupId string `json:"remote_address_group_id"` +} + +// SecurityGroup defines an HuaweiCloud security group. +type SecurityGroup struct { + // ID is a unique identifier. + ID string `json:"id"` + + // Name is the security group name. + Name string `json:"name"` + + // IngressRules is the inbound rules associated with the security group. + // +optional + SecurityGroupRules []SecurityGroupRule `json:"ingressRule,omitempty"` +} + +// String returns a string representation of the security group. +func (s *SecurityGroup) String() string { + return fmt.Sprintf("id=%s/name=%s", s.ID, s.Name) +} + +// NetworkStatus encapsulates HuaweiCloud networking resources. +type NetworkStatus struct { + // SecurityGroups is a map from the role/kind of the security group to its unique name, if any. + SecurityGroups map[SecurityGroupRole]SecurityGroup `json:"securityGroups,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 95bec87..ca0f57f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -104,6 +104,7 @@ func (in *HuaweiCloudClusterSpec) DeepCopy() *HuaweiCloudClusterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HuaweiCloudClusterStatus) DeepCopyInto(out *HuaweiCloudClusterStatus) { *out = *in + in.Network.DeepCopyInto(&out.Network) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(v1beta1.Conditions, len(*in)) @@ -352,6 +353,63 @@ func (in *NetworkSpec) DeepCopy() *NetworkSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkStatus) DeepCopyInto(out *NetworkStatus) { + *out = *in + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make(map[SecurityGroupRole]SecurityGroup, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkStatus. +func (in *NetworkStatus) DeepCopy() *NetworkStatus { + if in == nil { + return nil + } + out := new(NetworkStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { + *out = *in + if in.SecurityGroupRules != nil { + in, out := &in.SecurityGroupRules, &out.SecurityGroupRules + *out = make([]SecurityGroupRule, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroup. +func (in *SecurityGroup) DeepCopy() *SecurityGroup { + if in == nil { + return nil + } + out := new(SecurityGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroupRule) DeepCopyInto(out *SecurityGroupRule) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroupRule. +func (in *SecurityGroupRule) DeepCopy() *SecurityGroupRule { + if in == nil { + return nil + } + out := new(SecurityGroupRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SubnetSpec) DeepCopyInto(out *SubnetSpec) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_huaweicloudclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_huaweicloudclusters.yaml index b81beb6..41ce306 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_huaweicloudclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_huaweicloudclusters.yaml @@ -178,6 +178,91 @@ spec: - type type: object type: array + networkStatus: + description: NetworkStatus encapsulates HuaweiCloud networking resources. + properties: + securityGroups: + additionalProperties: + description: SecurityGroup defines an HuaweiCloud security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: SecurityGroupRule + properties: + description: + description: Description is the description of the + security group rule. + type: string + direction: + description: Direction is the direction of the security + group rule. Accepted values are "ingress" and "egress". + type: string + ethertype: + description: Ethertype is the IP protocol type. The + value can be IPv4 or IPv6. + type: string + id: + description: ID is the unique identifier of the security + group rule. + type: string + port_range_max: + description: PortRangeMax is the end of port range. + format: int32 + type: integer + port_range_min: + description: PortRangeMin is the start of port range. + format: int32 + type: integer + protocol: + description: Protocol is the protocol for the security + group rule. + type: string + remote_address_group_id: + description: RemoteAddressGroupId is the remote address + group id. + type: string + remote_group_id: + description: RemoteGroupId is the remote security + group id. + type: string + remote_ip_prefix: + description: RemoteIpPrefix is the CIDR block to allow + access from. + type: string + security_group_id: + description: SecurityGroupId is the security group + id. + type: string + required: + - description + - direction + - ethertype + - id + - port_range_max + - port_range_min + - protocol + - remote_address_group_id + - remote_group_id + - remote_ip_prefix + - security_group_id + type: object + type: array + name: + description: Name is the security group name. + type: string + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object ready: default: false type: boolean diff --git a/internal/controller/huaweicloudcluster_controller.go b/internal/controller/huaweicloudcluster_controller.go index 5c49f1e..cabc460 100644 --- a/internal/controller/huaweicloudcluster_controller.go +++ b/internal/controller/huaweicloudcluster_controller.go @@ -39,6 +39,13 @@ import ( "github.com/pkg/errors" ) +var defaultHCSecurityGroupRoles = []infrav1alpha1.SecurityGroupRole{ + infrav1alpha1.SecurityGroupAPIServerLB, + infrav1alpha1.SecurityGroupLB, + infrav1alpha1.SecurityGroupControlPlane, + infrav1alpha1.SecurityGroupNode, +} + // HuaweiCloudClusterReconciler reconciles a HuaweiCloudCluster object type HuaweiCloudClusterReconciler struct { client.Client @@ -46,6 +53,14 @@ type HuaweiCloudClusterReconciler struct { Credentials *basic.Credentials } +// securityGroupRolesForCluster returns the security group roles determined by the cluster configuration. +func securityGroupRolesForCluster() []infrav1alpha1.SecurityGroupRole { + // Copy to ensure we do not modify the package-level variable. + roles := make([]infrav1alpha1.SecurityGroupRole, len(defaultHCSecurityGroupRoles)) + copy(roles, defaultHCSecurityGroupRoles) + return roles +} + // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=huaweicloudclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=huaweicloudclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=huaweicloudclusters/finalizers,verbs=update @@ -146,7 +161,7 @@ func (r *HuaweiCloudClusterReconciler) reconcileNormal(clusterScope *scope.Clust } // reconcile security group - sgSvc, err := securitygroup.NewService(clusterScope) + sgSvc, err := securitygroup.NewService(clusterScope, securityGroupRolesForCluster()) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to create security group service") } @@ -186,7 +201,7 @@ func (r *HuaweiCloudClusterReconciler) reconcileDelete(clusterScope *scope.Clust } // delete security group - sgSvc, err := securitygroup.NewService(clusterScope) + sgSvc, err := securitygroup.NewService(clusterScope, securityGroupRolesForCluster()) if err != nil { return errors.Wrap(err, "failed to create security group service") } diff --git a/pkg/scope/cluster.go b/pkg/scope/cluster.go index 43c76fb..ff29757 100644 --- a/pkg/scope/cluster.go +++ b/pkg/scope/cluster.go @@ -120,6 +120,16 @@ func (s *ClusterScope) Region() string { return s.HCCluster.Spec.Region } +// SecurityGroups returns the cluster security groups as a map, it creates the map if empty. +func (s *ClusterScope) SecurityGroups() map[infrav1alpha1.SecurityGroupRole]infrav1alpha1.SecurityGroup { + return s.HCCluster.Status.Network.SecurityGroups +} + +// SetSecurityGroups updates the cluster security groups. +func (s *ClusterScope) SetSecurityGroups(sg map[infrav1alpha1.SecurityGroupRole]infrav1alpha1.SecurityGroup) { + s.HCCluster.Status.Network.SecurityGroups = sg +} + // PatchObject persists the cluster configuration and status. func (s *ClusterScope) PatchObject() error { applicableConditions := []clusterv1.ConditionType{ diff --git a/pkg/services/network/network.go b/pkg/services/network/network.go index b2ef414..8195538 100644 --- a/pkg/services/network/network.go +++ b/pkg/services/network/network.go @@ -32,8 +32,6 @@ func (s *Service) ReconcileNetwork() error { return err } - // TODO: Public IPs - // TODO: Routing tables klog.Infof("Reconcile network completed successfully") @@ -67,8 +65,6 @@ func (s *Service) DeleteNetwork() error { clusterv1.DeletedReason, clusterv1.ConditionSeverityInfo, "") - // TODO: Delete Public IPs - // TODO: Delete Route Tables // Delete VPC diff --git a/pkg/services/network/subnet.go b/pkg/services/network/subnet.go index aee7b1f..c9a8d3e 100644 --- a/pkg/services/network/subnet.go +++ b/pkg/services/network/subnet.go @@ -15,12 +15,15 @@ func (s *Service) reconcileSubnets() error { } // Check if subnet exists, if not create it - request := &model.ListSubnetsRequest{} + request := &model.ListSubnetsRequest{ + VpcId: &s.scope.VPC().Id, + } response, err := s.vpcClient.ListSubnets(request) if err != nil { return errors.Wrap(err, "failed to list subnets") } + var subnet *model.Subnet if len(*response.Subnets) == 0 { createRequest := &model.CreateSubnetRequest{} subnetbody := &model.CreateSubnetOption{ @@ -37,23 +40,25 @@ func (s *Service) reconcileSubnets() error { return errors.Wrap(err, "failed to create subnet") } - s.scope.SetSubnets([]infrav1alpha1.SubnetSpec{ - { - Id: response.Subnet.Id, - Name: response.Subnet.Name, - Cidr: response.Subnet.Cidr, - GatewayIp: response.Subnet.GatewayIp, - VpcId: response.Subnet.VpcId, - NeutronNetworkId: response.Subnet.NeutronNetworkId, - NeutronSubnetId: response.Subnet.NeutronSubnetId, - }, - }) - klog.Infof("Subnet create response: %v", response) - klog.Infof("Created subnet") + subnet = response.Subnet + klog.Infof("Subnet created, response: %v", response) } else { + subnet = &(*response.Subnets)[0] klog.Infof("Subnet already exists") } + s.scope.SetSubnets([]infrav1alpha1.SubnetSpec{ + { + Id: subnet.Id, + Name: subnet.Name, + Cidr: subnet.Cidr, + GatewayIp: subnet.GatewayIp, + VpcId: subnet.VpcId, + NeutronNetworkId: subnet.NeutronNetworkId, + NeutronSubnetId: subnet.NeutronSubnetId, + }, + }) + // Persist the new default subnets to HCCluster if err := s.scope.PatchObject(); err != nil { klog.Errorf("Failed to patch HCCluster: %v", err) diff --git a/pkg/services/network/vpc.go b/pkg/services/network/vpc.go index 57e9796..1cadd08 100644 --- a/pkg/services/network/vpc.go +++ b/pkg/services/network/vpc.go @@ -12,43 +12,30 @@ import ( ) func (s *Service) reconcileVPC() error { - // Check if VPC exists, if not create it + // check if VPC exists, if not create it if s.scope.VPC().Id != "" { klog.Infof("VPC %s already exists", s.scope.VPC().Id) return nil } - request := &model.ListVpcsRequest{} - response, err := s.vpcClient.ListVpcs(request) - if err != nil { - return errors.Wrap(err, "failed to list VPCs") + createRequest := &model.CreateVpcRequest{} + cidrVpc := "192.168.0.0/16" + nameVpc := "vpc-caph" + vpcbody := &model.CreateVpcOption{ + Cidr: &cidrVpc, + Name: &nameVpc, + } + createRequest.Body = &model.CreateVpcRequestBody{ + Vpc: vpcbody, } - var vpc *model.Vpc - if len(*response.Vpcs) == 0 { - - createRequest := &model.CreateVpcRequest{} - cidrVpc := "192.168.0.0/16" - nameVpc := "vpc-caph" - vpcbody := &model.CreateVpcOption{ - Cidr: &cidrVpc, - Name: &nameVpc, - } - createRequest.Body = &model.CreateVpcRequestBody{ - Vpc: vpcbody, - } - - createRes, err := s.vpcClient.CreateVpc(createRequest) - if err != nil { - return errors.Wrap(err, "failed to create VPC") - } - vpc = createRes.Vpc - klog.Infof("VPC create response: %v", createRes) - klog.Infof("Created VPC %s", vpc.Id) - } else { - vpc = &(*response.Vpcs)[0] - klog.Infof("VPC %s already exists", vpc.Id) + createRes, err := s.vpcClient.CreateVpc(createRequest) + if err != nil { + return errors.Wrap(err, "failed to create VPC") } + vpc := createRes.Vpc + klog.Infof("VPC create response: %v", createRes) + klog.Infof("Created VPC %s", vpc.Id) s.scope.VPC().Id = vpc.Id s.scope.VPC().Name = vpc.Name diff --git a/pkg/services/securitygroup/securitygroups.go b/pkg/services/securitygroup/securitygroups.go index 59abc8b..ac11816 100644 --- a/pkg/services/securitygroup/securitygroups.go +++ b/pkg/services/securitygroup/securitygroups.go @@ -70,6 +70,7 @@ func (s *Service) ReconcileSecurityGroups() error { }, } + securityGroupRules := []infrav1alpha1.SecurityGroupRule{} // Check and create ingress rules for _, rule := range ingressRules { if !s.securityGroupRuleExists(securityGroupID, rule) { @@ -78,17 +79,46 @@ func (s *Service) ReconcileSecurityGroups() error { SecurityGroupRule: &rule, }, } - _, err := s.vpcClient.NeutronCreateSecurityGroupRule(createSecurityGroupRuleRequest) + ruleRep, err := s.vpcClient.NeutronCreateSecurityGroupRule(createSecurityGroupRuleRequest) if err != nil { return fmt.Errorf("failed to create security group rule: %v", err) } + + securityGroupRules = append(securityGroupRules, infrav1alpha1.SecurityGroupRule{ + Id: ruleRep.SecurityGroupRule.Id, + Direction: ruleRep.SecurityGroupRule.Direction.Value(), + Ethertype: ruleRep.SecurityGroupRule.Ethertype, + PortRangeMin: ruleRep.SecurityGroupRule.PortRangeMin, + PortRangeMax: ruleRep.SecurityGroupRule.PortRangeMax, + Protocol: ruleRep.SecurityGroupRule.Protocol, + RemoteIpPrefix: ruleRep.SecurityGroupRule.RemoteIpPrefix, + }) klog.Infof("Created security group rule: %+v", rule) } else { klog.Infof("Security group rule already exists: %+v", rule) } } + securityGroups := s.scope.SecurityGroups() + if securityGroups == nil { + securityGroups = make(map[infrav1alpha1.SecurityGroupRole]infrav1alpha1.SecurityGroup) + s.scope.SetSecurityGroups(securityGroups) + } + + for _, role := range s.roles { + securityGroups[role] = infrav1alpha1.SecurityGroup{ + ID: securityGroupID, + Name: securityGroupName, + SecurityGroupRules: securityGroupRules, + } + } + + s.scope.SetSecurityGroups(securityGroups) + conditions.MarkTrue(s.scope.InfraCluster(), infrav1alpha1.ClusterSecurityGroupsReadyCondition) + if err := s.scope.PatchObject(); err != nil { + return fmt.Errorf("failed to patch HCCluster: %v", err) + } return nil } diff --git a/pkg/services/securitygroup/service.go b/pkg/services/securitygroup/service.go index e728795..f80bbd4 100644 --- a/pkg/services/securitygroup/service.go +++ b/pkg/services/securitygroup/service.go @@ -3,6 +3,7 @@ package securitygroup import ( "k8s.io/klog/v2" + infrav1alpha1 "github.com/HuaweiCloudDeveloper/cluster-api-provider-huawei/api/v1alpha1" "github.com/HuaweiCloudDeveloper/cluster-api-provider-huawei/pkg/scope" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" vpc "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/vpc/v2" @@ -11,10 +12,11 @@ import ( type Service struct { scope *scope.ClusterScope + roles []infrav1alpha1.SecurityGroupRole vpcClient *vpc.VpcClient } -func NewService(scope *scope.ClusterScope) (*Service, error) { +func NewService(scope *scope.ClusterScope, roles []infrav1alpha1.SecurityGroupRole) (*Service, error) { region, err := region.SafeValueOf(scope.Region()) if err != nil { klog.Errorf("Failed to get region: %v", err) @@ -34,6 +36,7 @@ func NewService(scope *scope.ClusterScope) (*Service, error) { return &Service{ scope: scope, + roles: roles, vpcClient: vpcCli, }, nil }