Add files via upload

prototype/2. Technical_ EU STR - Infra Single Digital Entry Point/README.md

@@ -0,0 +1,183 @@
+# Pulumi projects
+This repository holds 3 pulumi projects:
+* The kafka project that is used to deploy a kafka cluster on Confluent
+* the AWS project that is to deploy all AWS infrastructure including the EKS cluster
+* The charts project that is used to deploy Helm Charts on the EKS cluster
+
+To deploy a new version of the application code on the EKS cluster. First the application docker image needs to be built, 
+This is automated on the application code repository: `https://github.com/your-org/your-repo`. This repository has a semver package for the docker image.
+
+After the image build is done, the Helm Chart app version can be updated here: `https://github.com/your-org/your-repo/blob/your-commit-id/chart/str/Chart.yaml#L24`.
+
+The deployment of the new version happens by the Pulumi project: `charts`.
+
+The deployment can be verified with `kubectl` (you should have the `kubeconfig.yaml` of the environment and AWS credentials of the AWS account).
+
+```text
+kubectl describe deployment your-app -n your-namespace  
+
+```
+```text
+Name:                   [your-deployment]  
+Namespace:              [your-namespace]  
+CreationTimestamp:      [Mon, 17 Jun 2024 12:11:50 +0200]  
+Labels:                 app.kubernetes.io/instance=[your-deployment]  
+                        app.kubernetes.io/managed-by=[Helm]  
+                        app.kubernetes.io/name=[your-deployment]  
+                        app.kubernetes.io/version=[v0.0.24]  
+                        helm.sh/chart=[your-deployment-0.1.0]  
+Annotations:            deployment.kubernetes.io/revision: [4]  
+Selector:               app.kubernetes.io/instance=[your-deployment],app.kubernetes.io/name=[your-deployment]  
+Replicas:               [2] desired | [2] updated | [2] total | [2] available | [0] unavailable  
+StrategyType:           [RollingUpdate]  
+MinReadySeconds:        [0]  
+RollingUpdateStrategy:  [25%] max unavailable, [25%] max surge  
+Pod Template:  
+  Labels:           app.kubernetes.io/instance=[your-deployment]  
+                    app.kubernetes.io/managed-by=[Helm]  
+                    app.kubernetes.io/name=[your-deployment]  
+                    app.kubernetes.io/version=[v0.0.24] (Latest version of the code)  
+                    helm.sh/chart=[your-deployment-0.1.0]  
+  Service Account:  [your-service-account]  
+  Containers:  
+   [your-container]:  
+    Image:        ghcr.io/[your-org]/[your-repo]:[v0.0.24]  
+    Port:         [4443/TCP]  
+  
+'''
+
+## Kafka configuration
+```zsh
+pulumi config set project:name [your-name]   
+pulumi config set confluentcloud:cloudApiKey [your-key] [add key]  
+pulumi config set confluentcloud:cloudApiSecret [your-secret] --secret
+
+pulumi config set --path 'kafka:topics[0]' listings
+pulumi config set --path 'kafka:topics[1]' activity-data
+```
+
+to get stack output
+```zsh
+pulumi stack output
+```
+
+To get the api secret
+```zsh
+pulumi stack output appApiSecret --show-secrets
+```
+
+## AWS configuration
+### template
+# General config  
+pulumi config set aws: [choose region]  
+pulumi config set project:name [project name]  
+pulumi config set project:namespace [namespace]  
+pulumi config set project:vpcCidr [CIDR block e.g., "10.30.0.0/21"]  
+
+# Domain to expose app on  
+pulumi config set project:hostname [hostname]  
+pulumi config set project:domain [domain e.g., "example.com"]  
+pulumi config set project:domainHostedZoneId [hosted zone ID]  
+
+# The nlb generated by nginx-ingress reference  
+pulumi config set project:nlbHostedZoneId [NLB hosted zone ID]  
+pulumi config set project:nlbDnsName [NLB DNS name]  
+
+# Export of the k8s config  
+pulumi stack output kubeconfig > [kubeconfig file name].yml  
+export KUBECONFIG=./[kubeconfig file name].yml
+
+## Chart configuration
+```zsh
+# General config
+pulumi config set aws:region [your-region]   
+pulumi config set project:namespace [your-namespace]   
+pulumi config set project:name [your-project-name]   
+pulumi config set project:efs [your-efs-id]  
+
+Config for letsencrypt  
+pulumi config set cert:emailContact [your-email@example.com]   
+pulumi config set acme:serverUrl [https://acme-v02.api.letsencrypt.org/directory]   
+pulumi config set project:awsStackRef [your-org]/[your-project]/[your-environment]  
+
+Secret for docker image pull  
+pulumi config set image:password [your-docker-password] --secret  
+
+Next 2 values are only needed for local development, when using cert-manager the tls secret will be generated  
+pulumi config set tls:key $(base64 -i [your-path]/[your-key-file].pem) --secret   
+pulumi config set tls:crt $(base64 -i [your-path]/[your-crt-file].pem) --secret  
+
+Reference to the local chart definition  
+pulumi config set chart:file [your-path]/[your-chart-directory]   
+pulumi config set values:file [your-path]/[your-values-file].yaml  
+
+References for the app connection towards kafka  
+pulumi config set kafka:boostrapServers [your-kafka-servers]   
+pulumi config set kafka:username [your-kafka-username] --secret   
+pulumi config set kafka:password [your-kafka-password] --secret  
+```
+
+### Overview of resources
+![aws.png](aws.png)
+
+![charts.png](charts.png)
+
+## AWS account bootstrap
+* Activate MFA
+* IAM user and role access to Billing information
+* IAM user creation with role:
+* Creation of `[your-group-name]` group with role: `AdministratorAccess`
+* Creation of IAM user: [your-email@example.com]
+* IAM user login URL: [https://your-account-id.signin.aws.amazon.com/console]
+* Login with IAM user and setup MFA
+* Domain registration: `[your-domain]` without auto-renew!
+
+todo:
+* Set alternate contacts
+
+## Procedure to deploy a new version of the code
+
+Your code changes should be integrated (merged) into the main branch of the repo.
+
+In the GitHub releases or on Swagger you can see the current version number, e.g., `v0.0.24`. 
+You should tag your commit with the next version, e.g., `v0.0.25`.
+Note that semver uses a small v!
+
+![tag-commit.png](img/tag-commit.png)
+
+And push this tag to GitHub.
+
+![push-tag.png](img/push-tag.png)
+
+This will launch a GitHub action in the repository.
+
+![github-action-docker-build.png](img/github-action-docker-build.png)
+
+Verify the build is successful; the build will create a new release and a new version of the Docker image.
+
+![new-release.png](img/new-release.png)
+
+The details of the build:
+
+![github-action-docker-build-details.png](img/github-action-docker-build-details.png)
+
+If the new version `v0.0.25` is ready to be deployed, update the version number of the chart.
+
+![chart-version.png](img/chart-version.png)
+
+Commit and push this change.
+
+In this repository [https://github.com/your-org/your-repo-infra-internal] a GitHub action is foreseen for Helm charts deployment.
+This workflow needs to be triggered manually.
+
+![pulumi-charts-run-workflow.png](img/pulumi-charts-run-workflow.png)
+
+![pulumi-charts-run-workflow-active.png](img/pulumi-charts-run-workflow-active.png)
+
+Note, this can take a couple of minutes.
+
+The log will indicate the deployment will be updated:
+```sh
+~  kubernetes:apps/v1:Deployment your-deployment:your-namespace/your-deployment updating (4s) [diff: ~metadata,spec]; Waiting for app ReplicaSet to be available (0/1 Pods available)
+```
+![str-pods.png](img/str-pods.png)

prototype/2. Technical_ EU STR - Infra Single Digital Entry Point/aws/Pulumi.dev.yaml

@@ -0,0 +1,9 @@
+config:
+  aws:region: eu-west-1
+  project:domain: turpin.ch
+  project:domainHostedZoneId: Z01857511ZHXCOF620CLC
+  project:hostname: eu-str
+  project:name: eu-str-ap
+  project:nlbDnsName: aaa502d5c733a46948e320914f5eb6ee-057019931f384024.elb.eu-west-1.amazonaws.com
+  project:nlbHostedZoneId: Z2IFOLAFXWLO4F
+  project:vpcCidr: 10.30.0.0/21

prototype/2. Technical_ EU STR - Infra Single Digital Entry Point/aws/Pulumi.pilot.yaml

@@ -0,0 +1,10 @@
+config:
+  aws:region: eu-west-1
+  project:domain: sdep-pilot.eu
+  project:domainHostedZoneId: Z01073105SAGDC764ISD
+  project:hostname: eu-str
+  project:name: eu-str-ap
+  project:namespace: str
+  project:nlbDnsName: aff96b68d9c20426d86c6ad5085fcdd2-909d7836c7796b7e.elb.eu-west-1.amazonaws.com
+  project:nlbHostedZoneId: Z2IFOLAFXWLO4F
+  project:vpcCidr: 10.30.0.0/21

prototype/2. Technical_ EU STR - Infra Single Digital Entry Point/aws/Pulumi.yaml

@@ -0,0 +1,7 @@
+name: str-ap-infra-internal-aws
+runtime: go
+description: AWS infra
+config:
+  pulumi:tags:
+    value:
+      pulumi:template: aws-go

prototype/2. Technical_ EU STR - Infra Single Digital Entry Point/aws/dns.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/route53"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
+)
+
+func setDnsAlias(ctx *pulumi.Context) error {
+
+	// Get config values
+	hostname := config.Get(ctx, "project:hostname")
+	domain := config.Get(ctx, "project:domain")
+	domainHostedZoneId := config.Get(ctx, "project:domainHostedZoneId")
+	nlbHostedZoneId := config.Get(ctx, "project:nlbHostedZoneId")
+	nlbDnsName := config.Get(ctx, "project:nlbDnsName")
+
+	fqdn := hostname + "." + domain
+
+	_, err := route53.NewRecord(ctx, hostname, &route53.RecordArgs{
+		ZoneId: pulumi.String(domainHostedZoneId),
+		Type:   pulumi.String(route53.RecordTypeA),
+		Name:   pulumi.String(fqdn),
+		Aliases: route53.RecordAliasArray{
+			&route53.RecordAliasArgs{
+				Name:                 pulumi.String(nlbDnsName),
+				ZoneId:               pulumi.String(nlbHostedZoneId),
+				EvaluateTargetHealth: pulumi.Bool(true),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Export the application route53 hostedzoneId and fqdn
+	ctx.Export("domainHostedZoneId", pulumi.String(domainHostedZoneId))
+	ctx.Export("fqdn", pulumi.String(fqdn))
+
+	return nil
+}

prototype/2. Technical_ EU STR - Infra Single Digital Entry Point/aws/efs.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	awsec2 "github.com/pulumi/pulumi-aws/sdk/v6/go/aws/ec2"
+	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/efs"
+	awsvpc "github.com/pulumi/pulumi-aws/sdk/v6/go/aws/vpc"
+	"github.com/pulumi/pulumi-awsx/sdk/v2/go/awsx/ec2"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+)
+
+func createFileShare(ctx *pulumi.Context, vpc *ec2.Vpc) (*efs.FileSystem, error) {
+	projectName := getProjectName(ctx)
+
+	fileShare, _ := efs.NewFileSystem(ctx, projectName, &efs.FileSystemArgs{
+		CreationToken: pulumi.String(projectName),
+		Tags: pulumi.StringMap{
+			"Name": pulumi.String(projectName),
+		},
+	})
+
+	// Create SG for fileShare
+	allowFileShare, _ := awsec2.NewSecurityGroup(ctx, "allowFileShare", &awsec2.SecurityGroupArgs{
+		Name:        pulumi.String("allow_efs_file_share"),
+		Description: pulumi.String("Allow efs file share"),
+		VpcId:       vpc.VpcId,
+		Tags: pulumi.StringMap{
+			"Name": pulumi.String("allow_efs_file_share"),
+		},
+	})
+
+	vpc.PrivateSubnetIds.ApplyT(func(subnetIds []string) error {
+		for _, subnetId := range subnetIds {
+			subnet, err := awsec2.LookupSubnet(ctx, &awsec2.LookupSubnetArgs{
+				Id: pulumi.StringRef(subnetId),
+			}, nil)
+			if err != nil {
+				return err
+			}
+
+			_, err = awsvpc.NewSecurityGroupIngressRule(ctx, "allow_efs_ingress-"+subnetId, &awsvpc.SecurityGroupIngressRuleArgs{
+				SecurityGroupId: allowFileShare.ID(),
+				CidrIpv4:        pulumi.String(subnet.CidrBlock),
+				FromPort:        pulumi.Int(2049),
+				IpProtocol:      pulumi.String("tcp"),
+				ToPort:          pulumi.Int(2049),
+			})
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	_, _ = awsvpc.NewSecurityGroupEgressRule(ctx, "allow_efs_egress", &awsvpc.SecurityGroupEgressRuleArgs{
+		SecurityGroupId: allowFileShare.ID(),
+		CidrIpv4:        pulumi.String("0.0.0.0/0"),
+		FromPort:        pulumi.Int(0),
+		IpProtocol:      pulumi.String("-1"),
+		ToPort:          pulumi.Int(0),
+	})
+
+	// Create EFS mount targets for each private subnet
+	vpc.PrivateSubnetIds.ApplyT(func(subnetIds []string) error {
+		for _, subnetId := range subnetIds {
+			_, err := efs.NewMountTarget(ctx, projectName+"-"+subnetId, &efs.MountTargetArgs{
+				FileSystemId: fileShare.ID(),
+				SubnetId:     pulumi.String(subnetId),
+				SecurityGroups: pulumi.StringArray{
+					allowFileShare.ID(),
+				},
+			}, pulumi.DependsOn([]pulumi.Resource{fileShare}))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	return fileShare, nil
+}