From 33c2de2717235be700a5269b3bd20c4aeb0362d3 Mon Sep 17 00:00:00 2001 From: Rohit Kulshreshtha Date: Wed, 6 Nov 2024 16:04:54 -0800 Subject: [PATCH] feat(Gossip KV): Cleanup Local and AWS Deployments (#1545) Closes #1529 (P0), #1528 (P1). Summary of changes: 1. Local Deployments use Terraform. 2. Both Local / AWS Terraform projects have been split into separate "infra" and "application" sections because there are intervening steps (building images, etc.) that need to be done before applications can be deployed. 3. Independent documentation for each local/AWS deployment - should make it less confusing because these are expected to diverge. --- datastores/gossip_kv/Makefile | 83 ---- datastores/gossip_kv/README.md | 37 +- datastores/gossip_kv/deployment/aws/Makefile | 76 +++ datastores/gossip_kv/deployment/aws/README.md | 89 ++++ .../seed_node_config.yaml} | 0 .../deployment/aws/terraform/.gitignore | 31 +- .../terraform/application/.terraform.lock.hcl | 22 + .../aws/terraform/application/main.tf | 447 ++++++++++++++++++ .../aws/terraform/application/outputs.tf | 9 + .../aws/terraform/application/variables.tf | 17 + .../aws/terraform/infra/.terraform.lock.hcl | 125 +++++ .../deployment/aws/terraform/infra/main.tf | 143 ++++++ .../deployment/aws/terraform/infra/outputs.tf | 28 ++ .../aws/terraform/infra/variables.tf | 17 + .../gossip_kv/deployment/local/Makefile | 46 ++ .../gossip_kv/deployment/local/README.md | 109 +++++ .../gossip_kv/deployment/local/objects.yaml | 90 ---- .../deployment/local/seed_node_config.yaml | 17 + .../deployment/local/terraform/.gitignore | 30 ++ .../terraform/application/.terraform.lock.hcl | 21 + .../local/terraform/application/main.tf | 153 ++++++ .../local/terraform/infra/.terraform.lock.hcl | 40 ++ .../deployment/local/terraform/infra/main.tf | 26 + .../local/terraform/infra/outputs.tf | 18 + datastores/gossip_kv/server/README.md | 44 -- 25 files changed, 1468 insertions(+), 250 deletions(-) delete mode 100644 datastores/gossip_kv/Makefile create mode 100644 datastores/gossip_kv/deployment/aws/Makefile create mode 100644 datastores/gossip_kv/deployment/aws/README.md rename datastores/gossip_kv/deployment/{local/updated_seed_node_config.yaml => aws/seed_node_config.yaml} (100%) create mode 100644 datastores/gossip_kv/deployment/aws/terraform/application/.terraform.lock.hcl create mode 100644 datastores/gossip_kv/deployment/aws/terraform/application/main.tf create mode 100644 datastores/gossip_kv/deployment/aws/terraform/application/outputs.tf create mode 100644 datastores/gossip_kv/deployment/aws/terraform/application/variables.tf create mode 100644 datastores/gossip_kv/deployment/aws/terraform/infra/.terraform.lock.hcl create mode 100644 datastores/gossip_kv/deployment/aws/terraform/infra/main.tf create mode 100644 datastores/gossip_kv/deployment/aws/terraform/infra/outputs.tf create mode 100644 datastores/gossip_kv/deployment/aws/terraform/infra/variables.tf create mode 100644 datastores/gossip_kv/deployment/local/Makefile create mode 100644 datastores/gossip_kv/deployment/local/README.md delete mode 100644 datastores/gossip_kv/deployment/local/objects.yaml create mode 100644 datastores/gossip_kv/deployment/local/seed_node_config.yaml create mode 100644 datastores/gossip_kv/deployment/local/terraform/.gitignore create mode 100644 datastores/gossip_kv/deployment/local/terraform/application/.terraform.lock.hcl create mode 100644 datastores/gossip_kv/deployment/local/terraform/application/main.tf create mode 100644 datastores/gossip_kv/deployment/local/terraform/infra/.terraform.lock.hcl create mode 100644 datastores/gossip_kv/deployment/local/terraform/infra/main.tf create mode 100644 datastores/gossip_kv/deployment/local/terraform/infra/outputs.tf delete mode 100644 datastores/gossip_kv/server/README.md diff --git a/datastores/gossip_kv/Makefile b/datastores/gossip_kv/Makefile deleted file mode 100644 index dbfe144f0d78..000000000000 --- a/datastores/gossip_kv/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# Makefile - -## Minikube Options. -MINIKUBE_DISK_SIZE:=100g -MINIKUBE_CPUS:=16 -MINIKUBE_MEMORY:=32768 - -BASE_IMAGE_VERSION:=latest -SERVER_IMAGE_VERSION:=latest -CLI_IMAGE_VERSION:=latest -LOAD_TEST_IMAGE_VERSION:=latest - -# Docker Image Tags -BASE_IMAGE_TAG:=hydroflow-gossip-kv-base-image:$(BASE_IMAGE_VERSION) -SERVER_IMAGE_TAG:=hydroflow-gossip-kv-server:$(SERVER_IMAGE_VERSION) -CLI_IMAGE_TAG:=hydroflow-gossip-kv-cli:$(CLI_IMAGE_VERSION) -LOAD_TEST_IMAGE_TAG:=hydroflow-gossip-kv-load-test:$(LOAD_TEST_IMAGE_VERSION) - -AWS_TERRAFORM_PATH=../../datastores/gossip_kv/deployment/aws/terraform - -# Target to start Minikube with specific options -start_minikube: - minikube start --disk-size=$(MINIKUBE_DISK_SIZE) --cpus=$(MINIKUBE_CPUS) --memory=$(MINIKUBE_MEMORY) - @echo "Please run 'eval \$$(minikube docker-env)' to use the Minikube Docker daemon" - -# Target to build the Docker images -build_docker_images: build_base_image build_server_image build_cli_image build_load_test_image - -build_base_image: - docker build -t "$(BASE_IMAGE_TAG)" -f ../../datastores/gossip_kv/server/baseimage.Dockerfile ../.. - -build_server_image: - docker build -t "$(SERVER_IMAGE_TAG)" -f ../../datastores/gossip_kv/server/Dockerfile ../.. - -build_cli_image: - docker build -t "$(CLI_IMAGE_TAG)" -f ../../datastores/gossip_kv/cli/Dockerfile ../.. - -build_load_test_image: - docker build -t "$(LOAD_TEST_IMAGE_TAG)" -f ../../datastores/gossip_kv/load_test_server/Dockerfile ../.. - -# Target to clean up the Minikube cluster -clean_local: - minikube delete - -# Target to deploy the Gossip KV Server to the Minikube cluster -deploy_local: - kubectl apply -f ../../datastores/gossip_kv/server/local - -# Target to delete the Minikube cluster and build again -rebuild_local: clean_local start_minikube build_docker_images - -aws_terraform_init: - terraform -chdir="$(AWS_TERRAFORM_PATH)" init - -aws_terraform_apply: - terraform -chdir="$(AWS_TERRAFORM_PATH)" apply - -aws_setup_kubectl: - @echo "Setting up kubectl to work with AWS EKS Cluster" - aws eks update-kubeconfig --region $$(terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw region) --name $$(terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw cluster_name) - -aws_upload_docker_images: build_docker_images - $(eval SERVER_REPO_URL := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -json repository_urls | jq -r '.["gossip_kv_server"]')) - $(eval CLI_REPO_URL := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -json repository_urls | jq -r '.["gossip_kv_cli"]')) - $(eval LOAD_TEST_REPO_URL := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -json repository_urls | jq -r '.["gossip_kv_load_test"]')) - $(eval REGION := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw region)) - docker tag $(SERVER_IMAGE_TAG) $(SERVER_REPO_URL):$(SERVER_IMAGE_VERSION) - docker tag $(CLI_IMAGE_TAG) $(CLI_REPO_URL):$(CLI_IMAGE_VERSION) - docker tag $(LOAD_TEST_IMAGE_TAG) $(LOAD_TEST_REPO_URL):$(LOAD_TEST_IMAGE_VERSION) - aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(SERVER_REPO_URL) - docker push $(SERVER_REPO_URL):$(SERVER_IMAGE_VERSION) - aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(CLI_REPO_URL) - docker push $(CLI_REPO_URL):$(CLI_IMAGE_VERSION) - aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(LOAD_TEST_REPO_URL) - docker push $(LOAD_TEST_REPO_URL):$(LOAD_TEST_IMAGE_VERSION) - -aws_tunnel_grafana: - $(eval GRAFANA_PORT := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw grafana_port)) - kubectl port-forward svc/grafana $(GRAFANA_PORT):$(GRAFANA_PORT) - -aws_tunnel_prometheus: - $(eval PROMETHEUS_PORT := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw prometheus_port)) - kubectl port-forward svc/prometheus $(PROMETHEUS_PORT):$(PROMETHEUS_PORT) \ No newline at end of file diff --git a/datastores/gossip_kv/README.md b/datastores/gossip_kv/README.md index b282dafe0f1b..e798fedc52a1 100644 --- a/datastores/gossip_kv/README.md +++ b/datastores/gossip_kv/README.md @@ -48,36 +48,9 @@ The `sys` data section contains system data / state that is required by the key- ### `usr` Data -## Protocol +# Deployment +## Local (Minikube) Deployment +See [Minikube Deployment](./deployment/local/README.md) for more information. -## Checkpoints - -# Running Locally Using Minikube -## Install Docker Desktop -```shell -brew install --cask docker -``` -### Run docker (macOS) -``` -open -a Docker -``` - -## Install Minikube -Read more [here](https://minikube.sigs.k8s.io/docs/start/) -```shell -brew install minikube -``` - -## Start Minikube -```shell -minikube start -``` - -## Install `kubectl` -```shell -brew install kubectl -``` -## Configure Minikube to use your Docker Environment -```shell -eval $(minikube -p minikube docker-env) -``` \ No newline at end of file +## AWS Deployment +See [AWS Deployment](./deployment/aws/README.md) for more information. \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/aws/Makefile b/datastores/gossip_kv/deployment/aws/Makefile new file mode 100644 index 000000000000..1ac35ba5c8f8 --- /dev/null +++ b/datastores/gossip_kv/deployment/aws/Makefile @@ -0,0 +1,76 @@ +INFRA_PATH=./terraform/infra +APPLICATION_PATH=./terraform/application + +BASE_IMAGE_VERSION:=latest +SERVER_IMAGE_VERSION:=latest +CLI_IMAGE_VERSION:=latest +LOAD_TEST_IMAGE_VERSION:=latest + +# Docker Image Tags +BASE_IMAGE_TAG:=hydroflow-gossip-kv-base-image:$(BASE_IMAGE_VERSION) +SERVER_IMAGE_TAG:=hydroflow-gossip-kv-server:$(SERVER_IMAGE_VERSION) +CLI_IMAGE_TAG:=hydroflow-gossip-kv-cli:$(CLI_IMAGE_VERSION) +LOAD_TEST_IMAGE_TAG:=hydroflow-gossip-kv-load-test:$(LOAD_TEST_IMAGE_VERSION) + +.PHONY : init infra docker_images base_image server_image cli_image upload_docker_images application config clean + +init: + terraform -chdir="$(INFRA_PATH)" init + terraform -chdir="$(APPLICATION_PATH)" init + +infra: + terraform -chdir="$(INFRA_PATH)" apply -auto-approve + +kubectl_setup: + @echo "Setting up kubectl to work with AWS EKS Cluster" + aws eks update-kubeconfig --region $$(terraform -chdir=$(INFRA_PATH) output -raw region) --name $$(terraform -chdir=$(INFRA_PATH) output -raw cluster_name) + +docker_images: base_image server_image cli_image + +base_image: + docker build -t "$(BASE_IMAGE_TAG)" -f ../../../../datastores/gossip_kv/server/baseimage.Dockerfile ../../../.. + +server_image: + docker build -t "$(SERVER_IMAGE_TAG)" -f ../../../../datastores/gossip_kv/server/Dockerfile ../../../.. + +cli_image: + docker build -t "$(CLI_IMAGE_TAG)" -f ../../../../datastores/gossip_kv/cli/Dockerfile ../../../.. + +upload_docker_images: docker_images + $(eval SERVER_REPO_URL := $(shell terraform -chdir=$(INFRA_PATH) output -json repository_urls | jq -r '.["gossip_kv_server"]')) + $(eval CLI_REPO_URL := $(shell terraform -chdir=$(INFRA_PATH) output -json repository_urls | jq -r '.["gossip_kv_cli"]')) + $(eval LOAD_TEST_REPO_URL := $(shell terraform -chdir=$(INFRA_PATH) output -json repository_urls | jq -r '.["gossip_kv_load_test"]')) + $(eval REGION := $(shell terraform -chdir=$(INFRA_PATH) output -raw region)) + echo $(SERVER_REPO_URL) + docker tag $(SERVER_IMAGE_TAG) $(SERVER_REPO_URL):$(SERVER_IMAGE_VERSION) + docker tag $(CLI_IMAGE_TAG) $(CLI_REPO_URL):$(CLI_IMAGE_VERSION) + docker tag $(LOAD_TEST_IMAGE_TAG) $(LOAD_TEST_REPO_URL):$(LOAD_TEST_IMAGE_VERSION) + aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(SERVER_REPO_URL) + docker push $(SERVER_REPO_URL):$(SERVER_IMAGE_VERSION) + aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(CLI_REPO_URL) + docker push $(CLI_REPO_URL):$(CLI_IMAGE_VERSION) + aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(LOAD_TEST_REPO_URL) + docker push $(LOAD_TEST_REPO_URL):$(LOAD_TEST_IMAGE_VERSION) + +application: + terraform -chdir="$(APPLICATION_PATH)" apply -auto-approve + +tunnel_grafana: + $(eval GRAFANA_PORT := $(shell terraform -chdir=$(APPLICATION_PATH) output -raw grafana_port)) + @echo "Grafana will be accessible at http://localhost:$(GRAFANA_PORT)" + kubectl port-forward svc/grafana $(GRAFANA_PORT):$(GRAFANA_PORT) + +tunnel_prometheus: + $(eval PROMETHEUS_PORT := $(shell terraform -chdir=$(APPLICATION_PATH) output -raw prometheus_port)) + @echo "Prometheus will be accessible at http://localhost:$(PROMETHEUS_PORT)" + kubectl port-forward svc/prometheus $(PROMETHEUS_PORT):$(PROMETHEUS_PORT) + + +config: + kubectl apply -f seed_node_config.yaml + +clean: + terraform -chdir="$(APPLICATION_PATH)" destroy -auto-approve + terraform -chdir="$(INFRA_PATH)" destroy -auto-approve + rm -rf $(INFRA_PATH)/.terraform $(INFRA_PATH)/terraform.tfstate $(INFRA_PATH)/terraform.tfstate.backup + rm -rf $(APPLICATION_PATH)/.terraform $(APPLICATION_PATH)/terraform.tfstate $(APPLICATION_PATH)/terraform.tfstate.backup diff --git a/datastores/gossip_kv/deployment/aws/README.md b/datastores/gossip_kv/deployment/aws/README.md new file mode 100644 index 000000000000..7c821f0003a7 --- /dev/null +++ b/datastores/gossip_kv/deployment/aws/README.md @@ -0,0 +1,89 @@ +# AWS Deployment + +## Pre-requisites +- [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) +- [Docker](https://docs.docker.com/get-docker/) +- [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +- [Make](https://www.gnu.org/software/make/) +- [k9s](https://k9scli.io/) + +### Mac Setup Using Homebrew +```shell +brew install terraform +brew install awscli +brew install docker +brew install kubernetes-cli +brew install k9s +``` + +## Initialize Setup +Initializes terraform providers required for local deployment. +```shell +make init +``` + +## Configure AWS Credentials +Make sure you have the AWS credentials configured. Subsequent terraform commands will use these credentials to create +the infrastructure. +```shell +aws configure +``` + +## Create Infrastructure +Creates a local kubernetes cluster using minikube. + +```shell +make infra +``` + +## Kubectl +To enable kubectl usage with the newly created AWS EKS cluster, use the following command. + +```shell +make kubectl_setup +``` + +## K9s +To monitor the cluster using k9s, use the following command. + +```shell +k9s +``` + +## Build & Upload Docker Image +Build and upload the docker image for the application, into ECR. +```shell +make upload_docker_images +``` + +## Create Application +```shell +make application +``` + +## Check if the application is running +```shell +kubectl get pods # All gossip-kv-* pods should show status as "Running" +```` + +## Access Grafana Dashboards +```shell +make tunnel_grafana +``` + +## Access Prometheus +```shell +make tunnel_prometheus +``` + +## Update seed node configuration +```shell +make config +``` + +## Clean +Destroys the AWS resources and cleans up terraform state. +```shell +make clean +``` \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/local/updated_seed_node_config.yaml b/datastores/gossip_kv/deployment/aws/seed_node_config.yaml similarity index 100% rename from datastores/gossip_kv/deployment/local/updated_seed_node_config.yaml rename to datastores/gossip_kv/deployment/aws/seed_node_config.yaml diff --git a/datastores/gossip_kv/deployment/aws/terraform/.gitignore b/datastores/gossip_kv/deployment/aws/terraform/.gitignore index 3fa8c86b7b04..e63962a88795 100644 --- a/datastores/gossip_kv/deployment/aws/terraform/.gitignore +++ b/datastores/gossip_kv/deployment/aws/terraform/.gitignore @@ -1 +1,30 @@ -.terraform +### TERRAFORM IGNORES ### +# Terraform Plugin and module directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +*.tfvars +*.tfvars.json + +# Ignore local override files +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Terraform CLI configuration files +.terraformrc +terraform.rc + +# Backup files created by Terraform during state edits +*.backup + +# Terraform plan files (these are generated and can be large) +*.tfplan diff --git a/datastores/gossip_kv/deployment/aws/terraform/application/.terraform.lock.hcl b/datastores/gossip_kv/deployment/aws/terraform/application/.terraform.lock.hcl new file mode 100644 index 000000000000..d3440c12f8c2 --- /dev/null +++ b/datastores/gossip_kv/deployment/aws/terraform/application/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.33.0" + constraints = ">= 2.32.0" + hashes = [ + "h1:HDyytvOlqNw5fJ0SB/nzgqCWniK4LAZNx23LaPavQq8=", + "zh:255b35790b706d405e987750190658dcaefb663741b96803a9529ba5d7435329", + "zh:362feba1aa820a8e02869ec71d1a08e87243dbce43671dc0995fa6c5a2fafa1d", + "zh:39332abcf75b5dd9c78c79c7c0c094f7d4ca908d1b76bbd2aae67e8e3516710c", + "zh:3e8e7f758bb09a9b5b613c8866e77541f8f00b521070cc86bc095ce61f010baf", + "zh:427883b889b9c36630c3eec4d5c07bc4ae12cc0d358fc17ea42a8049bf8d5275", + "zh:69bfc4ed067a5e4844db1a1809343652ff239aa0a8da089b1671524c44e8740a", + "zh:6b9f731062b945c5020e0930ed9a1b1b50afd2caf751f0e70a282d165c970979", + "zh:6faf9ec006af7ee7014a9c3251d65b701792abb823f149b0b7e4ac4433848201", + "zh:b706f76d695104a47682ee6ab842870f9c70a680f979fa9e7efe34278c0831bc", + "zh:b9bca48de2c92f57389ed58dd2fac564deaccd79a92cafd08edeed3ba6b91d4d", + "zh:bbd3336dbee5aed9880f98e36fb8340e0c6d8f0399a05787521af599ccb3dac4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/datastores/gossip_kv/deployment/aws/terraform/application/main.tf b/datastores/gossip_kv/deployment/aws/terraform/application/main.tf new file mode 100644 index 000000000000..b707edaa89c4 --- /dev/null +++ b/datastores/gossip_kv/deployment/aws/terraform/application/main.tf @@ -0,0 +1,447 @@ +terraform { + backend "local" { + path = "../state/application.tfstate" + } + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.32.0" + } + } +} + +data "terraform_remote_state" "infra" { + backend = "local" + config = { + path = "../state/infra.tfstate" + } +} + + +provider "kubernetes" { + host = data.terraform_remote_state.infra.outputs.cluster_endpoint + cluster_ca_certificate = base64decode(data.terraform_remote_state.infra.outputs.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + data.terraform_remote_state.infra.outputs.cluster_name, + ] + } +} + +resource "kubernetes_stateful_set" "gossip_kv_seed_nodes" { + metadata { + name = "gossip-kv-seed-nodes" + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + service_name = "gossip-kv-seed-nodes" + replicas = 1 + + selector { + match_labels = { + app = "gossip-kv-seed-nodes" + } + } + + template { + metadata { + labels = { + app = "gossip-kv-seed-nodes" + } + annotations = { + "prometheus.io/scrape" : "true" + "prometheus.io/port" : var.pod_monitoring_port + } + } + + spec { + termination_grace_period_seconds = 5 + + container { + name = "gossip-kv-server" + image = "${data.terraform_remote_state.infra.outputs.repository_urls.gossip_kv_server}:latest" + image_pull_policy = "Always" + + env { + name = "RUST_LOG" + value = "trace" + } + + env { + name = "RUST_BACKTRACE" + value = "full" + } + + port { + container_port = 3001 + protocol = "UDP" + } + + port { + container_port = var.pod_monitoring_port + protocol = "TCP" + } + + volume_mount { + name = "gossip-kv-dynamic-config" + mount_path = "/config/dynamic" + } + } + + volume { + name = "gossip-kv-dynamic-config" + + config_map { + name = "gossip-kv-dynamic-config" + } + } + } + } + } +} + +resource "kubernetes_deployment" "gossip_kv_cli" { + metadata { + name = "gossip-kv-cli" + labels = { + app = "gossip-kv-cli" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "gossip-kv-cli" + } + } + + template { + metadata { + labels = { + app = "gossip-kv-cli" + } + } + + spec { + termination_grace_period_seconds = 5 + + container { + name = "gossip-kv-cli" + image = "${data.terraform_remote_state.infra.outputs.repository_urls.gossip_kv_cli}:latest" + image_pull_policy = "Always" + command = ["/bin/sh"] + args = ["-c", "while true; do sleep 3600; done"] + tty = true + + env { + name = "RUST_LOG" + value = "info" + } + } + } + } + } +} + +resource "kubernetes_service" "gossip_kv_seed_nodes" { + metadata { + name = "gossip-kv-seed-nodes" + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + cluster_ip = "None" + selector = { + app = "gossip-kv-seed-nodes" + } + + port { + port = 3001 + target_port = 3001 + protocol = "UDP" + } + } +} + +resource "kubernetes_config_map" "gossip_kv_dynamic_config" { + metadata { + name = "gossip-kv-dynamic-config" + } + + data = { + "dynamic.toml" = < repo} + repository_name = each.value + + repository_read_write_access_arns = [data.aws_caller_identity.current.arn] + repository_lifecycle_policy = jsonencode({ + rules = [ + { + rulePriority = 1, + description = "Keep last 30 images", + selection = { + tagStatus = "tagged", + tagPrefixList = ["v"], + countType = "imageCountMoreThan", + countNumber = 30 + }, + action = { + type = "expire" + } + } + ] + }) + + repository_image_tag_mutability = "MUTABLE" + tags = { + Terraform = "true" + Environment = "dev" + } +} \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/aws/terraform/infra/outputs.tf b/datastores/gossip_kv/deployment/aws/terraform/infra/outputs.tf new file mode 100644 index 000000000000..f965d56a2157 --- /dev/null +++ b/datastores/gossip_kv/deployment/aws/terraform/infra/outputs.tf @@ -0,0 +1,28 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane" + value = module.eks_cluster.cluster_endpoint +} + +output "cluster_name" { + description = "Kubernetes Cluster Name" + value = module.eks_cluster.cluster_name +} + +output "vpc_id" { + value = module.vpc.vpc_id +} + +output "region" { + description = "AWS region" + value = var.region +} + +output "repository_urls" { + description = "URLs of all ECR repositories created" + value = { for repo, details in module.ecr : repo => details.repository_url } +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded PEM certificate data for the cluster" + value = module.eks_cluster.cluster_certificate_authority_data +} \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/aws/terraform/infra/variables.tf b/datastores/gossip_kv/deployment/aws/terraform/infra/variables.tf new file mode 100644 index 000000000000..7522b20fcc27 --- /dev/null +++ b/datastores/gossip_kv/deployment/aws/terraform/infra/variables.tf @@ -0,0 +1,17 @@ +variable "region" { + description = "AWS region" + type = string + default = "us-east-2" +} + +variable "instance_type" { + description = "Instance type for the EKS nodes" + type = string + default = "t3.small" +} + +variable "ecr_repositories" { + description = "List of ECR repository names" + type = list(string) + default = ["gossip_kv_server", "gossip_kv_cli", "gossip_kv_load_test"] +} \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/local/Makefile b/datastores/gossip_kv/deployment/local/Makefile new file mode 100644 index 000000000000..ee030a22207a --- /dev/null +++ b/datastores/gossip_kv/deployment/local/Makefile @@ -0,0 +1,46 @@ + +INFRA_PATH=./terraform/infra +APPLICATION_PATH=./terraform/application + +BASE_IMAGE_VERSION:=latest +SERVER_IMAGE_VERSION:=latest +CLI_IMAGE_VERSION:=latest +LOAD_TEST_IMAGE_VERSION:=latest + +# Docker Image Tags +BASE_IMAGE_TAG:=hydroflow-gossip-kv-base-image:$(BASE_IMAGE_VERSION) +SERVER_IMAGE_TAG:=hydroflow-gossip-kv-server:$(SERVER_IMAGE_VERSION) +CLI_IMAGE_TAG:=hydroflow-gossip-kv-cli:$(CLI_IMAGE_VERSION) +LOAD_TEST_IMAGE_TAG:=hydroflow-gossip-kv-load-test:$(LOAD_TEST_IMAGE_VERSION) + +.PHONY : init infra docker_images base_image server_image cli_image application config clean + +init: + terraform -chdir="$(INFRA_PATH)" init + terraform -chdir="$(APPLICATION_PATH)" init + +infra: + terraform -chdir="$(INFRA_PATH)" apply -auto-approve + +docker_images: base_image server_image cli_image + +base_image: + docker build -t "$(BASE_IMAGE_TAG)" -f ../../../../datastores/gossip_kv/server/baseimage.Dockerfile ../../../.. + +server_image: + docker build -t "$(SERVER_IMAGE_TAG)" -f ../../../../datastores/gossip_kv/server/Dockerfile ../../../.. + +cli_image: + docker build -t "$(CLI_IMAGE_TAG)" -f ../../../../datastores/gossip_kv/cli/Dockerfile ../../../.. + +application: + terraform -chdir="$(APPLICATION_PATH)" apply -auto-approve + +config: + kubectl apply -f seed_node_config.yaml + +clean: + terraform -chdir="$(APPLICATION_PATH)" destroy -auto-approve + terraform -chdir="$(INFRA_PATH)" destroy -auto-approve + rm -rf $(INFRA_PATH)/.terraform $(INFRA_PATH)/terraform.tfstate $(INFRA_PATH)/terraform.tfstate.backup + rm -rf $(APPLICATION_PATH)/.terraform $(APPLICATION_PATH)/terraform.tfstate $(APPLICATION_PATH)/terraform.tfstate.backup diff --git a/datastores/gossip_kv/deployment/local/README.md b/datastores/gossip_kv/deployment/local/README.md new file mode 100644 index 000000000000..e45c622f9280 --- /dev/null +++ b/datastores/gossip_kv/deployment/local/README.md @@ -0,0 +1,109 @@ +# Local (Minikube) Deployment + +## Pre-requisites +- [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) +- [Minikube](https://minikube.sigs.k8s.io/docs/start/) +- [Docker](https://docs.docker.com/get-docker/) +- [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +- [Make](https://www.gnu.org/software/make/) +- [k9s](https://k9scli.io/) + +### Mac Setup Using Homebrew +```shell +brew install terraform +brew install minikube +brew install docker +brew install kubernetes-cli +brew install k9s +``` + +## Initialize Setup +Initializes terraform providers required for local deployment. +```shell +make init +``` + +## Create Infrastructure +Creates a local kubernetes cluster using minikube. + +```shell +make infra +``` +### List Minikube Profiles +The `make infra` command creates minikube profile 'terraform-provider-minikube'. The details of the profile can be +listed using the following command. +```shell +minikube profile list +``` +Other minikube commands can be specified using the profile name as shown below. +```shell +minikube --profile=terraform-provider-minikube +``` + +### Kubectl +The kubectl context is set to the minikube profile 'terraform-provider-minikube' by default. + +To switch back to the 'terraform-provider-minikube' context at anytime, use the following command. +```shell +kubectl config use-context terraform-provider-minikube +``` + +### K9s +To monitor the cluster using k9s, use the following command. + +```shell +k9s +``` + +If the current context is not set to `terraform-provider-minikube`, use the following command. +```shell +k9s --context terraform-provider-minikube +``` + +### Build Docker Image +Setup the docker environment to use the minikube docker registry. +```shell +eval $(minikube -p terraform-provider-minikube docker-env) +``` + +Build the docker image for the application, into the local minikube docker registry. +```shell +make docker_images +``` +You can view the docker images in the minikube registry using the following command. +```shell +docker images +``` + +If docker isn't pointing to the minikube registry, use the following command. +```shell +minikube -p terraform-provider-minikube ssh docker images +``` + +## Create Application +```shell +make application +``` + +### Check if the application is running +```shell +kubectl get pods # All gossip-kv-* pods should show status as "Running" +```` + +### Update seed node configuration +```shell +make config +``` + +## Clean + +Destroys the mini-kube cluster and removes the terraform state files. + +```shell +make clean +``` + +If terraform is an inconsistent state, use the following command to blow away the minikube cluster and start fresh. +```shell +minikube delete -p terraform-provider-minikube +``` \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/local/objects.yaml b/datastores/gossip_kv/deployment/local/objects.yaml deleted file mode 100644 index 7421df268e1e..000000000000 --- a/datastores/gossip_kv/deployment/local/objects.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: gossip-kv-seed-nodes - labels: - app: gossip-kv-seed-nodes -spec: - replicas: 3 - serviceName: gossip-kv-seed-nodes - selector: - matchLabels: - app: gossip-kv-seed-nodes - template: - metadata: - labels: - app: gossip-kv-seed-nodes - spec: - terminationGracePeriodSeconds: 5 # Really aggressive, but makes teardown faster. Not recommended beyond benchmarking. - containers: - - name: gossip-kv-server - image: docker.io/hydroflow/gossip-kv-server:latest - imagePullPolicy: IfNotPresent -# Uncomment the following for debugging -# command: [ "/bin/sh" ] -# args: [ "-c", "while true; do sleep 3600; done" ] - env: - - name: RUST_LOG - value: "trace" - - name: RUST_BACKTRACE - value: "full" - ports: - - containerPort: 3001 - protocol: UDP - volumeMounts: - - name: gossip-kv-dynamic-config - mountPath: /config/dynamic - volumes: - - name: gossip-kv-dynamic-config - configMap: - name: gossip-kv-dynamic-config ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: gossip-kv-cli - labels: - app: gossip-kv-cli -spec: - replicas: 1 - selector: - matchLabels: - app: gossip-kv-cli - template: - metadata: - labels: - app: gossip-kv-cli - spec: - terminationGracePeriodSeconds: 5 # Really aggressive, but makes teardown faster. Not recommended beyond benchmarking. - containers: - - name: gossip-kv-cli - image: docker.io/hydroflow/gossip-kv-cli:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh"] - args: ["-c", "while true; do sleep 3600; done"] - tty: true - env: - - name: RUST_LOG - value: "info" ---- -apiVersion: v1 -kind: Service -metadata: - name: gossip-kv-seed-nodes - labels: - app: gossip-kv-seed-nodes -spec: - ports: - - port: 3001 - targetPort: 3001 - protocol: UDP - clusterIP: None - selector: - app: gossip-kv-seed-nodes ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: gossip-kv-dynamic-config -data: - dynamic.toml: | \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/local/seed_node_config.yaml b/datastores/gossip_kv/deployment/local/seed_node_config.yaml new file mode 100644 index 000000000000..bebeda8cb5e4 --- /dev/null +++ b/datastores/gossip_kv/deployment/local/seed_node_config.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gossip-kv-dynamic-config +data: + dynamic.toml: | + [[seed_nodes]] + id = "gossip-kv-seed-nodes-0" + address = "gossip-kv-seed-nodes-0.gossip-kv-seed-nodes.default.svc.cluster.local:3000" + + [[seed_nodes]] + id = "gossip-kv-seed-nodes-1" + address = "gossip-kv-seed-nodes-1.gossip-kv-seed-nodes.default.svc.cluster.local:3000" + + [[seed_nodes]] + id = "gossip-kv-seed-nodes-2" + address = "gossip-kv-seed-nodes-2.gossip-kv-seed-nodes.default.svc.cluster.local:3000" \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/local/terraform/.gitignore b/datastores/gossip_kv/deployment/local/terraform/.gitignore new file mode 100644 index 000000000000..e63962a88795 --- /dev/null +++ b/datastores/gossip_kv/deployment/local/terraform/.gitignore @@ -0,0 +1,30 @@ +### TERRAFORM IGNORES ### +# Terraform Plugin and module directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +*.tfvars +*.tfvars.json + +# Ignore local override files +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Terraform CLI configuration files +.terraformrc +terraform.rc + +# Backup files created by Terraform during state edits +*.backup + +# Terraform plan files (these are generated and can be large) +*.tfplan diff --git a/datastores/gossip_kv/deployment/local/terraform/application/.terraform.lock.hcl b/datastores/gossip_kv/deployment/local/terraform/application/.terraform.lock.hcl new file mode 100644 index 000000000000..0bc31d25bae2 --- /dev/null +++ b/datastores/gossip_kv/deployment/local/terraform/application/.terraform.lock.hcl @@ -0,0 +1,21 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.33.0" + hashes = [ + "h1:HDyytvOlqNw5fJ0SB/nzgqCWniK4LAZNx23LaPavQq8=", + "zh:255b35790b706d405e987750190658dcaefb663741b96803a9529ba5d7435329", + "zh:362feba1aa820a8e02869ec71d1a08e87243dbce43671dc0995fa6c5a2fafa1d", + "zh:39332abcf75b5dd9c78c79c7c0c094f7d4ca908d1b76bbd2aae67e8e3516710c", + "zh:3e8e7f758bb09a9b5b613c8866e77541f8f00b521070cc86bc095ce61f010baf", + "zh:427883b889b9c36630c3eec4d5c07bc4ae12cc0d358fc17ea42a8049bf8d5275", + "zh:69bfc4ed067a5e4844db1a1809343652ff239aa0a8da089b1671524c44e8740a", + "zh:6b9f731062b945c5020e0930ed9a1b1b50afd2caf751f0e70a282d165c970979", + "zh:6faf9ec006af7ee7014a9c3251d65b701792abb823f149b0b7e4ac4433848201", + "zh:b706f76d695104a47682ee6ab842870f9c70a680f979fa9e7efe34278c0831bc", + "zh:b9bca48de2c92f57389ed58dd2fac564deaccd79a92cafd08edeed3ba6b91d4d", + "zh:bbd3336dbee5aed9880f98e36fb8340e0c6d8f0399a05787521af599ccb3dac4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/datastores/gossip_kv/deployment/local/terraform/application/main.tf b/datastores/gossip_kv/deployment/local/terraform/application/main.tf new file mode 100644 index 000000000000..c9e439b31ea0 --- /dev/null +++ b/datastores/gossip_kv/deployment/local/terraform/application/main.tf @@ -0,0 +1,153 @@ +terraform { + backend "local" { + path = "../state/application.tfstate" + } +} + +data "terraform_remote_state" "infra" { + backend = "local" + config = { + path = "../state/infra.tfstate" + } +} + +provider "kubernetes" { + host = data.terraform_remote_state.infra.outputs.kubernetes_host + client_certificate = data.terraform_remote_state.infra.outputs.kubernetes_client_certificate + client_key = data.terraform_remote_state.infra.outputs.kubernetes_client_key + cluster_ca_certificate = data.terraform_remote_state.infra.outputs.kubernetes_cluster_ca_certificate +} + +resource "kubernetes_stateful_set" "gossip_kv_seed_nodes" { + metadata { + name = "gossip-kv-seed-nodes" + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + replicas = 3 + service_name = "gossip-kv-seed-nodes" + + selector { + match_labels = { + app = "gossip-kv-seed-nodes" + } + } + + template { + metadata { + labels = { + app = "gossip-kv-seed-nodes" + } + } + spec { + termination_grace_period_seconds = 5 # Allows for quick restarts. Not recommended for production. + + container { + name = "gossip-kv-server" + image = "docker.io/hydroflow-gossip-kv-server:latest" + image_pull_policy = "IfNotPresent" + env { + name = "RUST_LOG" + value = "trace" + } + env { + name = "RUST_BACKTRACE" + value = "full" + } + port { + container_port = 3001 + protocol = "UDP" + } + volume_mount { + name = "gossip-kv-dynamic-config" + mount_path = "/config/dynamic" + } + } + + volume { + name = "gossip-kv-dynamic-config" + config_map { + name = "gossip-kv-dynamic-config" + } + } + } + } + } +} + +resource "kubernetes_deployment" "gossip_kv_cli" { + metadata { + name = "gossip-kv-cli" + labels = { + app = "gossip-kv-cli" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "gossip-kv-cli" + } + } + + template { + metadata { + labels = { + app = "gossip-kv-cli" + } + } + spec { + termination_grace_period_seconds = 5 + container { + name = "gossip-kv-cli" + image = "docker.io/hydroflow-gossip-kv-cli:latest" + image_pull_policy = "IfNotPresent" + command = ["/bin/sh"] + args = ["-c", "while true; do sleep 3600; done"] + tty = true + env { + name = "RUST_LOG" + value = "info" + } + } + } + } + } +} + +resource "kubernetes_service" "gossip_kv_seed_nodes" { + metadata { + name = "gossip-kv-seed-nodes" + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + port { + port = 3001 + target_port = 3001 + protocol = "UDP" + } + cluster_ip = "None" + selector = { + app = "gossip-kv-seed-nodes" + } + } +} + + +resource "kubernetes_config_map" "gossip_kv_dynamic_config" { + metadata { + name = "gossip-kv-dynamic-config" + } + + data = { + "dynamic.toml" = "" + } +} \ No newline at end of file diff --git a/datastores/gossip_kv/deployment/local/terraform/infra/.terraform.lock.hcl b/datastores/gossip_kv/deployment/local/terraform/infra/.terraform.lock.hcl new file mode 100644 index 000000000000..fc26421f157d --- /dev/null +++ b/datastores/gossip_kv/deployment/local/terraform/infra/.terraform.lock.hcl @@ -0,0 +1,40 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.33.0" + hashes = [ + "h1:HDyytvOlqNw5fJ0SB/nzgqCWniK4LAZNx23LaPavQq8=", + "zh:255b35790b706d405e987750190658dcaefb663741b96803a9529ba5d7435329", + "zh:362feba1aa820a8e02869ec71d1a08e87243dbce43671dc0995fa6c5a2fafa1d", + "zh:39332abcf75b5dd9c78c79c7c0c094f7d4ca908d1b76bbd2aae67e8e3516710c", + "zh:3e8e7f758bb09a9b5b613c8866e77541f8f00b521070cc86bc095ce61f010baf", + "zh:427883b889b9c36630c3eec4d5c07bc4ae12cc0d358fc17ea42a8049bf8d5275", + "zh:69bfc4ed067a5e4844db1a1809343652ff239aa0a8da089b1671524c44e8740a", + "zh:6b9f731062b945c5020e0930ed9a1b1b50afd2caf751f0e70a282d165c970979", + "zh:6faf9ec006af7ee7014a9c3251d65b701792abb823f149b0b7e4ac4433848201", + "zh:b706f76d695104a47682ee6ab842870f9c70a680f979fa9e7efe34278c0831bc", + "zh:b9bca48de2c92f57389ed58dd2fac564deaccd79a92cafd08edeed3ba6b91d4d", + "zh:bbd3336dbee5aed9880f98e36fb8340e0c6d8f0399a05787521af599ccb3dac4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/scott-the-programmer/minikube" { + version = "0.4.2" + constraints = "0.4.2" + hashes = [ + "h1:kUXMe9QKg/SS9WI36FRVK31z3gj59DaljIJKO0cHH68=", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:74f33b5a5ca42defdf44a2e6c4e08336a353c514f6c37f2ff41b5471324f5b03", + "zh:7f2fa2bcc6bde40098647fb171c0fd952a4d9146ab3df113f16b50565359b4fb", + "zh:88e01b8a9615d2c9ee8b0b34398ed5b00617ba4581edb2ca50a18608a9642072", + "zh:89384deab32270c985f8d0f0e762b483ec37339900ef600156b89ea10f45c77b", + "zh:9164e86a0200cf2189ed939a554d1156a1acff67c3f79dfc296d1b9226d1e814", + "zh:ab0cb857884de38dcce31b66b112f5a886ef5a85abbaa6e96af3c23f312947af", + "zh:bd4fd61701bbc4b6ede08612c7e944813e0948887d666fa0ff64186e97e93c22", + "zh:cf48a09a4c92d77f373ea359415087891788ed11a2537000c79600c05524bd26", + "zh:fedc68ccc3f1b1447f9ed97e486f30ed526f9e098854dc18f9fc9b26f2f5ec43", + "zh:ff97171d251fb6658c729676c62b2315bcc3b056115992dbb19737e6ba550f0d", + ] +} diff --git a/datastores/gossip_kv/deployment/local/terraform/infra/main.tf b/datastores/gossip_kv/deployment/local/terraform/infra/main.tf new file mode 100644 index 000000000000..5262f27c06f7 --- /dev/null +++ b/datastores/gossip_kv/deployment/local/terraform/infra/main.tf @@ -0,0 +1,26 @@ +terraform { + backend "local" { + path = "../state/infra.tfstate" + } + + required_providers { + minikube = { + source = "scott-the-programmer/minikube" + version = "0.4.2" + } + } +} + +resource "minikube_cluster" "gossip_kv" { + driver = "docker" + cpus = 16 + memory = 32768 + disk_size = "100g" +} + +provider "kubernetes" { + host = minikube_cluster.gossip_kv.host + client_certificate = minikube_cluster.gossip_kv.client_certificate + client_key = minikube_cluster.gossip_kv.client_key + cluster_ca_certificate = minikube_cluster.gossip_kv.cluster_ca_certificate +} diff --git a/datastores/gossip_kv/deployment/local/terraform/infra/outputs.tf b/datastores/gossip_kv/deployment/local/terraform/infra/outputs.tf new file mode 100644 index 000000000000..07550ca955eb --- /dev/null +++ b/datastores/gossip_kv/deployment/local/terraform/infra/outputs.tf @@ -0,0 +1,18 @@ +output "kubernetes_host" { + value = minikube_cluster.gossip_kv.host +} + +output "kubernetes_client_certificate" { + value = minikube_cluster.gossip_kv.client_certificate + sensitive = true +} + +output "kubernetes_client_key" { + value = minikube_cluster.gossip_kv.client_key + sensitive = true +} + +output "kubernetes_cluster_ca_certificate" { + value = minikube_cluster.gossip_kv.cluster_ca_certificate + sensitive = true +} \ No newline at end of file diff --git a/datastores/gossip_kv/server/README.md b/datastores/gossip_kv/server/README.md deleted file mode 100644 index 434315e29f4f..000000000000 --- a/datastores/gossip_kv/server/README.md +++ /dev/null @@ -1,44 +0,0 @@ -From the `hydroflow` directory, run - -## Minikube - -### Start Minikube -Disk allocation is done by the driver used to create the VM. Setting this to a high value will do nothing if the -driver isn't correctly configured. You'll only notice that hydroflow runs out of disk space while compiling. -For Docker, the disk size is set in the Docker Desktop settings. Also, provide as many CPUs here as possible, since -building the code is CPU-intensive. -```shell -minikube start --disk-size=100g --cpus=16 --memory=32768 -``` - -### Use the Docker daemon from minikube -```shell -eval $(minikube docker-env) -``` - -## Build Docker Base Image -Speeds up code changes by caching build dependencies. -```shell -docker build -t "hydroflow/gossip-kv-server-base-image:latest" -f datastores/gossip_kv/server/baseimage.Dockerfile . -``` - -## Build Docker Image for Gossip Server -```shell -docker build -t "hydroflow/gossip-kv-server:latest" -f datastores/gossip_kv/server/Dockerfile . -``` - -## Build Docker Image for Gossip CLI -```shell -docker build -t "hydroflow/gossip-kv-cli:latest" -f datastores/gossip_kv/cli/Dockerfile . -``` - -## Check if minikube has the image -You should see "hydroflow/gossip-kv" -```shell -minikube image ls --format tablemin -``` - -## Deploy to Minikube -```shell -kubectl apply -f datastores/gossip_kv/server/deployment/local/objects.yaml -``` \ No newline at end of file