Merge pull request #75 from GSA-TTS/mutliple-manager-instances

Parallel job execution
GSA-TTS · Jan 15, 2025 · ba4b185 · ba4b185
2 parents f5056df + f20137d
commit ba4b185
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 2 deletions.
diff --git a/doc/compliance b/doc/compliance
diff --git a/main.tf b/main.tf
@@ -86,7 +86,7 @@ resource "cloudfoundry_app" "gitlab-runner-manager" {
   path              = data.archive_file.src.output_path
   source_code_hash  = data.archive_file.src.output_base64sha256
   buildpacks        = ["https://github.com/cloudfoundry/apt-buildpack", "binary_buildpack"]
-  instances         = 1
+  instances         = var.manager_instances
   command           = "gitlab-runner run"
   memory            = var.manager_memory
   health_check_type = "process"
@@ -118,6 +118,7 @@ resource "cloudfoundry_app" "gitlab-runner-manager" {
     # https://docs.gitlab.com/runner/faq/#enable-debug-logging-mode
     # and ensuring job logs are removed to avoid leaking secrets.
     RUNNER_DEBUG              = "false"
+    RUNNER_CONCURRENCY        = var.runner_concurrency
     OBJECT_STORE_INSTANCE     = var.object_store_instance
     PROXY_CREDENTIAL_INSTANCE = cloudfoundry_service_instance.egress-proxy-credentials.name
     PROXY_APP_NAME            = var.egress_app_name

diff --git a/runner-manager/.profile b/runner-manager/.profile
@@ -81,6 +81,7 @@ if pgrep 'gitlab-runner' > /dev/null ; then
 else
     echo "Registering GitLab Runner with name $RUNNER_NAME"
     if gitlab-runner register; then
+        sed -e "s/concurrent = 1$/concurrent = $RUNNER_CONCURRENCY/" -i.bak .gitlab-runner/config.toml
         echo "GitLab Runner successfully registered"
     else
         exit_with_failure "GitLab Runner not registered"

diff --git a/sandbox-deploy/main.tf b/sandbox-deploy/main.tf
@@ -30,6 +30,7 @@ module "sandbox-runner" {
   ci_server_token         = var.ci_server_token
   docker_hub_user         = var.docker_hub_user
   docker_hub_token        = var.docker_hub_token
+  manager_instances       = 1
   developer_emails        = var.developer_emails
   worker_disk_size        = var.worker_disk_size
   worker_egress_allowlist = var.worker_egress_allowlist

diff --git a/variables.tf b/variables.tf
@@ -47,6 +47,18 @@ variable "runner_executor" {
   description = "Runner Executer"
 }
 
+variable "manager_instances" {
+  type        = number
+  default     = 2
+  description = "Number of manager instances to run"
+}
+
+variable "runner_concurrency" {
+  type        = number
+  default     = 10
+  description = "The number of parallel jobs a single manager instance will support"
+}
+
 variable "manager_memory" {
   type        = number
   default     = 512