Merge branch 'master' into 562-evict-metrics-from-redis-after-120-days

seqeralabs · Sep 24, 2024 · 0336b1b · 0336b1b
2 parents 8e963c8 + 2f5a1fe
commit 0336b1b
Show file tree

Hide file tree

Showing 216 changed files with 8,498 additions and 3,620 deletions.
diff --git a/README.md b/README.md
@@ -14,10 +14,9 @@ images.
 * Augment container images i.e. dynamically add one or more container layers to existing images;
 * Build container images on-demand for a given container file (aka Dockerfile);
 * Build container images on-demand based on one or more Conda packages;
-* Build container images on-demand based on one or more Spack packages, Spack support will be removed in future releases;
 * Build container images for a specified target platform (currently linux/amd64 and linux/arm64);
 * Push and cache built containers to a user-provided container repository;
-* Build Singularity native containers both using a Singularity spec file, Conda package(s) and Spack package(s);
+* Build Singularity native containers both using a Singularity spec file, Conda package(s);
 * Push Singularity native container images to OCI-compliant registries;
 
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.11.0
+1.12.3
diff --git a/build.gradle b/build.gradle
@@ -34,7 +34,7 @@ dependencies {
     compileOnly("io.micronaut:micronaut-http-validation")
     implementation("jakarta.persistence:jakarta.persistence-api:3.0.0")
     api 'io.seqera:lib-mail:1.0.0'
-    api 'io.seqera:wave-api:0.10.0'
+    api 'io.seqera:wave-api:0.12.0'
     api 'io.seqera:wave-utils:0.13.1'
 
     implementation("io.micronaut:micronaut-http-client")
@@ -70,6 +70,7 @@ dependencies {
     implementation "software.amazon.awssdk:ecrpublic"
     implementation 'software.amazon.awssdk:ses'
     implementation 'org.yaml:snakeyaml:2.0'
+    implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8'
     //object storage dependency
     implementation("io.micronaut.objectstorage:micronaut-object-storage-aws")
     // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570

diff --git a/changelog.txt b/changelog.txt
@@ -1,4 +1,57 @@
 # Wave changelog
+1.12.3 - 22 Sep 2024
+- Fix build status completion of submit exception [3c3af360]
+- Fix singularity build mounts [3b338b29]
+
+1.12.2 - 18 Sep 2024 
+- Fix Remove entries permanently from stream once consumed [adfad9d6]
+- Refactor container build service [1a858c12]
+- Remove unused code [268c76ea]
+
+1.12.1 - 17 Sep 2024 
+- Fix stream check for new messages [16a7e256]
+
+1.12.0 - 17 Sep 2024 
+- Add Job manager (#605) [00daf919]
+  - Add support for build and scan operation via Job manager (#620) [0e5e5ca4]
+  - Do not retry on build failure (#632) [e6568d1e]
+  - Fix Blob cache failure duration (#643) [ebf65adc]
+  - Fix K8s job status detection (#630) [d5b45d8d] [7a9046ed] [e26811dd]
+  - Fix Retry policy delay multiplier (#629) [80037565]
+  - Improve blob cache info (#644) [8b96173a]
+  - Improve blob cache logging [e4c75671]
+  - Improve blob cache reliability (#596) [dfb64bad]
+  - Improve build & scan logging [b086f3d8]
+  - Improve job dispatcher (#645) [fee3db9d]
+  - Remove unneeded timeout logic (#633) [5eabf285]
+  - Deferred resources cleanup (#636) [c6b3e9b8]
+  - Change k8s Job deletion to foreground pods propagation (#595) [b5baea03]
+  - Run Docker process in background (#647) [1fcb4c94]
+- Add build in progress status in build page (#607) [3d940e88]
+- Add container image name to container scan view  (#635) [7858b95f]
+- Add entropy to cron services (#640) [a6d1d884]
+- Add link to build Id in container request view [57129960]
+- Add rate limiter to container request [a05c1094]
+- Add trusted builds timeout (#600) [63b58088]
+- Add /v1alpha2/container/{containerId} endpoint (#609) [6c05498c]
+- Add /v1alpha2/container/{token} in typespec (#618) [5cbd67a8]
+- Fix failing type checks [bd704bea]
+- Fix too many requests error code (#610) [ec43fa0d]
+- Increase blob cache timeout to 10m and decrease status to 1h [cf4b7588]
+- Improve container view page (#615) [d9b8cab8]
+- Improve registry auth error handling (#628) [c9185730]
+- Increase cache-tower-client to 1min (#641) [df32b305]
+- Message queue name refactoring [861d0580]
+- Simplify tests (#627) [cf53cba2]
+- Update dev default logs [6b588f4c]
+- Update nextflow.mdx (#612) [fe9b4273]
+- Update scan model (#637) [94d37637]
+- Use public repo for s5cmd (#639) [c16c0959]
+- Bump Trivy 0.55 (#638) [b69d34c4]
+
+1.11.1 - 5 Sep 2024 
+- Add rate limiter to container request [a3c63525]
+
 1.11.0 - 23 Aug 2024
 - added /v1alpha2/container/{containerId} (#609) [5221b5a0]
 - Improve contaiener view page (#615) [9e15b455]

diff --git a/configuration.md b/configuration.md
@@ -107,20 +107,6 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.build.force-compression`**: determines whether to force the compression for each cache layers produced by the build process. The default is `false`, enabling compression for more efficient storage. *Optional*.
 
-### Spack configuration for wave build process
-
-**Note**: Spack support will be removed in future releases.
-
-Spack configuration consists of the path of its secret file, the mount path for the secret file in the spack container, and the optional S3 bucket name for the spack binary cache.
-
-**Note**: these configuration are mandatory to support Spack in a wave installation.
-
-- **`wave.build.spack.secretKeyFile`**: the path to the file containing the PGP private key used to [sign Spack packages built by Wave](https://spack.readthedocs.io/en/latest/binary_caches.html#build-cache-signing). For example, `/efs/wave/spack/key`. *Mandatory*.
-
-- **`wave.build.spack.secretMountPath`**: sets the mount path inside the Spack Docker image for the PGP private key specified by `wave.build.spack.secretKeyFile`. For instance `/var/seqera/spack/key`. Indicating where the PGP private key should be mounted inside the Spack Docker image. *Mandatory*.
-
-- **`wave.build.spack.cacheBucket`**: specifies the S3 bucket for the Spack binary cache, for example, `s3://spack-binarycache`. *Optional*.
-
 ### Build process logs configuration
 
 This configuration specifies attributes for the persistence of the logs fetched from containers or k8s pods used for building requested images, which can be accessed later and also attached to the build completion email.
@@ -216,7 +202,7 @@ Wave offers a feature to provide a cache for Docker blobs, which improves the pe
 
 - **`wave.blobCache.enabled`**: whether to enable the blob cache. It is `false` by default. *Optional*.
 
-- **`wave.blobCache.s5cmdImage`**: the Docker image that supplies the [s5cmd tool](https://github.com/peak/s5cmd). This tool is used to upload blob binaries to the S3 bucket. The default image used by Wave is `cr.seqera.io/public/wave/s5cmd:v2.2.2`. *Optional*.
+- **`wave.blobCache.s5cmdImage`**: the Docker image that supplies the [s5cmd tool](https://github.com/peak/s5cmd). This tool is used to upload blob binaries to the S3 bucket. The default image used by Wave is `public.cr.seqera.io/wave/s5cmd:v2.2.2`. *Optional*.
 
 - **`wave.blobCache.status.delay`**: the time delay in checking the status of the transfer of the blob binary from the repository to the cache. Its default value is `5s`. *Optional*.
 

diff --git a/docs/api.mdx b/docs/api.mdx
@@ -52,7 +52,6 @@ This API endpoint is deprecated in current versions of Wave.
         ]
     },
     condaFile: string,
-    spackFile: string,
     containerPlatform: string,
     buildRepository: string,
     cacheRepository: string,
@@ -81,7 +80,6 @@ This API endpoint is deprecated in current versions of Wave.
 | `containerConfig.layers.gzipSize`   | The size in bytes of the the provided layer tar gzip file.                                                                                         |
 | `containerFile`                     | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted.        |
 | `condaFile`                         | Conda environment file encoded as base64 string.                                                                                                   |
-| `spackFile`                         | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases.                                                                                                        |
 | `containerPlatform`                 | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64.                     |
 | `buildRepository`                   | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional).                                            |
 | `cacheRepository`                   | Container repository used to cache build layers `docker.io/user/my-cache` (optional).                                                              |
@@ -136,7 +134,6 @@ The endpoint returns the name of the container request made available by Wave.
         ]
     },
     condaFile: string,
-    spackFile: string,
     containerPlatform: string,
     buildRepository: string,
     cacheRepository: string,
@@ -157,10 +154,6 @@ The endpoint returns the name of the container request made available by Wave.
                                 commands: string[],
                                 basePackages: string
                               }
-                  spackOpts:{
-                                commands: string[],
-                                basePackages: string
-                            }
 
               },
     nameStrategy: string
@@ -182,7 +175,6 @@ The endpoint returns the name of the container request made available by Wave.
 | `containerConfig.layers.gzipSize`   | The size in bytes of the the provided layer tar gzip file.                                                                                         |
 | `containerFile`                     | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted.        |
 | `condaFile`                         | Conda environment file encoded as base64 string.                                                                                                   |
-| `spackFile`                         | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases.                                                                                                           |
 | `containerPlatform`                 | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64.                     |
 | `buildRepository`                   | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional).                                            |
 | `cacheRepository`                   | Container repository used to cache build layers `docker.io/user/my-cache` (optional).                                                              |

diff --git a/docs/get-started.mdx b/docs/get-started.mdx
@@ -18,8 +18,8 @@ In this guide, you'll request a containerized Conda package from Seqera Containe
 ### Request a Conda package as a Seqera Container
 
 1. Open [Seqera Containers][sc] in a browser.
-1. In the search box, enter `faker`.
-1. In the search results, select **Add** in the `conda-forge::faker` result, and then **Get Container** to initiate the container build.
+1. In the search box, enter `samtools`.
+1. In the search results, select **Add** in the `bioconda::samtools` result, and then **Get Container** to initiate the container build.
 1. From the **Fetching container** modal, copy the the durable container image URI that Seqera Containers provides.
 1. Optional: Select **View build details** to watch Seqera Containers build the requested container in real time.
 
@@ -39,25 +39,25 @@ Nextflow can use the container that Seqera Containers built in the previous sect
 1. Create a `main.nf` file with the following contents:
 
     ```groovy
-    process FAKER {
+    process SAMTOOLS {
       container '<container_uri>'
       debug true
-
       """
-      faker address
+      samtools --version-only
       """
     }
-
     workflow {
-      FAKER()
+      SAMTOOLS()
     }
     ```
 
-    Substitute `<container_uri>` for the container URI that you received from Seqera Containers in the previous section.
+    Substitute `<container_uri>` for the container URI that you received from Seqera Containers in the previous section. e.g.
+    - `community.wave.seqera.io/library/samtools:1.20--b5dfbd93de237464` for linux/amd64.
+    - `community.wave.seqera.io/library/samtools:1.20--497854c5df637867` for linux/arm64.
 
 ### Run the Nextflow pipeline
 
-To confirm that the `faker` command is available from your pipeline, run the following command:
+To confirm that the `samtools` command is available from your pipeline, run the following command:
 
 ```
 nextflow run main.nf
@@ -66,12 +66,14 @@ nextflow run main.nf
 The output from a successful execution is displayed in the following example:
 
 ```
-Launching `main.nf` [jolly_edison] DSL2 - revision: 5c414bd927
+ N E X T F L O W   ~  version 24.04.4
+
+Launching `samtools.nf` [furious_carlsson] DSL2 - revision: 04817f962f
 
 executor >  local (1)
-[86/0d56e8] faker | 1 of 1 ✔
-234 Nicholas Circle
-Masonport, MS 98018
+[2f/d2ccc7] process > SAMTOOLS [100%] 1 of 1 ✔
+1.20+htslib-1.20
+
 ```
 ## Nextflow
 

diff --git a/s5cmd/Makefile b/s5cmd/Makefile
@@ -5,5 +5,5 @@ build:
 			--push \
 			--platform linux/amd64,linux/arm64 \
 			--build-arg version=${version} \
-			--tag cr.seqera.io/public/wave/s5cmd:v${version} \
+			--tag public.cr.seqera.io/wave/s5cmd:v${version} \
 			.
diff --git a/s5cmd/dist.sh b/s5cmd/dist.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+#  Wave, containers provisioning service
+#  Copyright (c) 2023-2024, Seqera Labs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+arch=$(uname -m)
+
+case $arch in
+    x86_64|amd64)
+        echo "https://github.com/peak/s5cmd/releases/download/v$1/s5cmd_$1_Linux-64bit.tar.gz"
+        ;;
+    aarch64|arm64)
+	echo "https://github.com/peak/s5cmd/releases/download/v$1/s5cmd_$1_Linux-arm64.tar.gz"
+        ;;
+    *)
+        echo "Unknown architecture: $arch"
+        ;;
+esac
diff --git a/src/main/groovy/io/seqera/wave/ErrorHandler.groovy b/src/main/groovy/io/seqera/wave/ErrorHandler.groovy
@@ -18,8 +18,6 @@
 
 package io.seqera.wave
 
-import java.util.function.BiFunction
-
 import groovy.util.logging.Slf4j
 import io.micronaut.context.annotation.Value
 import io.micronaut.http.HttpRequest
@@ -32,6 +30,7 @@ import io.seqera.wave.exception.DockerRegistryException
 import io.seqera.wave.exception.ForbiddenException
 import io.seqera.wave.exception.HttpResponseException
 import io.seqera.wave.exception.NotFoundException
+import io.seqera.wave.exception.RegistryForwardException
 import io.seqera.wave.exception.SlowDownException
 import io.seqera.wave.exception.UnauthorizedException
 import io.seqera.wave.exception.WaveException
@@ -46,10 +45,14 @@ import jakarta.inject.Singleton
 @Singleton
 class ErrorHandler {
 
+    static interface Mapper<T> {
+        T apply(String message, String errorCode)
+    }
+
     @Value('${wave.debug:false}')
     private Boolean debug
 
-    def <T> HttpResponse<T> handle(HttpRequest httpRequest, Throwable t, BiFunction<String,String,T> responseFactory) {
+    def <T> HttpResponse<T> handle(HttpRequest httpRequest, Throwable t, Mapper<T> responseFactory) {
         final errId = LongRndKey.rndHex()
         final request = httpRequest?.toString()
         def msg = t.message
@@ -78,6 +81,14 @@ class ErrorHandler {
             log.error(render, t)
         }
 
+        if( t instanceof RegistryForwardException ) {
+            // report this error as it has been returned by the target registry
+            return HttpResponse
+                    .status(HttpStatus.valueOf(t.statusCode))
+                    .body(t.response)
+                    .headers(t.headers)
+        }
+
         if( t instanceof DockerRegistryException ) {
             final resp = responseFactory.apply(msg, t.error)
             return HttpResponseFactory.INSTANCE.status(t.statusCode).body(resp)

diff --git a/src/main/groovy/io/seqera/wave/WaveDefault.groovy b/src/main/groovy/io/seqera/wave/WaveDefault.groovy
@@ -17,11 +17,14 @@
  */
 
 package io.seqera.wave
+
+import groovy.transform.CompileStatic
 /**
  * Wave app defaults
  *
  * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
  */
+@CompileStatic
 interface WaveDefault {
 
     final static public String DOCKER_IO = 'docker.io'
@@ -38,7 +41,7 @@ interface WaveDefault {
                     'application/vnd.docker.distribution.manifest.list.v2+json' ) )
 
 
-    final public static int[] HTTP_REDIRECT_CODES = List.of(301, 302, 303, 307, 308)
+    final public static List<Integer> HTTP_REDIRECT_CODES = List.of(301, 302, 303, 307, 308)
 
     final public static List<Integer> HTTP_SERVER_ERRORS = List.of(500, 502, 503, 504)
 

diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy
@@ -18,6 +18,8 @@
 
 package io.seqera.wave.auth
 
+import io.seqera.wave.exception.RegistryUnauthorizedAccessException
+
 /**
  * Declares container registry authentication & authorization operations
  *
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,6 +18,8 @@ @@
     package io.seqera.wave.auth
+    import io.seqera.wave.exception.RegistryUnauthorizedAccessException
     /**
      * Declares container registry authentication & authorization operations
      *
@@ Expand Down @@