From 1e13c9f8f3401f36f1a0fff32c8c41493bbb009e Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 19 Dec 2024 10:16:28 +0100 Subject: [PATCH 01/28] Added AwsEcrCache Signed-off-by: munishchouhan --- .../wave/service/aws/AwsEcrService.groovy | 32 ++++++----- .../wave/service/aws/cache/AwsEcrCache.groovy | 57 +++++++++++++++++++ .../wave/store/cache/TieredCacheKey.groovy | 30 ++++++++++ 3 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy create mode 100644 src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 3db479f4b..083aaaaf3 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -25,10 +25,13 @@ import java.util.regex.Pattern import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine +import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.service.aws.cache.AwsEcrCache +import io.seqera.wave.store.cache.TieredCacheKey import io.seqera.wave.util.StringUtils import jakarta.annotation.PostConstruct import jakarta.inject.Inject @@ -56,11 +59,22 @@ class AwsEcrService { static final private Pattern AWS_ECR_PUBLIC = ~/public\.ecr\.aws/ @Canonical - private static class AwsCreds { + private static class AwsCreds implements TieredCacheKey { String accessKey String secretKey String region boolean ecrPublic + + @Override + String stableHash() { + final h = Hashing.sipHash24().newHasher() + for( Object it : [accessKey, secretKey, region, ecrPublic] ) { + if( it!=null ) + h.putUnencodedChars(it.toString()) + h.putUnencodedChars('/') + } + return h.hash() + } } @Canonical @@ -82,18 +96,8 @@ class AwsEcrService { @Named(TaskExecutors.BLOCKING) private ExecutorService ioExecutor - // FIXME https://github.com/seqeralabs/wave/issues/747 - private AsyncLoadingCache cache - - @PostConstruct - private void init() { - cache = Caffeine - .newBuilder() - .maximumSize(10_000) - .expireAfterWrite(3, TimeUnit.HOURS) - .executor(ioExecutor) - .buildAsync(loader) - } + @Inject + AwsEcrCache cache private EcrClient ecrClient(String accessKey, String secretKey, String region) { EcrClient.builder() @@ -142,7 +146,7 @@ class AwsEcrService { // get the token from the cache, if missing the it's automatically // fetch using the AWS ECR client // FIXME https://github.com/seqeralabs/wave/issues/747 - return cache.synchronous().get(new AwsCreds(accessKey,secretKey,region,isPublic)) + return cache.get(new AwsCreds(accessKey,secretKey,region,isPublic).stableHash()) } catch (Exception e) { final type = isPublic ? "ECR public" : "ECR" diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy new file mode 100644 index 000000000..9986f8415 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -0,0 +1,57 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.aws.cache + +import java.time.Duration + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable +import io.seqera.wave.store.cache.AbstractTieredCache +import io.seqera.wave.store.cache.L2TieredCache +import io.seqera.wave.tower.client.cache.ClientEncoder +import jakarta.inject.Singleton + +/** + * Implement a tiered cache for AWS ECR client + * + * @author Munish Chouhan + */ +@Slf4j +@Singleton +@CompileStatic +class AwsEcrCache extends AbstractTieredCache { + AwsEcrCache(@Nullable L2TieredCache l2, + @Value('${wave.aws.ecr.cache.duration:3h}') Duration duration, + @Value('${wave.aws.ecr.cache.max-size:10000}') int maxSize) + { + super(l2, ClientEncoder.instance(), duration, maxSize) + } + + @Override + protected getName() { + return 'aws-ecr-cache' + } + + @Override + protected String getPrefix() { + return 'aws-ecr-cache/v1' + } +} diff --git a/src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy b/src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy new file mode 100644 index 000000000..40ea9beaf --- /dev/null +++ b/src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy @@ -0,0 +1,30 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.store.cache + +/** + * Implement a tiered cache for AWS ECR client + * + * @author Munish Chouhan + */ +interface TieredCacheKey { + + String stableHash() + +} From 00b4e5f5bce95edcd471f8447af8c0189388321a Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 24 Dec 2024 22:02:02 +0100 Subject: [PATCH 02/28] fixed tests Signed-off-by: munishchouhan --- .../io/seqera/wave/proxy/ProxyCache.groovy | 3 +- .../wave/service/aws/AwsEcrService.groovy | 29 ++++++------- .../wave/service/aws/cache/Token.groovy | 21 ++++++++++ .../store/cache/AbstractTieredCache.groovy | 23 ++++++++++- .../service/aws/cache/AwsEcrCacheTest.groovy | 41 +++++++++++++++++++ .../cache/AbstractTieredCacheTest.groovy | 2 +- 6 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy index a65c6258c..82d01081e 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy @@ -28,6 +28,7 @@ import io.seqera.wave.encoder.MoshiEncodeStrategy import io.seqera.wave.encoder.MoshiExchange import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache +import io.seqera.wave.store.cache.TieredCacheKey import jakarta.inject.Singleton /** * Implements a tiered cache for proxied http responses @@ -36,7 +37,7 @@ import jakarta.inject.Singleton */ @Singleton @CompileStatic -class ProxyCache extends AbstractTieredCache { +class ProxyCache extends AbstractTieredCache { @Value('${wave.proxy-cache.duration:30m}') private Duration duration diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 083aaaaf3..ed8329eac 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -18,22 +18,20 @@ package io.seqera.wave.service.aws +import java.time.Duration import java.util.concurrent.ExecutorService -import java.util.concurrent.TimeUnit import java.util.regex.Pattern -import com.github.benmanes.caffeine.cache.AsyncLoadingCache -import com.github.benmanes.caffeine.cache.CacheLoader -import com.github.benmanes.caffeine.cache.Caffeine import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.aws.cache.AwsEcrCache +import io.seqera.wave.service.aws.cache.Token import io.seqera.wave.store.cache.TieredCacheKey import io.seqera.wave.util.StringUtils -import jakarta.annotation.PostConstruct import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton @@ -83,13 +81,11 @@ class AwsEcrService { String region } - private CacheLoader loader = new CacheLoader() { - @Override - String load(AwsCreds creds) throws Exception { - return creds.ecrPublic - ? getLoginToken1(creds.accessKey, creds.secretKey, creds.region) - : getLoginToken0(creds.accessKey, creds.secretKey, creds.region) - } + Token load(AwsCreds creds) throws Exception { + def token = creds.ecrPublic + ? getLoginToken1(creds.accessKey, creds.secretKey, creds.region) + : getLoginToken0(creds.accessKey, creds.secretKey, creds.region) + return new Token(token) } @Inject @@ -99,6 +95,9 @@ class AwsEcrService { @Inject AwsEcrCache cache + @Value('${wave.aws.ecr.cache.duration:24h}') + private Duration cacheDuration + private EcrClient ecrClient(String accessKey, String secretKey, String region) { EcrClient.builder() .region( Region.of(region)) @@ -143,10 +142,8 @@ class AwsEcrService { assert region, "Missing AWS region argument" try { - // get the token from the cache, if missing the it's automatically - // fetch using the AWS ECR client - // FIXME https://github.com/seqeralabs/wave/issues/747 - return cache.get(new AwsCreds(accessKey,secretKey,region,isPublic).stableHash()) + final key = new AwsCreds(accessKey,secretKey,region,isPublic) + return cache.getOrCompute(key, (k) -> load(key), cacheDuration).value } catch (Exception e) { final type = isPublic ? "ECR public" : "ECR" diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy new file mode 100644 index 000000000..d5e8cee36 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy @@ -0,0 +1,21 @@ +package io.seqera.wave.service.aws.cache + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import io.seqera.wave.encoder.MoshiExchange +/** + * Implement a tiered cache for AWS ECR client + * + * @author Munish Chouhan + */ +@CompileStatic +@EqualsAndHashCode +@ToString(includePackage = false, includeNames = true) +class Token implements MoshiExchange { + String value + + Token(String value) { + this.value = value + } +} diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index cb5c40e79..70f4b18a8 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -45,7 +45,7 @@ import org.jetbrains.annotations.Nullable */ @Slf4j @CompileStatic -abstract class AbstractTieredCache implements TieredCache { +abstract class AbstractTieredCache implements TieredCache { @Canonical static class Entry implements MoshiExchange { @@ -139,6 +139,27 @@ abstract class AbstractTieredCache implements TieredCac }) } + /** + * Retrieve the value associated with the specified key + * + * @param key + * The key of the value to be retrieved + * @param loader + * A function invoked to load the value the entry with the specified key is not available + * @return + * The value associated with the specified key, or {@code null} otherwise + */ + V getOrCompute(K key, Function loader, Duration ttl) { + final hash = key.stableHash() + if( loader==null ) { + return getOrCompute0(hash, null) + } + return getOrCompute0(hash, (String k)-> { + V v = loader.apply(k) + return v != null ? new Tuple2<>(v, ttl) : null + }) + } + /** * Retrieve the value associated with the specified key * diff --git a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy new file mode 100644 index 000000000..736545d41 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy @@ -0,0 +1,41 @@ +package io.seqera.wave.service.aws.cache + +import spock.lang.Shared +import spock.lang.Specification + +import io.micronaut.context.ApplicationContext +import io.seqera.wave.test.RedisTestContainer + +class AwsEcrCacheTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext applicationContext + + def setup() { + applicationContext = ApplicationContext.run([ + REDIS_HOST: redisHostName, + REDIS_PORT: redisPort + ], 'test', 'redis') + sleep(500) // workaround to wait for Redis connection + } + + def cleanup() { + applicationContext.close() + } + + def 'should cache user info response'() { + given: + def AWAIT = 150 + def store = applicationContext.getBean(RedisL2TieredCache) + def cache1 = new AwsEcrCache(store, Duration.ofMillis(AWAIT), 100) + def cache2 = new AwsEcrCache(store, Duration.ofMillis(AWAIT), 100) + and: + def k = UUID.randomUUID().toString() + def resp = "aaaa" + + when: + cache1.put(k, resp) + then: + cache2.get(k) == resp + } +} diff --git a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy index 0e70c1236..0e6b5a633 100644 --- a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy @@ -48,7 +48,7 @@ class AbstractTieredCacheTest extends Specification implements RedisTestContaine return new MoshiEncodeStrategy(factory) {} } - static class MyCache extends AbstractTieredCache { + static class MyCache extends AbstractTieredCache { static String PREFIX = 'foo/v1' From 3d21816dbce85d5804c524580c18571f5e152e23 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 24 Dec 2024 22:02:11 +0100 Subject: [PATCH 03/28] fixed tests Signed-off-by: munishchouhan --- .../wave/service/aws/cache/AwsEcrCache.groovy | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index 9986f8415..dfbc0b1b4 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -26,9 +26,8 @@ import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache -import io.seqera.wave.tower.client.cache.ClientEncoder +import io.seqera.wave.store.cache.TieredCacheKey import jakarta.inject.Singleton - /** * Implement a tiered cache for AWS ECR client * @@ -37,12 +36,21 @@ import jakarta.inject.Singleton @Slf4j @Singleton @CompileStatic -class AwsEcrCache extends AbstractTieredCache { - AwsEcrCache(@Nullable L2TieredCache l2, - @Value('${wave.aws.ecr.cache.duration:3h}') Duration duration, - @Value('${wave.aws.ecr.cache.max-size:10000}') int maxSize) - { - super(l2, ClientEncoder.instance(), duration, maxSize) +class AwsEcrCache extends AbstractTieredCache { + + @Value('${wave.aws.ecr.cache.duration:3h}') + private Duration duration + + @Value('${wave.aws.ecr.cache.max-size:10000}') + private int maxSize + + AwsEcrCache(@Nullable L2TieredCache l2) { + super(l2, null) + } + + @Override + int getMaxSize() { + return maxSize } @Override @@ -54,4 +62,5 @@ class AwsEcrCache extends AbstractTieredCache { protected String getPrefix() { return 'aws-ecr-cache/v1' } + } From 12f623411422f1c8adc193e3c111bac2d114dbc9 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 26 Dec 2024 10:54:58 +0100 Subject: [PATCH 04/28] fixed license Signed-off-by: munishchouhan --- .../wave/service/aws/cache/AwsEcrCache.groovy | 2 +- .../seqera/wave/service/aws/cache/Token.groovy | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index dfbc0b1b4..37b7c8282 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -1,6 +1,6 @@ /* * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs + * Copyright (c) 2024, Seqera Labs * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy index d5e8cee36..3408153dd 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy @@ -1,3 +1,21 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package io.seqera.wave.service.aws.cache import groovy.transform.CompileStatic From d52e410687d12d394389c63ba1346771d0cd82a1 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 26 Dec 2024 17:21:23 +0100 Subject: [PATCH 05/28] fixed tests Signed-off-by: munishchouhan --- .../wave/service/aws/cache/AwsEcrCache.groovy | 24 ++++++++++++++++++- .../service/aws/cache/AwsEcrCacheTest.groovy | 15 +++++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index 37b7c8282..5c82fd44f 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -20,13 +20,25 @@ package io.seqera.wave.service.aws.cache import java.time.Duration +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.encoder.MoshiExchange import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache import io.seqera.wave.store.cache.TieredCacheKey +import io.seqera.wave.tower.User +import io.seqera.wave.tower.client.CredentialsDescription +import io.seqera.wave.tower.client.GetCredentialsKeysResponse +import io.seqera.wave.tower.client.ListCredentialsResponse +import io.seqera.wave.tower.client.UserInfoResponse +import io.seqera.wave.tower.compute.ComputeEnv +import io.seqera.wave.tower.compute.DescribeWorkflowLaunchResponse +import io.seqera.wave.tower.compute.WorkflowLaunchResponse import jakarta.inject.Singleton /** * Implement a tiered cache for AWS ECR client @@ -45,7 +57,7 @@ class AwsEcrCache extends AbstractTieredCache { private int maxSize AwsEcrCache(@Nullable L2TieredCache l2) { - super(l2, null) + super(l2, encoder()) } @Override @@ -63,4 +75,14 @@ class AwsEcrCache extends AbstractTieredCache { return 'aws-ecr-cache/v1' } + static MoshiEncodeStrategy encoder() { + new MoshiEncodeStrategy(factory()) {} + } + + static JsonAdapter.Factory factory() { + PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) + .withSubtype(Token.class, Token.simpleName) + } + } diff --git a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy index 736545d41..191ee0e57 100644 --- a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy @@ -3,7 +3,10 @@ package io.seqera.wave.service.aws.cache import spock.lang.Shared import spock.lang.Specification +import java.time.Duration + import io.micronaut.context.ApplicationContext +import io.seqera.wave.store.cache.RedisL2TieredCache import io.seqera.wave.test.RedisTestContainer class AwsEcrCacheTest extends Specification implements RedisTestContainer { @@ -23,19 +26,19 @@ class AwsEcrCacheTest extends Specification implements RedisTestContainer { applicationContext.close() } - def 'should cache user info response'() { + def 'should cache ecr token response'() { given: def AWAIT = 150 def store = applicationContext.getBean(RedisL2TieredCache) - def cache1 = new AwsEcrCache(store, Duration.ofMillis(AWAIT), 100) - def cache2 = new AwsEcrCache(store, Duration.ofMillis(AWAIT), 100) + def cache1 = new AwsEcrCache(store) + def cache2 = new AwsEcrCache(store) and: def k = UUID.randomUUID().toString() - def resp = "aaaa" + def token = new Token('token') when: - cache1.put(k, resp) + cache1.put(k, token, Duration.ofSeconds(30)) then: - cache2.get(k) == resp + cache2.get(k) == token } } From 8660e8af17f3ce104110987b122bf84f21858170 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 26 Dec 2024 17:40:47 +0100 Subject: [PATCH 06/28] Added RegistryLookupCache Signed-off-by: munishchouhan --- .../io/seqera/wave/auth/RegistryAuth.groovy | 3 +- .../auth/RegistryLookupServiceImpl.groovy | 51 +++++------- .../auth/cache/RegistryLookupCache.groovy | 81 +++++++++++++++++++ .../wave/service/aws/cache/AwsEcrCache.groovy | 8 -- .../auth/cache/RegistryLookupCacheTest.groovy | 67 +++++++++++++++ .../service/aws/cache/AwsEcrCacheTest.groovy | 24 +++++- 6 files changed, 191 insertions(+), 43 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy create mode 100644 src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy index aa43a4395..6077c5a8c 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy @@ -23,6 +23,7 @@ import java.util.regex.Pattern import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString +import io.seqera.wave.encoder.MoshiExchange /** * Model container registry authentication meta-info @@ -32,7 +33,7 @@ import groovy.transform.ToString @Canonical @CompileStatic @ToString(includePackage = false, includeNames = true) -class RegistryAuth { +class RegistryAuth implements MoshiExchange { private static final Pattern AUTH = ~/(?i)(?.+) realm="(?[^"]+)",service="(?[^"]+)"/ // some registries doesn't send the service diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 13775d4a4..935ef8666 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -20,21 +20,19 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration import java.util.concurrent.CompletionException import java.util.concurrent.ExecutorService -import java.util.concurrent.TimeUnit -import com.github.benmanes.caffeine.cache.AsyncLoadingCache -import com.github.benmanes.caffeine.cache.CacheLoader -import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.auth.cache.RegistryLookupCache import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.http.HttpClientFactory import io.seqera.wave.util.Retryable -import jakarta.annotation.PostConstruct import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton @@ -63,36 +61,26 @@ class RegistryLookupServiceImpl implements RegistryLookupService { @Named(TaskExecutors.BLOCKING) private ExecutorService ioExecutor - private CacheLoader loader = new CacheLoader() { - @Override - RegistryAuth load(URI endpoint) throws Exception { - // check if there's a record in the store cache (redis) - def result = store.get(endpoint.toString()) - if( result ) { - log.debug "Authority lookup for endpoint: '$endpoint' => $result [from store]" - return result - } - // look-up using the corresponding API endpoint - result = lookup0(endpoint) - log.debug "Authority lookup for endpoint: '$endpoint' => $result" - // save it in the store cache (redis) - store.put(endpoint.toString(), result) + RegistryAuth load(URI endpoint) throws Exception { + // check if there's a record in the store cache (redis) + def result = store.get(endpoint.toString()) + if( result ) { + log.debug "Authority lookup for endpoint: '$endpoint' => $result [from store]" return result } + // look-up using the corresponding API endpoint + result = lookup0(endpoint) + log.debug "Authority lookup for endpoint: '$endpoint' => $result" + // save it in the store cache (redis) + store.put(endpoint.toString(), result) + return result } - // FIXME https://github.com/seqeralabs/wave/issues/747 - private AsyncLoadingCache cache + @Inject + private RegistryLookupCache cache - @PostConstruct - void init() { - cache = Caffeine - .newBuilder() - .maximumSize(10_000) - .expireAfterAccess(1, TimeUnit.HOURS) - .executor(ioExecutor) - .buildAsync(loader) - } + @Value('${wave.registry.cache.duration:24h}') + private Duration cacheDuration protected RegistryAuth lookup0(URI endpoint) { final httpClient = HttpClientFactory.followRedirectsHttpClient() @@ -131,8 +119,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService { RegistryInfo lookup(String registry) { try { final endpoint = registryEndpoint(registry) - // FIXME https://github.com/seqeralabs/wave/issues/747 - final auth = cache.synchronous().get(endpoint) + final auth = cache.getOrCompute(endpoint.toString(), (key) -> load(endpoint), cacheDuration) return new RegistryInfo(registry, endpoint, auth) } catch (CompletionException e) { diff --git a/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy b/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy new file mode 100644 index 000000000..c91f0ec43 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy @@ -0,0 +1,81 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth.cache + +import java.time.Duration + +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable +import io.seqera.wave.auth.RegistryAuth +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.store.cache.AbstractTieredCache +import io.seqera.wave.store.cache.L2TieredCache +import io.seqera.wave.store.cache.TieredCacheKey +import jakarta.inject.Singleton +/** + * Implement a tiered cache for Registry lookup + * + * @author Munish Chouhan + */ +@Slf4j +@Singleton +@CompileStatic +class RegistryLookupCache extends AbstractTieredCache { + + @Value('${wave.registry.cache.duration:1h}') + private Duration duration + + @Value('${wave.registry.cache.max-size:10000}') + private int maxSize + + RegistryLookupCache(@Nullable L2TieredCache l2) { + super(l2, encoder()) + } + + @Override + int getMaxSize() { + return maxSize + } + + @Override + protected getName() { + return 'registry-cache' + } + + @Override + protected String getPrefix() { + return 'registry-cache/v1' + } + + static MoshiEncodeStrategy encoder() { + new MoshiEncodeStrategy(factory()) {} + } + + static JsonAdapter.Factory factory() { + PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) + .withSubtype(RegistryAuth.class, RegistryAuth.name) + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index 5c82fd44f..69da38b6c 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -31,14 +31,6 @@ import io.seqera.wave.encoder.MoshiExchange import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache import io.seqera.wave.store.cache.TieredCacheKey -import io.seqera.wave.tower.User -import io.seqera.wave.tower.client.CredentialsDescription -import io.seqera.wave.tower.client.GetCredentialsKeysResponse -import io.seqera.wave.tower.client.ListCredentialsResponse -import io.seqera.wave.tower.client.UserInfoResponse -import io.seqera.wave.tower.compute.ComputeEnv -import io.seqera.wave.tower.compute.DescribeWorkflowLaunchResponse -import io.seqera.wave.tower.compute.WorkflowLaunchResponse import jakarta.inject.Singleton /** * Implement a tiered cache for AWS ECR client diff --git a/src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy b/src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy new file mode 100644 index 000000000..96d31485d --- /dev/null +++ b/src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy @@ -0,0 +1,67 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth.cache + +import spock.lang.Shared +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.context.ApplicationContext +import io.seqera.wave.auth.RegistryAuth +import io.seqera.wave.store.cache.RedisL2TieredCache +import io.seqera.wave.test.RedisTestContainer +/** + * + * @author Munish Chouhan + */ +class RegistryLookupCacheTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext applicationContext + + def setup() { + applicationContext = ApplicationContext.run([ + REDIS_HOST : redisHostName, + REDIS_PORT : redisPort + ], 'test', 'redis') + sleep(500) // workaround to wait for Redis connection + } + + def cleanup() { + applicationContext.close() + } + + def 'should cache registry auth response' () { + given: + def TTL = Duration.ofSeconds(1) + def store = applicationContext.getBean(RedisL2TieredCache) + def cache1 = new RegistryLookupCache(store) + def cache2 = new RegistryLookupCache(store) + and: + def k = UUID.randomUUID().toString() + def resp = new RegistryAuth(URI.create('seqera.io/auth'), 'seqera', RegistryAuth.Type.Basic) + + when: + cache1.put(k, resp, TTL) + then: + cache2.get(k) == resp + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy index 191ee0e57..d06f9fb13 100644 --- a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy @@ -1,3 +1,21 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package io.seqera.wave.service.aws.cache import spock.lang.Shared @@ -8,7 +26,10 @@ import java.time.Duration import io.micronaut.context.ApplicationContext import io.seqera.wave.store.cache.RedisL2TieredCache import io.seqera.wave.test.RedisTestContainer - +/** + * + * @author Munish Chouhan + */ class AwsEcrCacheTest extends Specification implements RedisTestContainer { @Shared @@ -28,7 +49,6 @@ class AwsEcrCacheTest extends Specification implements RedisTestContainer { def 'should cache ecr token response'() { given: - def AWAIT = 150 def store = applicationContext.getBean(RedisL2TieredCache) def cache1 = new AwsEcrCache(store) def cache2 = new AwsEcrCache(store) From c7e8536e64e6a87285a59da59f28b1448683ef47 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 26 Dec 2024 22:39:44 +0100 Subject: [PATCH 07/28] Added RegistryAuthCache Signed-off-by: munishchouhan --- .../wave/auth/RegistryAuthServiceImpl.groovy | 38 ++++----- .../wave/auth/cache/RegistryAuthCache.groovy | 81 +++++++++++++++++++ .../wave/auth/model/RegistryAuthToken.groovy | 35 ++++++++ .../wave/service/aws/AwsEcrService.groovy | 12 +-- .../auth/cache/RegistryAuthCacheTest.groovy | 67 +++++++++++++++ 5 files changed, 201 insertions(+), 32 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy create mode 100644 src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy create mode 100644 src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index a4252c4f3..2caa79d65 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine +import com.google.common.hash.Hashing import groovy.json.JsonSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic @@ -35,10 +36,13 @@ import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.auth.cache.RegistryAuthCache +import io.seqera.wave.auth.model.RegistryAuthToken import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.exception.RegistryUnauthorizedAccessException import io.seqera.wave.http.HttpClientFactory +import io.seqera.wave.store.cache.TieredCacheKey import io.seqera.wave.util.RegHelper import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils @@ -74,21 +78,19 @@ class RegistryAuthServiceImpl implements RegistryAuthService { @Canonical @ToString(includePackage = false, includeNames = true) - static private class CacheKey { + static private class CacheKey implements TieredCacheKey{ final String image final RegistryAuth auth final RegistryCredentials creds - String stableKey() { + @Override + String stableHash() { return RegHelper.sipHash(['content': toString()]) } } - private CacheLoader loader = new CacheLoader() { - @Override - String load(CacheKey key) throws Exception { - return getToken(key) - } + RegistryAuthToken load(CacheKey key) throws Exception { + return new RegistryAuthToken(getToken(key)) } protected String getToken(CacheKey key){ @@ -108,8 +110,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { return result } - // FIXME https://github.com/seqeralabs/wave/issues/747 - private AsyncLoadingCache cacheTokens + @Inject + private RegistryAuthCache cache @Inject private RegistryLookupService lookupService @@ -117,16 +119,6 @@ class RegistryAuthServiceImpl implements RegistryAuthService { @Inject private RegistryCredentialsFactory credentialsFactory - @PostConstruct - private void init() { - cacheTokens = Caffeine - .newBuilder() - .maximumSize(10_000) - .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) - .executor(ioExecutor) - .buildAsync(loader) - } - /** * Implements container registry login * @@ -284,8 +276,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - // FIXME https://github.com/seqeralabs/wave/issues/747 - return cacheTokens.synchronous().get(key) + return cache.getOrCompute(key, (k)->load(key), _1_HOUR) } catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup @@ -303,8 +294,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { */ void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) - // FIXME https://github.com/seqeralabs/wave/issues/747 - cacheTokens.synchronous().invalidate(key) + cache.invalidate(key) tokenStore.remove(getStableKey(key)) } @@ -312,6 +302,6 @@ class RegistryAuthServiceImpl implements RegistryAuthService { * Invalidate all cached authorization tokens */ private static String getStableKey(CacheKey key) { - return "key-" + key.stableKey() + return "key-" + key.stableHash() } } diff --git a/src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy b/src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy new file mode 100644 index 000000000..18eaa5f6a --- /dev/null +++ b/src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy @@ -0,0 +1,81 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth.cache + +import java.time.Duration + +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable +import io.seqera.wave.auth.model.RegistryAuthToken +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.store.cache.AbstractTieredCache +import io.seqera.wave.store.cache.L2TieredCache +import io.seqera.wave.store.cache.TieredCacheKey +import jakarta.inject.Singleton +/** + * Implement a tiered cache for Registry Auth + * + * @author Munish Chouhan + */ +@Slf4j +@Singleton +@CompileStatic +class RegistryAuthCache extends AbstractTieredCache { + + @Value('${wave.registry.auth.cache.duration:1h}') + private Duration duration + + @Value('${wave.registry.auth.cache.max-size:10000}') + private int maxSize + + RegistryAuthCache(@Nullable L2TieredCache l2) { + super(l2, encoder()) + } + + @Override + int getMaxSize() { + return maxSize + } + + @Override + protected getName() { + return 'registry-auth-cache' + } + + @Override + protected String getPrefix() { + return 'registry-auth-cache/v1' + } + + static MoshiEncodeStrategy encoder() { + new MoshiEncodeStrategy(factory()) {} + } + + static JsonAdapter.Factory factory() { + PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) + .withSubtype(RegistryAuthToken.class, RegistryAuthToken.name) + } + +} diff --git a/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy b/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy new file mode 100644 index 000000000..b2ce3a827 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy @@ -0,0 +1,35 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth.model + +import io.seqera.wave.encoder.MoshiExchange + +/** + * Model for registry auth token + * + * @author Munish Chouhan + */ +class RegistryAuthToken implements MoshiExchange{ + + String value + + RegistryAuthToken(String value) { + this.value = value + } +} diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index ed8329eac..01f3e6553 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -22,15 +22,16 @@ import java.time.Duration import java.util.concurrent.ExecutorService import java.util.regex.Pattern -import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.ToString import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.aws.cache.AwsEcrCache import io.seqera.wave.service.aws.cache.Token import io.seqera.wave.store.cache.TieredCacheKey +import io.seqera.wave.util.RegHelper import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Named @@ -57,6 +58,7 @@ class AwsEcrService { static final private Pattern AWS_ECR_PUBLIC = ~/public\.ecr\.aws/ @Canonical + @ToString(includePackage = false, includeNames = true) private static class AwsCreds implements TieredCacheKey { String accessKey String secretKey @@ -65,13 +67,7 @@ class AwsEcrService { @Override String stableHash() { - final h = Hashing.sipHash24().newHasher() - for( Object it : [accessKey, secretKey, region, ecrPublic] ) { - if( it!=null ) - h.putUnencodedChars(it.toString()) - h.putUnencodedChars('/') - } - return h.hash() + return RegHelper.sipHash(['content': toString()]) } } diff --git a/src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy b/src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy new file mode 100644 index 000000000..65c3c9c89 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy @@ -0,0 +1,67 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth.cache + +import spock.lang.Shared +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.context.ApplicationContext +import io.seqera.wave.auth.RegistryAuth +import io.seqera.wave.auth.model.RegistryAuthToken +import io.seqera.wave.store.cache.RedisL2TieredCache +import io.seqera.wave.test.RedisTestContainer +/** + * + * @author Munish Chouhan + */ +class RegistryAuthCacheTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext applicationContext + + def setup() { + applicationContext = ApplicationContext.run([ + REDIS_HOST: redisHostName, + REDIS_PORT: redisPort + ], 'test', 'redis') + sleep(500) // workaround to wait for Redis connection + } + + def cleanup() { + applicationContext.close() + } + + def 'should cache registry auth response'() { + given: + def store = applicationContext.getBean(RedisL2TieredCache) + def cache1 = new RegistryAuthCache(store) + def cache2 = new RegistryAuthCache(store) + and: + def k = UUID.randomUUID().toString() + def resp = new RegistryAuthToken('token') + + when: + cache1.put(k, resp, Duration.ofSeconds(30)) + + then: + cache2.get(k).value == resp.value + } +} From 0185fc8e6be6615ee7e214e8b689922c22dfff68 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 26 Dec 2024 22:39:58 +0100 Subject: [PATCH 08/28] Added invalidate(key) Signed-off-by: munishchouhan --- .../io/seqera/wave/store/cache/AbstractTieredCache.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index 70f4b18a8..0153fdab3 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -282,4 +282,8 @@ abstract class AbstractTieredCache Date: Mon, 30 Dec 2024 20:56:40 +0100 Subject: [PATCH 09/28] fixed tests Signed-off-by: munishchouhan --- .../wave/auth/RegistryAuthServiceImpl.groovy | 2 +- .../wave/auth/RegistryAuthServiceTest.groovy | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index 2caa79d65..fb4f19bda 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -276,7 +276,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - return cache.getOrCompute(key, (k)->load(key), _1_HOUR) + return cache.getOrCompute(key, (k)->load(key), _1_HOUR).value } catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy index 3f3f96d0c..df396756c 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy @@ -173,7 +173,7 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis def key = Mock(RegistryAuthServiceImpl.CacheKey) def expectedToken = "cachedToken" and: - tokenStore.put("key-" + key.stableKey(), expectedToken) + tokenStore.put("key-" + key.stableHash(), expectedToken) when: def result = impl.getToken(key) @@ -199,19 +199,19 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis def c5 = new RegistryAuthServiceImpl.CacheKey(i1, a1, k3) expect: - c1.stableKey() == '23476a51c7b6216a' - c1.stableKey() == c2.stableKey() - c1.stableKey() == c3.stableKey() + c1.stableHash() == '23476a51c7b6216a' + c1.stableHash() == c2.stableHash() + c1.stableHash() == c3.stableHash() and: - c1.stableKey() != c4.stableKey() - c1.stableKey() != c5.stableKey() + c1.stableHash() != c4.stableHash() + c1.stableHash() != c5.stableHash() } void 'invalidateAuthorization should remove token from cache'() { given: RegistryAuthServiceImpl impl = loginService as RegistryAuthServiceImpl def key = new RegistryAuthServiceImpl.CacheKey("image", Mock(RegistryAuth), Mock(RegistryCredentials)) - def stableKey = "key-" + key.stableKey() + def stableKey = "key-" + key.stableHash() tokenStore.put(stableKey, "token") when: From 5a7c9e2b54940620cd75107e7f686d39432c4163 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 31 Dec 2024 21:14:56 +0100 Subject: [PATCH 10/28] fixed error Signed-off-by: munishchouhan --- .../io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy index 83f4648b4..b5bdb94d5 100644 --- a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy @@ -48,7 +48,7 @@ class AbstractTieredCacheTest extends Specification implements RedisTestContaine return new MoshiEncodeStrategy(factory) {} } - static class MyCache extends AbstractTieredCache { + static class MyCache extends AbstractTieredCache { static String PREFIX = 'foo/v1' From 33049bff01080f1b82acce58d16dd2cec3dea369 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 2 Jan 2025 12:42:11 +0100 Subject: [PATCH 11/28] [release] bump 1.16.6-A0 Signed-off-by: munishchouhan --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0d92a1028..b48d72bef 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.5 +1.16.6-A0 From 682b845785e5434a47beaa1feb0964553797f315 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 2 Jan 2025 14:12:28 +0100 Subject: [PATCH 12/28] Revert "[release] bump 1.16.6-A0" This reverts commit 33049bff01080f1b82acce58d16dd2cec3dea369. --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b48d72bef..0d92a1028 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.6-A0 +1.16.5 From 8fe8bffa918f3fa1d3eefecef0c693e5324e8a3e Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 2 Jan 2025 14:37:42 +0100 Subject: [PATCH 13/28] refactored Signed-off-by: munishchouhan --- .../groovy/io/seqera/wave/service/aws/AwsEcrService.groovy | 6 +++--- .../aws/cache/{Token.groovy => AwsEcrAuthToken.groovy} | 4 ++-- .../io/seqera/wave/service/aws/cache/AwsEcrCache.groovy | 4 ++-- .../io/seqera/wave/store/cache/AbstractTieredCache.groovy | 4 ++++ ...eTest.groovy => RegistryAwsEcrAuthTokenStoreTest.groovy} | 2 +- .../io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) rename src/main/groovy/io/seqera/wave/service/aws/cache/{Token.groovy => AwsEcrAuthToken.groovy} (93%) rename src/test/groovy/io/seqera/wave/auth/{RegistryTokenStoreTest.groovy => RegistryAwsEcrAuthTokenStoreTest.groovy} (94%) diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 01f3e6553..1ca9c6055 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -29,7 +29,7 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.aws.cache.AwsEcrCache -import io.seqera.wave.service.aws.cache.Token +import io.seqera.wave.service.aws.cache.AwsEcrAuthToken import io.seqera.wave.store.cache.TieredCacheKey import io.seqera.wave.util.RegHelper import io.seqera.wave.util.StringUtils @@ -77,11 +77,11 @@ class AwsEcrService { String region } - Token load(AwsCreds creds) throws Exception { + AwsEcrAuthToken load(AwsCreds creds) throws Exception { def token = creds.ecrPublic ? getLoginToken1(creds.accessKey, creds.secretKey, creds.region) : getLoginToken0(creds.accessKey, creds.secretKey, creds.region) - return new Token(token) + return new AwsEcrAuthToken(token) } @Inject diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy similarity index 93% rename from src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy rename to src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy index 3408153dd..8a8eae2b2 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/Token.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy @@ -30,10 +30,10 @@ import io.seqera.wave.encoder.MoshiExchange @CompileStatic @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) -class Token implements MoshiExchange { +class AwsEcrAuthToken implements MoshiExchange { String value - Token(String value) { + AwsEcrAuthToken(String value) { this.value = value } } diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index 69da38b6c..c36f36186 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -40,7 +40,7 @@ import jakarta.inject.Singleton @Slf4j @Singleton @CompileStatic -class AwsEcrCache extends AbstractTieredCache { +class AwsEcrCache extends AbstractTieredCache { @Value('${wave.aws.ecr.cache.duration:3h}') private Duration duration @@ -74,7 +74,7 @@ class AwsEcrCache extends AbstractTieredCache { static JsonAdapter.Factory factory() { PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) - .withSubtype(Token.class, Token.simpleName) + .withSubtype(AwsEcrAuthToken.class, AwsEcrAuthToken.simpleName) } } diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index beb48ecdc..c3bc99768 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -131,6 +131,8 @@ abstract class AbstractTieredCache */ @MicronautTest -class RegistryTokenStoreTest extends Specification { +class RegistryAwsEcrAuthTokenStoreTest extends Specification { @Inject RegistryTokenStore store diff --git a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy index d06f9fb13..fae7ab92e 100644 --- a/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/aws/cache/AwsEcrCacheTest.groovy @@ -54,7 +54,7 @@ class AwsEcrCacheTest extends Specification implements RedisTestContainer { def cache2 = new AwsEcrCache(store) and: def k = UUID.randomUUID().toString() - def token = new Token('token') + def token = new AwsEcrAuthToken('token') when: cache1.put(k, token, Duration.ofSeconds(30)) From 53c2d5831d6b41221b409983b5ae2cd0bf1691bd Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 2 Jan 2025 14:51:31 +0100 Subject: [PATCH 14/28] refactored Signed-off-by: munishchouhan --- .../io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy index d10659d62..e990d756e 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy @@ -28,7 +28,7 @@ import jakarta.inject.Inject * @author Paolo Di Tommaso */ @MicronautTest -class RegistryAwsEcrAuthTokenStoreTest extends Specification { +class RegistryTokenStoreTest extends Specification { @Inject RegistryTokenStore store From 3de28240aca04e04ae4a147507c8030f03ac95a8 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 2 Jan 2025 14:53:34 +0100 Subject: [PATCH 15/28] fixed error Signed-off-by: munishchouhan --- ...EcrAuthTokenStoreTest.groovy => RegistryTokenStoreTest.groovy} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/test/groovy/io/seqera/wave/auth/{RegistryAwsEcrAuthTokenStoreTest.groovy => RegistryTokenStoreTest.groovy} (100%) diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryTokenStoreTest.groovy similarity index 100% rename from src/test/groovy/io/seqera/wave/auth/RegistryAwsEcrAuthTokenStoreTest.groovy rename to src/test/groovy/io/seqera/wave/auth/RegistryTokenStoreTest.groovy From ea3dccd664a051f1e3f797ab5b8c6125673e6722 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 27 Jan 2025 22:49:52 +0100 Subject: [PATCH 16/28] Simplified tiered cache key impl Signed-off-by: Paolo Di Tommaso --- .../wave/auth/model/RegistryAuthToken.groovy | 4 +- .../store/cache/AbstractTieredCache.groovy | 60 +++++++++---------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy b/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy index b2ce3a827..82560981b 100644 --- a/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy +++ b/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.auth.model +import groovy.transform.CompileStatic import io.seqera.wave.encoder.MoshiExchange /** @@ -25,9 +26,10 @@ import io.seqera.wave.encoder.MoshiExchange * * @author Munish Chouhan */ +@CompileStatic class RegistryAuthToken implements MoshiExchange{ - String value + final String value RegistryAuthToken(String value) { this.value = value diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index f386e86d1..dbc74927e 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -45,11 +45,17 @@ import org.jetbrains.annotations.Nullable * * This allow the use in distributed deployment. Note however strong consistently is not guaranteed. * + * @param + * The type of keys maintained by this cache. Note it must be either a + * subtype of {@link CharSequence} or an implementation of {@link TieredCacheKey} interface. + * @param + * The type of values maintained by this cache, which must extend {@link MoshiExchange}. + * * @author Paolo Di Tommaso */ @Slf4j @CompileStatic -abstract class AbstractTieredCache implements TieredCache { +abstract class AbstractTieredCache implements TieredCache { @Canonical @ToString(includePackage = false, includeNames = true) @@ -121,17 +127,15 @@ abstract class AbstractTieredCache loader, Duration ttl) { - if( loader==null ) { - return getOrCompute0(key, null) - } - return getOrCompute0(key, (String k)-> { - V v = loader.apply(key) - return v != null ? new Tuple2<>(v, ttl) : null - }) + @Override + V get(K key) { + getOrCompute0(k0(key), null) } /** @@ -169,11 +164,10 @@ abstract class AbstractTieredCache loader, Duration ttl) { - final hash = key.stableHash() if( loader==null ) { - return getOrCompute0(hash, null) + return getOrCompute0(k0(key), null) } - return getOrCompute0(hash, (String k)-> { + return getOrCompute0(k0(key), (String k)-> { V v = loader.apply(k) return v != null ? new Tuple2<>(v, ttl) : null }) @@ -189,8 +183,8 @@ abstract class AbstractTieredCache> loader) { - return getOrCompute0(key, loader) + V getOrCompute(K key, Function> loader) { + return getOrCompute0(k0(key), loader) } private V getOrCompute0(String key, Function> loader) { @@ -252,15 +246,15 @@ abstract class AbstractTieredCache Date: Tue, 28 Jan 2025 14:00:18 +0100 Subject: [PATCH 17/28] Revert "Added RegistryAuthCache" This reverts commit c7e8536e64e6a87285a59da59f28b1448683ef47. --- .../wave/auth/RegistryAuthServiceImpl.groovy | 38 +++++---- .../wave/auth/cache/RegistryAuthCache.groovy | 81 ------------------- .../wave/auth/model/RegistryAuthToken.groovy | 37 --------- .../wave/service/aws/AwsEcrService.groovy | 12 ++- .../auth/cache/RegistryAuthCacheTest.groovy | 67 --------------- 5 files changed, 32 insertions(+), 203 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy delete mode 100644 src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy delete mode 100644 src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index fb4f19bda..a4252c4f3 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine -import com.google.common.hash.Hashing import groovy.json.JsonSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic @@ -36,13 +35,10 @@ import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j import io.micronaut.scheduling.TaskExecutors -import io.seqera.wave.auth.cache.RegistryAuthCache -import io.seqera.wave.auth.model.RegistryAuthToken import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.exception.RegistryUnauthorizedAccessException import io.seqera.wave.http.HttpClientFactory -import io.seqera.wave.store.cache.TieredCacheKey import io.seqera.wave.util.RegHelper import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils @@ -78,19 +74,21 @@ class RegistryAuthServiceImpl implements RegistryAuthService { @Canonical @ToString(includePackage = false, includeNames = true) - static private class CacheKey implements TieredCacheKey{ + static private class CacheKey { final String image final RegistryAuth auth final RegistryCredentials creds - @Override - String stableHash() { + String stableKey() { return RegHelper.sipHash(['content': toString()]) } } - RegistryAuthToken load(CacheKey key) throws Exception { - return new RegistryAuthToken(getToken(key)) + private CacheLoader loader = new CacheLoader() { + @Override + String load(CacheKey key) throws Exception { + return getToken(key) + } } protected String getToken(CacheKey key){ @@ -110,8 +108,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { return result } - @Inject - private RegistryAuthCache cache + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cacheTokens @Inject private RegistryLookupService lookupService @@ -119,6 +117,16 @@ class RegistryAuthServiceImpl implements RegistryAuthService { @Inject private RegistryCredentialsFactory credentialsFactory + @PostConstruct + private void init() { + cacheTokens = Caffeine + .newBuilder() + .maximumSize(10_000) + .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) + .executor(ioExecutor) + .buildAsync(loader) + } + /** * Implements container registry login * @@ -276,7 +284,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - return cache.getOrCompute(key, (k)->load(key), _1_HOUR).value + // FIXME https://github.com/seqeralabs/wave/issues/747 + return cacheTokens.synchronous().get(key) } catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup @@ -294,7 +303,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { */ void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) - cache.invalidate(key) + // FIXME https://github.com/seqeralabs/wave/issues/747 + cacheTokens.synchronous().invalidate(key) tokenStore.remove(getStableKey(key)) } @@ -302,6 +312,6 @@ class RegistryAuthServiceImpl implements RegistryAuthService { * Invalidate all cached authorization tokens */ private static String getStableKey(CacheKey key) { - return "key-" + key.stableHash() + return "key-" + key.stableKey() } } diff --git a/src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy b/src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy deleted file mode 100644 index 18eaa5f6a..000000000 --- a/src/main/groovy/io/seqera/wave/auth/cache/RegistryAuthCache.groovy +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.auth.cache - -import java.time.Duration - -import com.squareup.moshi.JsonAdapter -import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Value -import io.micronaut.core.annotation.Nullable -import io.seqera.wave.auth.model.RegistryAuthToken -import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange -import io.seqera.wave.store.cache.AbstractTieredCache -import io.seqera.wave.store.cache.L2TieredCache -import io.seqera.wave.store.cache.TieredCacheKey -import jakarta.inject.Singleton -/** - * Implement a tiered cache for Registry Auth - * - * @author Munish Chouhan - */ -@Slf4j -@Singleton -@CompileStatic -class RegistryAuthCache extends AbstractTieredCache { - - @Value('${wave.registry.auth.cache.duration:1h}') - private Duration duration - - @Value('${wave.registry.auth.cache.max-size:10000}') - private int maxSize - - RegistryAuthCache(@Nullable L2TieredCache l2) { - super(l2, encoder()) - } - - @Override - int getMaxSize() { - return maxSize - } - - @Override - protected getName() { - return 'registry-auth-cache' - } - - @Override - protected String getPrefix() { - return 'registry-auth-cache/v1' - } - - static MoshiEncodeStrategy encoder() { - new MoshiEncodeStrategy(factory()) {} - } - - static JsonAdapter.Factory factory() { - PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") - .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) - .withSubtype(RegistryAuthToken.class, RegistryAuthToken.name) - } - -} diff --git a/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy b/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy deleted file mode 100644 index 82560981b..000000000 --- a/src/main/groovy/io/seqera/wave/auth/model/RegistryAuthToken.groovy +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.auth.model - -import groovy.transform.CompileStatic -import io.seqera.wave.encoder.MoshiExchange - -/** - * Model for registry auth token - * - * @author Munish Chouhan - */ -@CompileStatic -class RegistryAuthToken implements MoshiExchange{ - - final String value - - RegistryAuthToken(String value) { - this.value = value - } -} diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 1ca9c6055..2bb8c7295 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -22,16 +22,15 @@ import java.time.Duration import java.util.concurrent.ExecutorService import java.util.regex.Pattern +import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic -import groovy.transform.ToString import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.aws.cache.AwsEcrCache import io.seqera.wave.service.aws.cache.AwsEcrAuthToken import io.seqera.wave.store.cache.TieredCacheKey -import io.seqera.wave.util.RegHelper import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Named @@ -58,7 +57,6 @@ class AwsEcrService { static final private Pattern AWS_ECR_PUBLIC = ~/public\.ecr\.aws/ @Canonical - @ToString(includePackage = false, includeNames = true) private static class AwsCreds implements TieredCacheKey { String accessKey String secretKey @@ -67,7 +65,13 @@ class AwsEcrService { @Override String stableHash() { - return RegHelper.sipHash(['content': toString()]) + final h = Hashing.sipHash24().newHasher() + for( Object it : [accessKey, secretKey, region, ecrPublic] ) { + if( it!=null ) + h.putUnencodedChars(it.toString()) + h.putUnencodedChars('/') + } + return h.hash() } } diff --git a/src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy b/src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy deleted file mode 100644 index 65c3c9c89..000000000 --- a/src/test/groovy/io/seqera/wave/auth/cache/RegistryAuthCacheTest.groovy +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.auth.cache - -import spock.lang.Shared -import spock.lang.Specification - -import java.time.Duration - -import io.micronaut.context.ApplicationContext -import io.seqera.wave.auth.RegistryAuth -import io.seqera.wave.auth.model.RegistryAuthToken -import io.seqera.wave.store.cache.RedisL2TieredCache -import io.seqera.wave.test.RedisTestContainer -/** - * - * @author Munish Chouhan - */ -class RegistryAuthCacheTest extends Specification implements RedisTestContainer { - - @Shared - ApplicationContext applicationContext - - def setup() { - applicationContext = ApplicationContext.run([ - REDIS_HOST: redisHostName, - REDIS_PORT: redisPort - ], 'test', 'redis') - sleep(500) // workaround to wait for Redis connection - } - - def cleanup() { - applicationContext.close() - } - - def 'should cache registry auth response'() { - given: - def store = applicationContext.getBean(RedisL2TieredCache) - def cache1 = new RegistryAuthCache(store) - def cache2 = new RegistryAuthCache(store) - and: - def k = UUID.randomUUID().toString() - def resp = new RegistryAuthToken('token') - - when: - cache1.put(k, resp, Duration.ofSeconds(30)) - - then: - cache2.get(k).value == resp.value - } -} From 807ef92fb6348bd9e0446251094c8fa5ac25914b Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 28 Jan 2025 14:00:55 +0100 Subject: [PATCH 18/28] removed invalidate(K key) Signed-off-by: munishchouhan --- .../io/seqera/wave/store/cache/AbstractTieredCache.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index dbc74927e..551af5474 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -312,8 +312,4 @@ abstract class AbstractTieredCache implements Tiered l1.invalidateAll() } - void invalidate(K key) { - l1.invalidate(k0(key)) - } - } From dba3cd447e180213003495c731b33ab25a87c074 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 28 Jan 2025 17:29:23 +0100 Subject: [PATCH 19/28] fixed errors Signed-off-by: munishchouhan --- .../groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy | 2 +- src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy b/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy index c91f0ec43..12afacebb 100644 --- a/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy +++ b/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy @@ -41,7 +41,7 @@ import jakarta.inject.Singleton @Slf4j @Singleton @CompileStatic -class RegistryLookupCache extends AbstractTieredCache { +class RegistryLookupCache extends AbstractTieredCache { @Value('${wave.registry.cache.duration:1h}') private Duration duration diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy index 18ed6c71b..0d336dd70 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy @@ -39,7 +39,7 @@ import jakarta.inject.Singleton @Slf4j @Singleton @CompileStatic -class ProxyCache extends AbstractTieredCache { +class ProxyCache extends AbstractTieredCache { private ProxyCacheConfig config From 32855c5e16379f2ccf1b026f6d12d6bf0c57ab0b Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 28 Jan 2025 17:31:37 +0100 Subject: [PATCH 20/28] fixed errors Signed-off-by: munishchouhan --- .../io/seqera/wave/store/cache/AbstractTieredCache.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index 551af5474..30dd89482 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -127,7 +127,7 @@ abstract class AbstractTieredCache implements Tiered } } - protected static String k0(K key) { + protected String k0(K key) { if( key instanceof CharSequence ) return key.toString() if( key instanceof TieredCacheKey ) From c352959fe7cc9cc6a93cb570a1551bcbec7d0274 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 29 Jan 2025 11:22:16 +0100 Subject: [PATCH 21/28] fixed tests Signed-off-by: munishchouhan --- .../seqera/wave/auth/RegistryAuthServiceTest.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy index df396756c..215ef3ee0 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy @@ -199,19 +199,19 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis def c5 = new RegistryAuthServiceImpl.CacheKey(i1, a1, k3) expect: - c1.stableHash() == '23476a51c7b6216a' - c1.stableHash() == c2.stableHash() - c1.stableHash() == c3.stableHash() + c1.stableKey() == '23476a51c7b6216a' + c1.stableKey() == c2.stableKey() + c1.stableKey() == c3.stableKey() and: - c1.stableHash() != c4.stableHash() - c1.stableHash() != c5.stableHash() + c1.stableKey() != c4.stableKey() + c1.stableKey() != c5.stableKey() } void 'invalidateAuthorization should remove token from cache'() { given: RegistryAuthServiceImpl impl = loginService as RegistryAuthServiceImpl def key = new RegistryAuthServiceImpl.CacheKey("image", Mock(RegistryAuth), Mock(RegistryCredentials)) - def stableKey = "key-" + key.stableHash() + def stableKey = "key-" + key.stableKey() tokenStore.put(stableKey, "token") when: From 068b28dffb002005d96948a444ee985693f3128a Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 29 Jan 2025 11:23:28 +0100 Subject: [PATCH 22/28] fixed tests Signed-off-by: munishchouhan --- .../groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy index 215ef3ee0..3f3f96d0c 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy @@ -173,7 +173,7 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis def key = Mock(RegistryAuthServiceImpl.CacheKey) def expectedToken = "cachedToken" and: - tokenStore.put("key-" + key.stableHash(), expectedToken) + tokenStore.put("key-" + key.stableKey(), expectedToken) when: def result = impl.getToken(key) From 1289cde84c64683d3fb41f94582c0fe42567679e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 14 Feb 2025 21:33:18 +0100 Subject: [PATCH 23/28] Remove double cache + refactor Signed-off-by: Paolo Di Tommaso --- .../seqera/wave/auth/RegistryAuthStore.groovy | 61 ------------------- .../{cache => }/RegistryLookupCache.groovy | 20 +++--- .../auth/RegistryLookupServiceImpl.groovy | 26 +------- .../io/seqera/wave/proxy/ProxyCache.groovy | 1 - .../wave/service/aws/AwsEcrService.groovy | 18 ++---- .../service/aws/cache/AwsEcrAuthToken.groovy | 2 +- .../wave/service/aws/cache/AwsEcrCache.groovy | 4 +- .../store/cache/AbstractTieredCache.groovy | 4 +- ...TieredCacheKey.groovy => TieredKey.groovy} | 4 +- .../wave/auth/RegistryAuthStoreTest.groovy | 41 ------------- .../RegistryLookupCacheTest.groovy | 5 +- .../cache/AbstractTieredCacheTest.groovy | 2 +- 12 files changed, 28 insertions(+), 160 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy rename src/main/groovy/io/seqera/wave/auth/{cache => }/RegistryLookupCache.groovy (84%) rename src/main/groovy/io/seqera/wave/store/cache/{TieredCacheKey.groovy => TieredKey.groovy} (91%) delete mode 100644 src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy rename src/test/groovy/io/seqera/wave/auth/{cache => }/RegistryLookupCacheTest.groovy (94%) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy deleted file mode 100644 index ba1c834c4..000000000 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.auth - -import java.time.Duration - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Value -import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.store.state.AbstractStateStore -import io.seqera.wave.store.state.impl.StateProvider -import jakarta.inject.Singleton -/** - * Implement a cache store for {@link RegistryAuth} object that - * can be distributed across wave replicas - * - * @author Paolo Di Tommaso - */ -@Slf4j -@Singleton -@CompileStatic -class RegistryAuthStore extends AbstractStateStore { - - private Duration duration - - RegistryAuthStore( - StateProvider provider, - @Value('${wave.registry-auth.cache.duration:`3h`}') Duration duration) - { - super(provider, new MoshiEncodeStrategy() {}) - this.duration = duration - log.info "Creating Registry Auth cache store ― duration=$duration" - } - - @Override - protected String getPrefix() { - return 'registry-auth/v1' - } - - @Override - protected Duration getDuration() { - return duration - } -} diff --git a/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy similarity index 84% rename from src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy rename to src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy index 12afacebb..b3f573cdf 100644 --- a/src/main/groovy/io/seqera/wave/auth/cache/RegistryLookupCache.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy @@ -1,6 +1,6 @@ /* * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs + * Copyright (c) 2023-2024, Seqera Labs * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.auth.cache +package io.seqera.wave.auth import java.time.Duration @@ -26,15 +26,13 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable -import io.seqera.wave.auth.RegistryAuth import io.seqera.wave.encoder.MoshiEncodeStrategy import io.seqera.wave.encoder.MoshiExchange import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache -import io.seqera.wave.store.cache.TieredCacheKey import jakarta.inject.Singleton /** - * Implement a tiered cache for Registry lookup + * Implement a tiered cache for {@link RegistryLookupService} * * @author Munish Chouhan */ @@ -43,10 +41,10 @@ import jakarta.inject.Singleton @CompileStatic class RegistryLookupCache extends AbstractTieredCache { - @Value('${wave.registry.cache.duration:1h}') + @Value('${wave.registry-lookup.cache.duration:1h}') private Duration duration - @Value('${wave.registry.cache.max-size:10000}') + @Value('${wave.registry-lookup.cache.max-size:10000}') private int maxSize RegistryLookupCache(@Nullable L2TieredCache l2) { @@ -60,12 +58,16 @@ class RegistryLookupCache extends AbstractTieredCache { @Override protected getName() { - return 'registry-cache' + return 'registry-lookup-cache' + } + + Duration getDuration() { + return duration } @Override protected String getPrefix() { - return 'registry-cache/v1' + return 'registry-lookup-cache/v1' } static MoshiEncodeStrategy encoder() { diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 935ef8666..2698bd27a 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -20,15 +20,12 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse -import java.time.Duration import java.util.concurrent.CompletionException import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors -import io.seqera.wave.auth.cache.RegistryLookupCache import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.http.HttpClientFactory @@ -54,34 +51,13 @@ class RegistryLookupServiceImpl implements RegistryLookupService { @Inject private HttpClientConfig httpConfig - @Inject - private RegistryAuthStore store - @Inject @Named(TaskExecutors.BLOCKING) private ExecutorService ioExecutor - RegistryAuth load(URI endpoint) throws Exception { - // check if there's a record in the store cache (redis) - def result = store.get(endpoint.toString()) - if( result ) { - log.debug "Authority lookup for endpoint: '$endpoint' => $result [from store]" - return result - } - // look-up using the corresponding API endpoint - result = lookup0(endpoint) - log.debug "Authority lookup for endpoint: '$endpoint' => $result" - // save it in the store cache (redis) - store.put(endpoint.toString(), result) - return result - } - @Inject private RegistryLookupCache cache - @Value('${wave.registry.cache.duration:24h}') - private Duration cacheDuration - protected RegistryAuth lookup0(URI endpoint) { final httpClient = HttpClientFactory.followRedirectsHttpClient() final request = HttpRequest.newBuilder() .uri(endpoint) .GET() .build() @@ -119,7 +95,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService { RegistryInfo lookup(String registry) { try { final endpoint = registryEndpoint(registry) - final auth = cache.getOrCompute(endpoint.toString(), (key) -> load(endpoint), cacheDuration) + final auth = cache.getOrCompute(endpoint.toString(), (k) -> lookup0(endpoint), cache.duration) return new RegistryInfo(registry, endpoint, auth) } catch (CompletionException e) { diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy index 0d336dd70..608197621 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy @@ -29,7 +29,6 @@ import io.seqera.wave.encoder.MoshiEncodeStrategy import io.seqera.wave.encoder.MoshiExchange import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache -import io.seqera.wave.store.cache.TieredCacheKey import jakarta.inject.Singleton /** * Implements a tiered cache for proxied http responses diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 2bb8c7295..d9f775a28 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -22,15 +22,15 @@ import java.time.Duration import java.util.concurrent.ExecutorService import java.util.regex.Pattern -import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors -import io.seqera.wave.service.aws.cache.AwsEcrCache import io.seqera.wave.service.aws.cache.AwsEcrAuthToken -import io.seqera.wave.store.cache.TieredCacheKey +import io.seqera.wave.service.aws.cache.AwsEcrCache +import io.seqera.wave.store.cache.TieredKey +import io.seqera.wave.util.RegHelper import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Named @@ -57,7 +57,7 @@ class AwsEcrService { static final private Pattern AWS_ECR_PUBLIC = ~/public\.ecr\.aws/ @Canonical - private static class AwsCreds implements TieredCacheKey { + private static class AwsCreds implements TieredKey { String accessKey String secretKey String region @@ -65,13 +65,7 @@ class AwsEcrService { @Override String stableHash() { - final h = Hashing.sipHash24().newHasher() - for( Object it : [accessKey, secretKey, region, ecrPublic] ) { - if( it!=null ) - h.putUnencodedChars(it.toString()) - h.putUnencodedChars('/') - } - return h.hash() + RegHelper.sipHash(accessKey, secretKey, region, ecrPublic) } } @@ -93,7 +87,7 @@ class AwsEcrService { private ExecutorService ioExecutor @Inject - AwsEcrCache cache + private AwsEcrCache cache @Value('${wave.aws.ecr.cache.duration:24h}') private Duration cacheDuration diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy index 8a8eae2b2..67252a9cb 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy @@ -23,7 +23,7 @@ import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.encoder.MoshiExchange /** - * Implement a tiered cache for AWS ECR client + * Model a tiered cache value for {@link AwsEcrCache} * * @author Munish Chouhan */ diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index c36f36186..53723090e 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -30,7 +30,7 @@ import io.seqera.wave.encoder.MoshiEncodeStrategy import io.seqera.wave.encoder.MoshiExchange import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache -import io.seqera.wave.store.cache.TieredCacheKey +import io.seqera.wave.store.cache.TieredKey import jakarta.inject.Singleton /** * Implement a tiered cache for AWS ECR client @@ -40,7 +40,7 @@ import jakarta.inject.Singleton @Slf4j @Singleton @CompileStatic -class AwsEcrCache extends AbstractTieredCache { +class AwsEcrCache extends AbstractTieredCache { @Value('${wave.aws.ecr.cache.duration:3h}') private Duration duration diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index 30dd89482..075192a33 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -47,7 +47,7 @@ import org.jetbrains.annotations.Nullable * * @param * The type of keys maintained by this cache. Note it must be either a - * subtype of {@link CharSequence} or an implementation of {@link TieredCacheKey} interface. + * subtype of {@link CharSequence} or an implementation of {@link TieredKey} interface. * @param * The type of values maintained by this cache, which must extend {@link MoshiExchange}. * @@ -130,7 +130,7 @@ abstract class AbstractTieredCache implements Tiered protected String k0(K key) { if( key instanceof CharSequence ) return key.toString() - if( key instanceof TieredCacheKey ) + if( key instanceof TieredKey ) return key.stableHash() if( key==null ) throw new IllegalArgumentException("Tiered cache key cannot be null") diff --git a/src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy b/src/main/groovy/io/seqera/wave/store/cache/TieredKey.groovy similarity index 91% rename from src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy rename to src/main/groovy/io/seqera/wave/store/cache/TieredKey.groovy index 40ea9beaf..a00e178c5 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/TieredCacheKey.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/TieredKey.groovy @@ -19,11 +19,11 @@ package io.seqera.wave.store.cache /** - * Implement a tiered cache for AWS ECR client + * Define the contract for key used by {@link TieredCache} caches * * @author Munish Chouhan */ -interface TieredCacheKey { +interface TieredKey { String stableHash() diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy deleted file mode 100644 index c6fb81ffa..000000000 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.auth - -import spock.lang.Specification - -import io.micronaut.test.extensions.spock.annotation.MicronautTest -import jakarta.inject.Inject - -/** - * - * @author Paolo Di Tommaso - */ -@MicronautTest -class RegistryAuthStoreTest extends Specification { - - @Inject RegistryAuthStore store - - def 'should return entry key' () { - expect: - store.key0('foo') == 'registry-auth/v1:foo' - } - - -} diff --git a/src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryLookupCacheTest.groovy similarity index 94% rename from src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy rename to src/test/groovy/io/seqera/wave/auth/RegistryLookupCacheTest.groovy index 96d31485d..3d87d51ef 100644 --- a/src/test/groovy/io/seqera/wave/auth/cache/RegistryLookupCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryLookupCacheTest.groovy @@ -1,6 +1,6 @@ /* * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs + * Copyright (c) 2023-2024, Seqera Labs * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.auth.cache +package io.seqera.wave.auth import spock.lang.Shared import spock.lang.Specification @@ -24,7 +24,6 @@ import spock.lang.Specification import java.time.Duration import io.micronaut.context.ApplicationContext -import io.seqera.wave.auth.RegistryAuth import io.seqera.wave.store.cache.RedisL2TieredCache import io.seqera.wave.test.RedisTestContainer /** diff --git a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy index b5bdb94d5..b8e2eb345 100644 --- a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy @@ -48,7 +48,7 @@ class AbstractTieredCacheTest extends Specification implements RedisTestContaine return new MoshiEncodeStrategy(factory) {} } - static class MyCache extends AbstractTieredCache { + static class MyCache extends AbstractTieredCache { static String PREFIX = 'foo/v1' From e567e76760a30f951ecb1cb20795a79e5f43abf6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 14 Feb 2025 21:44:09 +0100 Subject: [PATCH 24/28] Refactor Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/auth/RegistryAuth.groovy | 4 ++-- .../io/seqera/wave/auth/RegistryLookupCache.groovy | 4 ++-- .../{MoshiExchange.groovy => MoshiSerializable.groovy} | 4 ++-- .../io/seqera/wave/proxy/DelegateResponse.groovy | 4 ++-- src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy | 4 ++-- .../wave/service/aws/cache/AwsEcrAuthToken.groovy | 4 ++-- .../seqera/wave/service/aws/cache/AwsEcrCache.groovy | 4 ++-- .../seqera/wave/store/cache/AbstractTieredCache.groovy | 10 +++++----- src/main/groovy/io/seqera/wave/tower/User.groovy | 4 ++-- .../wave/tower/client/CredentialsDescription.groovy | 4 ++-- .../tower/client/GetCredentialsKeysResponse.groovy | 4 ++-- .../wave/tower/client/GetUserInfoResponse.groovy | 4 ++-- .../wave/tower/client/ListCredentialsResponse.groovy | 4 ++-- .../seqera/wave/tower/client/cache/ClientCache.groovy | 4 ++-- .../io/seqera/wave/tower/compute/ComputeEnv.groovy | 4 ++-- .../compute/DescribeWorkflowLaunchResponse.groovy | 4 ++-- .../io/seqera/wave/tower/compute/WorkflowLaunch.groovy | 4 ++-- .../wave/store/cache/AbstractTieredCacheTest.groovy | 6 +++--- 18 files changed, 40 insertions(+), 40 deletions(-) rename src/main/groovy/io/seqera/wave/encoder/{MoshiExchange.groovy => MoshiSerializable.groovy} (91%) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy index 6077c5a8c..38ee62c5a 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuth.groovy @@ -23,7 +23,7 @@ import java.util.regex.Pattern import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Model container registry authentication meta-info @@ -33,7 +33,7 @@ import io.seqera.wave.encoder.MoshiExchange @Canonical @CompileStatic @ToString(includePackage = false, includeNames = true) -class RegistryAuth implements MoshiExchange { +class RegistryAuth implements MoshiSerializable { private static final Pattern AUTH = ~/(?i)(?.+) realm="(?[^"]+)",service="(?[^"]+)"/ // some registries doesn't send the service diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy index b3f573cdf..139e3c11d 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupCache.groovy @@ -27,7 +27,7 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache import jakarta.inject.Singleton @@ -75,7 +75,7 @@ class RegistryLookupCache extends AbstractTieredCache { } static JsonAdapter.Factory factory() { - PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + PolymorphicJsonAdapterFactory.of(MoshiSerializable.class, "@type") .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) .withSubtype(RegistryAuth.class, RegistryAuth.name) } diff --git a/src/main/groovy/io/seqera/wave/encoder/MoshiExchange.groovy b/src/main/groovy/io/seqera/wave/encoder/MoshiSerializable.groovy similarity index 91% rename from src/main/groovy/io/seqera/wave/encoder/MoshiExchange.groovy rename to src/main/groovy/io/seqera/wave/encoder/MoshiSerializable.groovy index c6d5fef54..78589b461 100644 --- a/src/main/groovy/io/seqera/wave/encoder/MoshiExchange.groovy +++ b/src/main/groovy/io/seqera/wave/encoder/MoshiSerializable.groovy @@ -19,9 +19,9 @@ package io.seqera.wave.encoder /** - * Marker interface for Moshi encoded exchange objects + * Marker interface for Moshi serializable objects * * @author Paolo Di Tommaso */ -interface MoshiExchange { +interface MoshiSerializable { } diff --git a/src/main/groovy/io/seqera/wave/proxy/DelegateResponse.groovy b/src/main/groovy/io/seqera/wave/proxy/DelegateResponse.groovy index 88c88afc3..3fa270be2 100644 --- a/src/main/groovy/io/seqera/wave/proxy/DelegateResponse.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/DelegateResponse.groovy @@ -19,14 +19,14 @@ package io.seqera.wave.proxy import groovy.transform.EqualsAndHashCode -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Model a response object to be forwarded to the client * * @author Paolo Di Tommaso */ @EqualsAndHashCode -class DelegateResponse implements MoshiExchange { +class DelegateResponse implements MoshiSerializable { int statusCode Map> headers byte[] body diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy index 608197621..763d7ddbb 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyCache.groovy @@ -26,7 +26,7 @@ import groovy.util.logging.Slf4j import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.ProxyCacheConfig import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache import jakarta.inject.Singleton @@ -50,7 +50,7 @@ class ProxyCache extends AbstractTieredCache { static MoshiEncodeStrategy encoder() { // json adapter factory - final factory = PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + final factory = PolymorphicJsonAdapterFactory.of(MoshiSerializable.class, "@type") .withSubtype(Entry.class, Entry.name) .withSubtype(DelegateResponse.class, DelegateResponse.simpleName) // the encoding strategy diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy index 67252a9cb..90663243e 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrAuthToken.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.service.aws.cache import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Model a tiered cache value for {@link AwsEcrCache} * @@ -30,7 +30,7 @@ import io.seqera.wave.encoder.MoshiExchange @CompileStatic @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) -class AwsEcrAuthToken implements MoshiExchange { +class AwsEcrAuthToken implements MoshiSerializable { String value AwsEcrAuthToken(String value) { diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index 53723090e..100535a69 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -27,7 +27,7 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache import io.seqera.wave.store.cache.TieredKey @@ -72,7 +72,7 @@ class AwsEcrCache extends AbstractTieredCache { } static JsonAdapter.Factory factory() { - PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + PolymorphicJsonAdapterFactory.of(MoshiSerializable.class, "@type") .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) .withSubtype(AwsEcrAuthToken.class, AwsEcrAuthToken.simpleName) } diff --git a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy index 075192a33..5b8a38d02 100644 --- a/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy +++ b/src/main/groovy/io/seqera/wave/store/cache/AbstractTieredCache.groovy @@ -37,7 +37,7 @@ import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.encoder.EncodingStrategy import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import org.jetbrains.annotations.Nullable /** * Implement a tiered-cache mechanism using a local caffeine cache as 1st level access @@ -49,18 +49,18 @@ import org.jetbrains.annotations.Nullable * The type of keys maintained by this cache. Note it must be either a * subtype of {@link CharSequence} or an implementation of {@link TieredKey} interface. * @param - * The type of values maintained by this cache, which must extend {@link MoshiExchange}. + * The type of values maintained by this cache, which must extend {@link MoshiSerializable}. * * @author Paolo Di Tommaso */ @Slf4j @CompileStatic -abstract class AbstractTieredCache implements TieredCache { +abstract class AbstractTieredCache implements TieredCache { @Canonical @ToString(includePackage = false, includeNames = true) - static class Entry implements MoshiExchange { - MoshiExchange value + static class Entry implements MoshiSerializable { + MoshiSerializable value long expiresAt } diff --git a/src/main/groovy/io/seqera/wave/tower/User.groovy b/src/main/groovy/io/seqera/wave/tower/User.groovy index a604216e1..8100b5a89 100644 --- a/src/main/groovy/io/seqera/wave/tower/User.groovy +++ b/src/main/groovy/io/seqera/wave/tower/User.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.tower import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import jakarta.validation.constraints.NotNull import jakarta.validation.constraints.Size @@ -33,7 +33,7 @@ import jakarta.validation.constraints.Size @ToString(includeNames = true, includePackage = false, includes = 'id,userName,email') @EqualsAndHashCode @CompileStatic -class User implements MoshiExchange { +class User implements MoshiSerializable { Long id diff --git a/src/main/groovy/io/seqera/wave/tower/client/CredentialsDescription.groovy b/src/main/groovy/io/seqera/wave/tower/client/CredentialsDescription.groovy index 9069f4f8f..b1a097574 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/CredentialsDescription.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/CredentialsDescription.groovy @@ -23,12 +23,12 @@ import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.WaveDefault -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable @EqualsAndHashCode @CompileStatic @ToString(includePackage = false, includeNames = true) -class CredentialsDescription implements MoshiExchange { +class CredentialsDescription implements MoshiSerializable { String id String provider diff --git a/src/main/groovy/io/seqera/wave/tower/client/GetCredentialsKeysResponse.groovy b/src/main/groovy/io/seqera/wave/tower/client/GetCredentialsKeysResponse.groovy index 9cd2b8eff..d0496f673 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/GetCredentialsKeysResponse.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/GetCredentialsKeysResponse.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.tower.client import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Models an encrypted credentials keys response @@ -31,7 +31,7 @@ import io.seqera.wave.encoder.MoshiExchange @ToString(includePackage = false, includeNames = true) @EqualsAndHashCode @CompileStatic -class GetCredentialsKeysResponse implements MoshiExchange { +class GetCredentialsKeysResponse implements MoshiSerializable { /** * Secret keys associated with the credentials diff --git a/src/main/groovy/io/seqera/wave/tower/client/GetUserInfoResponse.groovy b/src/main/groovy/io/seqera/wave/tower/client/GetUserInfoResponse.groovy index 7b9fc6f28..9dfa8654e 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/GetUserInfoResponse.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/GetUserInfoResponse.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.tower.client import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import io.seqera.wave.tower.User /** * Model a Tower user-info response @@ -30,6 +30,6 @@ import io.seqera.wave.tower.User @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) @CompileStatic -class GetUserInfoResponse implements MoshiExchange { +class GetUserInfoResponse implements MoshiSerializable { User user } diff --git a/src/main/groovy/io/seqera/wave/tower/client/ListCredentialsResponse.groovy b/src/main/groovy/io/seqera/wave/tower/client/ListCredentialsResponse.groovy index ef6969a32..77eadc3b1 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/ListCredentialsResponse.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/ListCredentialsResponse.groovy @@ -21,12 +21,12 @@ package io.seqera.wave.tower.client import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) @CompileStatic -class ListCredentialsResponse implements MoshiExchange { +class ListCredentialsResponse implements MoshiSerializable { List credentials diff --git a/src/main/groovy/io/seqera/wave/tower/client/cache/ClientCache.groovy b/src/main/groovy/io/seqera/wave/tower/client/cache/ClientCache.groovy index 4b72f4703..5a8dbcd95 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/cache/ClientCache.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/cache/ClientCache.groovy @@ -26,7 +26,7 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import io.seqera.wave.store.cache.AbstractTieredCache import io.seqera.wave.store.cache.L2TieredCache import io.seqera.wave.tower.User @@ -75,7 +75,7 @@ class ClientCache extends AbstractTieredCache { } static JsonAdapter.Factory factory() { - PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + PolymorphicJsonAdapterFactory.of(MoshiSerializable.class, "@type") .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) // add all exchange classes used by the tower client .withSubtype(ComputeEnv.class, ComputeEnv.simpleName) diff --git a/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy b/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy index 453ecd1f4..af7febbee 100644 --- a/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy +++ b/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.tower.compute import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Model the response of compute environment from seqera platform @@ -31,7 +31,7 @@ import io.seqera.wave.encoder.MoshiExchange @CompileStatic @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) -class ComputeEnv implements MoshiExchange { +class ComputeEnv implements MoshiSerializable { String id String platform String credentialsId diff --git a/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy b/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy index ac3653620..c1a9d05a8 100644 --- a/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy +++ b/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.tower.compute import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Model the response of workflow launch describe request @@ -31,7 +31,7 @@ import io.seqera.wave.encoder.MoshiExchange @CompileStatic @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) -class DescribeWorkflowLaunchResponse implements MoshiExchange { +class DescribeWorkflowLaunchResponse implements MoshiSerializable { WorkflowLaunch launch diff --git a/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunch.groovy b/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunch.groovy index dd6cf0775..45b9ec249 100644 --- a/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunch.groovy +++ b/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunch.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.tower.compute import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable /** * Model the response of workflow launch response from seqera platform @@ -31,6 +31,6 @@ import io.seqera.wave.encoder.MoshiExchange @CompileStatic @EqualsAndHashCode @ToString(includePackage = false, includeNames = true) -class WorkflowLaunch implements MoshiExchange { +class WorkflowLaunch implements MoshiSerializable { ComputeEnv computeEnv } diff --git a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy index b8e2eb345..4b50416eb 100644 --- a/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/cache/AbstractTieredCacheTest.groovy @@ -26,7 +26,7 @@ import groovy.transform.Canonical import groovy.transform.Memoized import io.micronaut.context.ApplicationContext import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.encoder.MoshiExchange +import io.seqera.wave.encoder.MoshiSerializable import io.seqera.wave.test.RedisTestContainer import spock.lang.Shared import spock.lang.Specification @@ -34,14 +34,14 @@ import spock.lang.Specification class AbstractTieredCacheTest extends Specification implements RedisTestContainer { @Canonical - static class MyBean implements MoshiExchange { + static class MyBean implements MoshiSerializable { String foo String bar } @Memoized static MoshiEncodeStrategy encoder() { - JsonAdapter.Factory factory = PolymorphicJsonAdapterFactory.of(MoshiExchange.class, "@type") + JsonAdapter.Factory factory = PolymorphicJsonAdapterFactory.of(MoshiSerializable.class, "@type") .withSubtype(AbstractTieredCache.Entry.class, AbstractTieredCache.Entry.name) .withSubtype(MyBean.class, MyBean.name) From 251bce30810b9239fff903bafec87bf3f1c403ae Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 14 Feb 2025 21:50:23 +0100 Subject: [PATCH 25/28] [release] 1.17.3-B0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 06fb41b63..393f1359a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.17.2 +1.17.3-B0 From 7491568f6fc2287ef00b3a7b949f99162eb8302a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 17 Feb 2025 16:21:17 +0100 Subject: [PATCH 26/28] Update VERSION [ci skip] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 393f1359a..06fb41b63 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.17.3-B0 +1.17.2 From aa8d4c3d58697f1e0f7fe2b036bd1a7b2a946fe4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 17 Feb 2025 16:33:08 +0100 Subject: [PATCH 27/28] Remove unused try catch Signed-off-by: Paolo Di Tommaso --- .../wave/auth/RegistryLookupServiceImpl.groovy | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 2698bd27a..7dacc9595 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse -import java.util.concurrent.CompletionException import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic @@ -93,16 +92,9 @@ class RegistryLookupServiceImpl implements RegistryLookupService { */ @Override RegistryInfo lookup(String registry) { - try { - final endpoint = registryEndpoint(registry) - final auth = cache.getOrCompute(endpoint.toString(), (k) -> lookup0(endpoint), cache.duration) - return new RegistryInfo(registry, endpoint, auth) - } - catch (CompletionException e) { - // this catches the exception thrown in the cache loader lookup - // and throws the causing exception that should be `RegistryUnauthorizedAccessException` - throw e.cause - } + final endpoint = registryEndpoint(registry) + final auth = cache.getOrCompute(endpoint.toString(), (k) -> lookup0(endpoint), cache.duration) + return new RegistryInfo(registry, endpoint, auth) } /** From a9ef143cbb514d4fd2b0b94face19f0605dca324 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 17 Feb 2025 16:38:00 +0100 Subject: [PATCH 28/28] Remove unused attribute Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/service/aws/AwsEcrService.groovy | 8 ++------ .../io/seqera/wave/service/aws/cache/AwsEcrCache.groovy | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index d9f775a28..5a4f30392 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -18,14 +18,13 @@ package io.seqera.wave.service.aws -import java.time.Duration + import java.util.concurrent.ExecutorService import java.util.regex.Pattern import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Value import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.aws.cache.AwsEcrAuthToken import io.seqera.wave.service.aws.cache.AwsEcrCache @@ -89,9 +88,6 @@ class AwsEcrService { @Inject private AwsEcrCache cache - @Value('${wave.aws.ecr.cache.duration:24h}') - private Duration cacheDuration - private EcrClient ecrClient(String accessKey, String secretKey, String region) { EcrClient.builder() .region( Region.of(region)) @@ -137,7 +133,7 @@ class AwsEcrService { try { final key = new AwsCreds(accessKey,secretKey,region,isPublic) - return cache.getOrCompute(key, (k) -> load(key), cacheDuration).value + return cache.getOrCompute(key, (k) -> load(key), cache.duration).value } catch (Exception e) { final type = isPublic ? "ECR public" : "ECR" diff --git a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy index 100535a69..11c291a32 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/cache/AwsEcrCache.groovy @@ -67,6 +67,10 @@ class AwsEcrCache extends AbstractTieredCache { return 'aws-ecr-cache/v1' } + Duration getDuration() { + return duration + } + static MoshiEncodeStrategy encoder() { new MoshiEncodeStrategy(factory()) {} }