From 416005cfb4d8a27c3a80165dba162da3b317f079 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 27 Mar 2024 17:27:09 +0100 Subject: [PATCH 01/12] Update default endpoint to api.cloud.seqera.io (#61) Signed-off-by: munishchouhan --- app/src/main/java/io/seqera/wave/cli/App.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/io/seqera/wave/cli/App.java b/app/src/main/java/io/seqera/wave/cli/App.java index 2c56a2d..e4db90b 100644 --- a/app/src/main/java/io/seqera/wave/cli/App.java +++ b/app/src/main/java/io/seqera/wave/cli/App.java @@ -88,7 +88,7 @@ public class App implements Runnable { private static final org.slf4j.Logger log = LoggerFactory.getLogger(App.class); private static final boolean isWindows = System.getProperty("os.name").toLowerCase().contains("windows"); - private static final String DEFAULT_TOWER_ENDPOINT = "https://api.tower.nf"; + private static final String DEFAULT_TOWER_ENDPOINT = "https://api.cloud.seqera.io"; private static final List VALID_PLATFORMS = List.of("amd64", "x86_64", "linux/amd64", "linux/x86_64", "arm64", "linux/arm64"); @@ -103,7 +103,7 @@ public class App implements Runnable { @Option(names = {"--tower-token"}, paramLabel = "''", description = "Tower service access token.") private String towerToken; - @Option(names = {"--tower-endpoint"}, paramLabel = "''", description = "Tower service endpoint e.g. https://api.tower.nf.") + @Option(names = {"--tower-endpoint"}, paramLabel = "''", description = "Tower service endpoint e.g. https://api.cloud.seqera.io.") private String towerEndpoint; @Option(names = {"--tower-workspace-id"}, paramLabel = "''", description = "Tower service workspace ID e.g. 1234567890.") From 9decb2a4a49a3527910b2249bf94b19fc98d778f Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Mon, 8 Apr 2024 16:47:28 +0200 Subject: [PATCH 02/12] change gc to g1gc for linux distribution of cli (#63) * enabled g1gc Signed-off-by: munishchouhan * enabled g1gc only for linux Signed-off-by: munishchouhan --------- Signed-off-by: munishchouhan --- app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build.gradle b/app/build.gradle index 3e5e72f..7d087d0 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -76,7 +76,7 @@ graalvmNative { configurationFileDirectories.from(file('conf')) if (System.env.getOrDefault("PLATFORM", "") == "linux-x86_64") { - buildArgs(['--static', '--libc=musl']) + buildArgs(['--static', '--libc=musl', '--gc=G1']) } javaLauncher = javaToolchains.launcherFor { From a923aacfe23e52903e836dabadae76502e3f96cb Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 11 Apr 2024 11:19:37 +0200 Subject: [PATCH 03/12] Use container-alpha1v2 endpoint (#62) Signed-off-by: Paolo Di Tommaso Signed-off-by: munishchouhan Co-authored-by: munishchouhan --- app/build.gradle | 5 +- app/conf/reflect-config.json | 49 +++- app/src/main/java/io/seqera/wave/cli/App.java | 146 ++++------ .../main/java/io/seqera/wave/cli/Client.java | 50 ++-- .../io/seqera/wave/cli/json/JsonHelper.java | 6 +- .../cli/model/ContainerInspectResponseEx.java | 35 +++ .../wave/cli/model/ContainerSpecEx.java | 44 +++ .../io/seqera/wave/cli/model/LayerRef.java | 36 +++ .../wave/cli/util/DurationConverter.java | 38 +++ .../io/seqera/wave/cli/util/YamlHelper.java | 8 +- .../seqera/wave/cli/AppCondaOptsTest.groovy | 251 +++--------------- .../seqera/wave/cli/AppConfigOptsTest.groovy | 48 ++-- .../seqera/wave/cli/AppSpackOptsTest.groovy | 78 +++--- .../groovy/io/seqera/wave/cli/AppTest.groovy | 73 ++++- .../wave/cli/json/JsonHelperTest.groovy | 15 +- .../wave/cli/util/YamlHelperTest.groovy | 37 ++- 16 files changed, 508 insertions(+), 411 deletions(-) create mode 100644 app/src/main/java/io/seqera/wave/cli/model/ContainerInspectResponseEx.java create mode 100644 app/src/main/java/io/seqera/wave/cli/model/ContainerSpecEx.java create mode 100644 app/src/main/java/io/seqera/wave/cli/model/LayerRef.java create mode 100644 app/src/main/java/io/seqera/wave/cli/util/DurationConverter.java diff --git a/app/build.gradle b/app/build.gradle index 7d087d0..f6aa595 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -21,8 +21,8 @@ repositories { } dependencies { - implementation 'io.seqera:wave-api:0.7.1' - implementation 'io.seqera:wave-utils:0.9.0' + implementation 'io.seqera:wave-api:0.9.1' + implementation 'io.seqera:wave-utils:0.12.0' implementation 'info.picocli:picocli:4.6.1' implementation 'com.squareup.moshi:moshi:1.15.0' implementation 'com.squareup.moshi:moshi-adapters:1.14.0' @@ -38,7 +38,6 @@ dependencies { testImplementation ("org.objenesis:objenesis:3.2") testImplementation ("org.spockframework:spock-core:2.3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } testImplementation ('org.spockframework:spock-junit4:2.3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation 'com.github.tomakehurst:wiremock:2.27.2' } test { diff --git a/app/conf/reflect-config.json b/app/conf/reflect-config.json index 9056b0d..29f267c 100644 --- a/app/conf/reflect-config.json +++ b/app/conf/reflect-config.json @@ -148,6 +148,16 @@ "allDeclaredFields":true, "methods":[{"name":"","parameterTypes":[] }] }, +{ + "name":"io.seqera.wave.api.BuildStatusResponse", + "allDeclaredFields":true, + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"io.seqera.wave.api.BuildStatusResponse$Status", + "fields":[{"name":"COMPLETED"}, {"name":"PENDING"}] +}, { "name":"io.seqera.wave.api.ContainerConfig", "allDeclaredFields":true, @@ -175,6 +185,15 @@ "allDeclaredFields":true, "methods":[{"name":"","parameterTypes":[] }] }, +{ + "name":"io.seqera.wave.api.PackagesSpec", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"io.seqera.wave.api.PackagesSpec$Type", + "fields":[{"name":"CONDA"}, {"name":"SPACK"}] +}, { "name":"io.seqera.wave.api.ServiceInfo", "allDeclaredFields":true, @@ -196,6 +215,12 @@ "queryAllPublicMethods":true, "methods":[{"name":"","parameterTypes":[] }] }, +{ + "name":"io.seqera.wave.api.SubmitContainerTokenResponseBeanInfo" +}, +{ + "name":"io.seqera.wave.api.SubmitContainerTokenResponseCustomizer" +}, { "name":"io.seqera.wave.cli.App", "allDeclaredFields":true, @@ -208,18 +233,40 @@ { "name":"io.seqera.wave.cli.json.DateTimeAdapter", "queryAllDeclaredMethods":true, - "methods":[{"name":"deserializeInstant","parameterTypes":["java.lang.String"] }, {"name":"serializeInstant","parameterTypes":["java.time.Instant"] }] + "methods":[{"name":"deserializeDuration","parameterTypes":["java.lang.String"] }, {"name":"deserializeInstant","parameterTypes":["java.lang.String"] }, {"name":"serializeInstant","parameterTypes":["java.time.Instant"] }] }, { "name":"io.seqera.wave.cli.json.PathAdapter", "queryAllDeclaredMethods":true }, +{ + "name":"io.seqera.wave.cli.model.ContainerInspectResponseEx", + "allDeclaredFields":true +}, +{ + "name":"io.seqera.wave.cli.model.ContainerSpecEx", + "allDeclaredFields":true +}, +{ + "name":"io.seqera.wave.cli.model.LayerRef", + "allDeclaredFields":true +}, { "name":"io.seqera.wave.cli.util.CliVersionProvider", "allDeclaredFields":true, "queryAllDeclaredMethods":true, "methods":[{"name":"","parameterTypes":[] }] }, +{ + "name":"io.seqera.wave.config.CondaOpts", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"io.seqera.wave.config.SpackOpts", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, { "name":"io.seqera.wave.core.spec.ConfigSpec", "allDeclaredFields":true, diff --git a/app/src/main/java/io/seqera/wave/cli/App.java b/app/src/main/java/io/seqera/wave/cli/App.java index e4db90b..bfd3b12 100644 --- a/app/src/main/java/io/seqera/wave/cli/App.java +++ b/app/src/main/java/io/seqera/wave/cli/App.java @@ -30,11 +30,11 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.time.Duration; import java.time.OffsetDateTime; import java.util.Arrays; import java.util.Base64; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import ch.qos.logback.classic.Level; @@ -44,6 +44,7 @@ import io.seqera.wave.api.ContainerInspectRequest; import io.seqera.wave.api.ContainerInspectResponse; import io.seqera.wave.api.ContainerLayer; +import io.seqera.wave.api.PackagesSpec; import io.seqera.wave.api.ServiceInfo; import io.seqera.wave.api.SubmitContainerTokenRequest; import io.seqera.wave.api.SubmitContainerTokenResponse; @@ -51,8 +52,11 @@ import io.seqera.wave.cli.exception.ClientConnectionException; import io.seqera.wave.cli.exception.IllegalCliArgumentException; import io.seqera.wave.cli.json.JsonHelper; +import io.seqera.wave.cli.model.ContainerInspectResponseEx; +import io.seqera.wave.cli.model.ContainerSpecEx; import io.seqera.wave.cli.util.BuildInfo; import io.seqera.wave.cli.util.CliVersionProvider; +import io.seqera.wave.cli.util.DurationConverter; import io.seqera.wave.cli.util.YamlHelper; import io.seqera.wave.config.CondaOpts; import io.seqera.wave.config.SpackOpts; @@ -62,16 +66,6 @@ import picocli.CommandLine; import static io.seqera.wave.cli.util.Checkers.isEmpty; import static io.seqera.wave.cli.util.Checkers.isEnvVar; -import static io.seqera.wave.util.DockerHelper.addPackagesToSpackFile; -import static io.seqera.wave.util.DockerHelper.condaFileFromPackages; -import static io.seqera.wave.util.DockerHelper.condaFileFromPath; -import static io.seqera.wave.util.DockerHelper.condaFileToDockerFile; -import static io.seqera.wave.util.DockerHelper.condaFileToSingularityFile; -import static io.seqera.wave.util.DockerHelper.condaPackagesToDockerFile; -import static io.seqera.wave.util.DockerHelper.condaPackagesToSingularityFile; -import static io.seqera.wave.util.DockerHelper.spackFileToDockerFile; -import static io.seqera.wave.util.DockerHelper.spackFileToSingularityFile; -import static io.seqera.wave.util.DockerHelper.spackPackagesToSpackFile; import static picocli.CommandLine.Command; import static picocli.CommandLine.Option; @@ -124,8 +118,8 @@ public class App implements Runnable { @Option(names = {"--platform"}, paramLabel = "''", description = "Platform to be used for the container build. One of: linux/amd64, linux/arm64.") private String platform; - @Option(names = {"--await"}, paramLabel = "false", description = "Await the container build to be available.") - private boolean await; + @Option(names = {"--await"}, paramLabel = "false", arity = "0..1", description = "Await the container build to be available. you can provide a timeout like --await 10m or 2s, by default its 15 minutes.") + private Duration await; @Option(names = {"--context"}, paramLabel = "''", description = "Directory path where the build context is stored e.g. /some/context/path.") private String contextDir; @@ -205,6 +199,9 @@ public static void main(String[] args) { final App app = new App(); final CommandLine cli = new CommandLine(app); + //register duration converter + cli.registerConverter(Duration.class, new DurationConverter()); + // add examples in help cli .getCommandSpec() @@ -376,7 +373,7 @@ protected void validateArgs() { throw new IllegalCliArgumentException("Context path is not a directory - offending value: " + contextDir); } - if( dryRun && await) + if( dryRun && await != null ) throw new IllegalCliArgumentException("Options --dry-run and --await conflicts each other"); if( !isEmpty(platform) && !VALID_PLATFORMS.contains(platform) ) @@ -392,8 +389,7 @@ protected SubmitContainerTokenRequest createRequest() { return new SubmitContainerTokenRequest() .withContainerImage(image) .withContainerFile(containerFileBase64()) - .withCondaFile(condaFileBase64()) - .withSpackFile(spackFileBase64()) + .withPackages(packagesSpec()) .withContainerPlatform(platform) .withTimestamp(OffsetDateTime.now()) .withBuildRepository(buildRepository) @@ -420,7 +416,8 @@ public void inspect() { ; final ContainerInspectResponse resp = client.inspect(req); - System.out.println(dumpOutput(resp)); + final ContainerSpecEx spec = new ContainerSpecEx(resp.getContainer()); + System.out.println(dumpOutput(new ContainerInspectResponseEx(spec))); } @Override @@ -437,8 +434,8 @@ public void run() { // submit it SubmitContainerTokenResponse resp = client.submit(request); // await build to be completed - if( await ) - client.awaitImage(resp.targetImage); + if( await != null && resp.buildId!=null && !resp.cached ) + client.awaitCompletion(resp.buildId, await); // print the wave container name System.out.println(dumpOutput(resp)); } @@ -576,69 +573,55 @@ private ContainerInspectRequest inspectRequest(String image) { private CondaOpts condaOpts() { return new CondaOpts() .withMambaImage(condaBaseImage) - .withCommands(condaRunCommands); + .withCommands(condaRunCommands) + ; } - protected String containerFileBase64() { - if( !isEmpty(containerFile) ) { - return encodePathBase64(containerFile); - } - - if (!isEmpty(condaFile) || !isEmpty(condaPackages)) { - String result; - final String lock = condaLock(); - if (!isEmpty(lock)) { - result = singularity - ? condaPackagesToSingularityFile(lock, condaChannels(), condaOpts()) - : condaPackagesToDockerFile(lock, condaChannels(), condaOpts()); - } else { - result = singularity - ? condaFileToSingularityFile(condaOpts()) - : condaFileToDockerFile(condaOpts()); - } - return encodeStringBase64(result); - } - - if( !isEmpty(spackFile) || spackPackages!=null ) { - final SpackOpts opts = new SpackOpts() .withCommands(spackRunCommands); - final String result = singularity - ? spackFileToSingularityFile(opts) - : spackFileToDockerFile(opts); - return encodeStringBase64(result); - } + private SpackOpts spackOpts() { + return new SpackOpts() + .withCommands(spackRunCommands); + } - return null; + protected String containerFileBase64() { + return !isEmpty(containerFile) + ? encodePathBase64(containerFile) + : null; } - protected String condaFileBase64() { - if (!isEmpty(condaFile)) { - // parse the attribute as a conda file path *and* append the base packages if any - // note 'channel' is null, because they are expected to be provided in the conda file - final Path path = condaFileFromPath(condaFile, null); - return path != null ? encodePathBase64(path.toString()) : null; + protected PackagesSpec packagesSpec() { + if( !isEmpty(condaFile) ) { + return new PackagesSpec() + .withType(PackagesSpec.Type.CONDA) + .withCondaOpts(condaOpts()) + .withEnvironment(encodePathBase64(condaFile)) + .withChannels(condaChannels()) + ; } - else if (!isEmpty(condaPackages) && isEmpty(condaLock())) { - // create a minimal conda file with package spec from user input - final String packages = condaPackages.stream().collect(Collectors.joining(" ")); - final Path path = condaFileFromPackages(packages, condaChannels()); - return path != null ? encodePathBase64(path.toString()) : null; + + if( !isEmpty(condaPackages) ) { + return new PackagesSpec() + .withType(PackagesSpec.Type.CONDA) + .withCondaOpts(condaOpts()) + .withEntries(condaPackages) + .withChannels(condaChannels()) + ; } - else - return null; - } - protected String spackFileBase64() { if( !isEmpty(spackFile) ) { - // parse the attribute as a spack file path *and* append the base packages if any - return encodePathBase64(addPackagesToSpackFile(spackFile, new SpackOpts()).toString()); + return new PackagesSpec() + .withType(PackagesSpec.Type.SPACK) + .withSpackOpts(spackOpts()) + .withEnvironment(encodePathBase64(spackFile)); } - else if( spackPackages!=null && spackPackages.size()>0 ) { - // create a minimal spack file with package spec from user input - final String packages = spackPackages.stream().collect(Collectors.joining(" ")); - return encodePathBase64(spackPackagesToSpackFile(packages, new SpackOpts()).toString()); + + if( !isEmpty(spackPackages) ) { + return new PackagesSpec() + .withType(PackagesSpec.Type.SPACK) + .withSpackOpts(spackOpts()) + .withEntries(spackPackages); } - else - return null; + + return null; } protected String dumpOutput(SubmitContainerTokenResponse resp) { @@ -651,12 +634,10 @@ protected String dumpOutput(SubmitContainerTokenResponse resp) { if( outputFormat!=null ) throw new IllegalArgumentException("Unexpected output format: "+outputFormat); - return freeze - ? resp.containerImage - : resp.targetImage; + return resp.targetImage; } - protected String dumpOutput(ContainerInspectResponse resp) { + protected String dumpOutput(ContainerInspectResponseEx resp) { if( "json".equals(outputFormat) || outputFormat==null ) { return JsonHelper.toJson(resp); } @@ -696,21 +677,6 @@ protected List condaChannels() { .collect(Collectors.toList()); } - protected String condaLock() { - if( isEmpty(condaPackages) ) - return null; - Optional result = condaPackages - .stream() - .filter(it->it.startsWith("http://") || it.startsWith("https://")) - .findFirst(); - if( !result.isPresent() ) - return null; - if( condaPackages.size()!=1 ) { - throw new IllegalCliArgumentException("No more than one Conda lock remote file can be specified at the same time"); - } - return result.get(); - } - void printInfo() { System.out.println(String.format("Client:")); System.out.println(String.format(" Version : %s", BuildInfo.getVersion())); diff --git a/app/src/main/java/io/seqera/wave/cli/Client.java b/app/src/main/java/io/seqera/wave/cli/Client.java index dbd58f7..eb89bcc 100644 --- a/app/src/main/java/io/seqera/wave/cli/Client.java +++ b/app/src/main/java/io/seqera/wave/cli/Client.java @@ -25,8 +25,10 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; +import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import dev.failsafe.Failsafe; @@ -35,6 +37,7 @@ import dev.failsafe.event.EventListener; import dev.failsafe.event.ExecutionAttemptedEvent; import dev.failsafe.function.CheckedSupplier; +import io.seqera.wave.api.BuildStatusResponse; import io.seqera.wave.api.ContainerInspectRequest; import io.seqera.wave.api.ContainerInspectResponse; import io.seqera.wave.api.ServiceInfo; @@ -112,7 +115,7 @@ ContainerInspectResponse inspect(ContainerInspectRequest request) { SubmitContainerTokenResponse submit(SubmitContainerTokenRequest request) { final String body = JsonHelper.toJson(request); - final URI uri = URI.create(endpoint + "/container-token"); + final URI uri = URI.create(endpoint + "/v1alpha2/container"); log.debug("Wave request: {} - payload: {}", uri, request); final HttpRequest req = HttpRequest.newBuilder() .uri(uri) @@ -194,24 +197,41 @@ protected URI imageToManifestUri(String image) { return URI.create(result); } - protected void awaitImage(String image) { - final URI manifest = imageToManifestUri(image); + void awaitCompletion(String buildId, Duration await) { + log.debug("Waiting for build completion: {} - timeout: {} Seconds", buildId, await.toSeconds()); + final long startTime = Instant.now().toEpochMilli(); + while (!isComplete(buildId)) { + if (System.currentTimeMillis() - startTime > await.toMillis()) { + break; + } + } + } + + protected boolean isComplete(String buildId) { + final String statusEndpoint = endpoint + "/v1alpha1/builds/"+buildId+"/status"; final HttpRequest req = HttpRequest.newBuilder() - .uri(manifest) - .headers(REQUEST_HEADERS) - .timeout(Duration.ofMinutes(5)) + .uri(URI.create(statusEndpoint)) + .headers("Content-Type","application/json") .GET() .build(); - final long begin = System.currentTimeMillis(); - final HttpResponse resp = httpSend(req); - final int code = resp.statusCode(); - if( code>=200 && code<400 ) { - final long delta = System.currentTimeMillis()-begin; - log.debug("Wave container available in {} [{}] {}", delta, code, resp.body()); + + try { + //interval of 10 seconds + TimeUnit.SECONDS.sleep(10); + + final HttpResponse resp = httpSend(req); + log.debug("Wave response: statusCode={}; body={}", resp.statusCode(), resp.body()); + if( resp.statusCode()==200 ) { + BuildStatusResponse result = JsonHelper.fromJson(resp.body(), BuildStatusResponse.class); + return result.status == BuildStatusResponse.Status.COMPLETED; + } + else { + String msg = String.format("Wave invalid response: [%s] %s", resp.statusCode(), resp.body()); + throw new BadClientResponseException(msg); + } } - else { - String message = String.format("Unexpected response for '%s': [%d] %s", manifest, resp.statusCode(), resp.body()); - throw new IllegalStateException(message); + catch (IOException | FailsafeException | InterruptedException e) { + throw new ClientConnectionException("Unable to connect Wave service: " + endpoint, e); } } diff --git a/app/src/main/java/io/seqera/wave/cli/json/JsonHelper.java b/app/src/main/java/io/seqera/wave/cli/json/JsonHelper.java index 2cba8fb..60b9183 100644 --- a/app/src/main/java/io/seqera/wave/cli/json/JsonHelper.java +++ b/app/src/main/java/io/seqera/wave/cli/json/JsonHelper.java @@ -22,9 +22,9 @@ import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import io.seqera.wave.api.ContainerInspectRequest; -import io.seqera.wave.api.ContainerInspectResponse; import io.seqera.wave.api.SubmitContainerTokenRequest; import io.seqera.wave.api.SubmitContainerTokenResponse; +import io.seqera.wave.cli.model.ContainerInspectResponseEx; /** * Helper class to encode and decode JSON payloads @@ -54,8 +54,8 @@ public static String toJson(ContainerInspectRequest request) { return adapter.toJson(request); } - public static String toJson(ContainerInspectResponse response) { - JsonAdapter adapter = moshi.adapter(ContainerInspectResponse.class); + public static String toJson(ContainerInspectResponseEx response) { + JsonAdapter adapter = moshi.adapter(ContainerInspectResponseEx.class); return adapter.toJson(response); } diff --git a/app/src/main/java/io/seqera/wave/cli/model/ContainerInspectResponseEx.java b/app/src/main/java/io/seqera/wave/cli/model/ContainerInspectResponseEx.java new file mode 100644 index 0000000..446c210 --- /dev/null +++ b/app/src/main/java/io/seqera/wave/cli/model/ContainerInspectResponseEx.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.cli.model; + +import io.seqera.wave.api.ContainerInspectResponse; +import io.seqera.wave.core.spec.ContainerSpec; + +/** + * @author Paolo Di Tommaso + */ +public class ContainerInspectResponseEx extends ContainerInspectResponse { + + public ContainerInspectResponseEx(ContainerInspectResponse response) { + super(new ContainerSpecEx(response.getContainer())); + } + + public ContainerInspectResponseEx(ContainerSpec spec) { + super(new ContainerSpecEx(spec)); + } +} diff --git a/app/src/main/java/io/seqera/wave/cli/model/ContainerSpecEx.java b/app/src/main/java/io/seqera/wave/cli/model/ContainerSpecEx.java new file mode 100644 index 0000000..27bf60e --- /dev/null +++ b/app/src/main/java/io/seqera/wave/cli/model/ContainerSpecEx.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.cli.model; + +import java.util.List; + +import io.seqera.wave.core.spec.ContainerSpec; +import io.seqera.wave.core.spec.ObjectRef; + +/** + * Wrapper for {@link ContainerSpec} that replaces + * {@link ObjectRef} with {@link LayerRef} objects + * + * @author Paolo Di Tommaso + */ +public class ContainerSpecEx extends ContainerSpec { + public ContainerSpecEx(ContainerSpec spec) { + super(spec); + // update the layers uri + if( spec.getManifest()!=null && spec.getManifest().getLayers()!=null ) { + List layers = spec.getManifest().getLayers(); + for( int i=0; i + */ +public class LayerRef extends ObjectRef { + + final public String uri; + + public LayerRef(ObjectRef obj, String uri) { + super(obj); + this.uri = uri; + } + +} diff --git a/app/src/main/java/io/seqera/wave/cli/util/DurationConverter.java b/app/src/main/java/io/seqera/wave/cli/util/DurationConverter.java new file mode 100644 index 0000000..79ab74b --- /dev/null +++ b/app/src/main/java/io/seqera/wave/cli/util/DurationConverter.java @@ -0,0 +1,38 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +package io.seqera.wave.cli.util; + +import picocli.CommandLine; + +import java.time.Duration; +/** + * Converter to convert cli argument to duration + * + * @author Munish Chouhan + */ +public class DurationConverter implements CommandLine.ITypeConverter { + @Override + public Duration convert(String value) { + if (value == null || value.trim().isEmpty()) { + return Duration.ofMinutes(15); + } + return Duration.parse("PT" + value.toUpperCase()); + } +} \ No newline at end of file diff --git a/app/src/main/java/io/seqera/wave/cli/util/YamlHelper.java b/app/src/main/java/io/seqera/wave/cli/util/YamlHelper.java index d7c7eeb..9a5b8ef 100644 --- a/app/src/main/java/io/seqera/wave/cli/util/YamlHelper.java +++ b/app/src/main/java/io/seqera/wave/cli/util/YamlHelper.java @@ -21,6 +21,9 @@ import io.seqera.wave.api.ContainerInspectResponse; import io.seqera.wave.api.SubmitContainerTokenResponse; +import io.seqera.wave.cli.model.ContainerInspectResponseEx; +import io.seqera.wave.cli.model.ContainerSpecEx; +import io.seqera.wave.cli.model.LayerRef; import io.seqera.wave.core.spec.ConfigSpec; import io.seqera.wave.core.spec.ContainerSpec; import io.seqera.wave.core.spec.ManifestSpec; @@ -51,7 +54,7 @@ public static String toYaml(SubmitContainerTokenResponse resp) { return yaml.dump(resp); } - public static String toYaml(ContainerInspectResponse resp) { + public static String toYaml(ContainerInspectResponseEx resp) { final DumperOptions opts = new DumperOptions(); opts.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); opts.setAllowReadOnlyProperties(true); @@ -59,9 +62,12 @@ public static String toYaml(ContainerInspectResponse resp) { final Representer representer = new Representer(opts) { { addClassTag(ContainerSpec.class, Tag.MAP); + addClassTag(ContainerSpecEx.class, Tag.MAP); addClassTag(ConfigSpec.class, Tag.MAP); addClassTag(ManifestSpec.class, Tag.MAP); addClassTag(ContainerInspectResponse.class, Tag.MAP); + addClassTag(ContainerInspectResponseEx.class, Tag.MAP); + addClassTag(LayerRef.class, Tag.MAP); representers.put(Instant.class, data -> representScalar(Tag.STR, data.toString())); } }; diff --git a/app/src/test/groovy/io/seqera/wave/cli/AppCondaOptsTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/AppCondaOptsTest.groovy index 1c54c8a..51b5e78 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/AppCondaOptsTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/AppCondaOptsTest.groovy @@ -19,7 +19,9 @@ package io.seqera.wave.cli import java.nio.file.Files +import io.seqera.wave.api.PackagesSpec import io.seqera.wave.cli.exception.IllegalCliArgumentException +import io.seqera.wave.config.CondaOpts import picocli.CommandLine import spock.lang.Specification /** @@ -129,17 +131,21 @@ class AppCondaOptsTest extends Specification { and: def req = app.createRequest() then: - new String(req.containerFile.decodeBase64()) == '''\ - FROM mambaorg/micromamba:1.5.5 - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml \\ - && micromamba install -y -n base conda-forge::procps-ng \\ - && micromamba clean -a -y - USER root - ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - '''.stripIndent() + req.packages.type == PackagesSpec.Type.CONDA and: - new String(req.condaFile.decodeBase64()) == CONDA_RECIPE + new String(req.packages.environment.decodeBase64()) == ''' + name: my-recipe + dependencies: + - one=1.0 + - two:2.0 + '''.stripIndent(true) + and: + req.packages.condaOpts == new CondaOpts(mambaImage: CondaOpts.DEFAULT_MAMBA_IMAGE, basePackages: CondaOpts.DEFAULT_PACKAGES) + req.packages.channels == ['seqera', 'conda-forge', 'bioconda', 'defaults'] + and: + !req.packages.entries + and: + !req.condaFile cleanup: folder?.deleteDir() @@ -156,165 +162,42 @@ class AppCondaOptsTest extends Specification { and: def req = app.createRequest() then: - new String(req.containerFile.decodeBase64()) == '''\ - FROM mambaorg/micromamba:1.5.5 - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml \\ - && micromamba install -y -n base conda-forge::procps-ng \\ - && micromamba clean -a -y - USER root - ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - '''.stripIndent() - and: - new String(req.condaFile.decodeBase64()) == '''\ - channels: - - seqera - - conda-forge - - bioconda - - defaults - dependencies: - - foo - '''.stripIndent(true) - } - - def 'should create docker env from conda lock file' () { - given: - def app = new App() - String[] args = ["--conda-package", "https://host.com/file-lock.yml"] - - when: - new CommandLine(app).parseArgs(args) - and: - def req = app.createRequest() - then: - new String(req.containerFile.decodeBase64()) == '''\ - FROM mambaorg/micromamba:1.5.5 - RUN \\ - micromamba install -y -n base -c seqera -c conda-forge -c bioconda -c defaults -f https://host.com/file-lock.yml \\ - && micromamba install -y -n base conda-forge::procps-ng \\ - && micromamba clean -a -y - USER root - ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - '''.stripIndent() + req.packages.type == PackagesSpec.Type.CONDA + req.packages.entries == ['foo'] and: - req.condaFile == null - } - - def 'should create docker file from conda package and custom options' () { - given: - def app = new App() - String[] args = [ - "--conda-package", "foo", - "--conda-package", "bar", - "--conda-base-image", "my/mamba:latest", - "--conda-channels", "alpha,beta", - "--conda-run-command", "RUN one", - "--conda-run-command", "RUN two", - - ] - - when: - new CommandLine(app).parseArgs(args) + req.packages.condaOpts == new CondaOpts(mambaImage: CondaOpts.DEFAULT_MAMBA_IMAGE, basePackages: CondaOpts.DEFAULT_PACKAGES) + req.packages.channels == ['seqera', 'conda-forge', 'bioconda', 'defaults'] and: - def req = app.createRequest() - then: - new String(req.containerFile.decodeBase64()) == '''\ - FROM my/mamba:latest - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml \\ - && micromamba install -y -n base conda-forge::procps-ng \\ - && micromamba clean -a -y - USER root - ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - RUN one - RUN two - '''.stripIndent() - + !req.packages.environment and: - new String(req.condaFile.decodeBase64()) == '''\ - channels: - - alpha - - beta - dependencies: - - foo - - bar - '''.stripIndent(true) + !req.condaFile } - - def 'should create singularity file from conda file' () { + def 'should create docker env from conda lock file' () { given: - def folder = Files.createTempDirectory('test') - def condaFile = folder.resolve('conda.yml'); - condaFile.text = 'MY CONDA FILE' - and: def app = new App() - String[] args = ['--singularity', "--conda-file", condaFile.toString()] + String[] args = ["--conda-package", "https://host.com/file-lock.yml"] when: new CommandLine(app).parseArgs(args) and: def req = app.createRequest() then: - new String(req.containerFile.decodeBase64()) == '''\ - BootStrap: docker - From: mambaorg/micromamba:1.5.5 - %files - {{wave_context_dir}}/conda.yml /scratch/conda.yml - %post - micromamba install -y -n base -f /scratch/conda.yml - micromamba install -y -n base conda-forge::procps-ng - micromamba clean -a -y - %environment - export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - '''.stripIndent() + req.packages.type == PackagesSpec.Type.CONDA + req.packages.entries == ['https://host.com/file-lock.yml'] and: - new String(req.condaFile.decodeBase64()) == 'MY CONDA FILE' - - cleanup: - folder?.deleteDir() - } - - - def 'should create singularity file from conda package' () { - given: - def app = new App() - String[] args = ['--singularity', "--conda-package", "foo"] - - when: - new CommandLine(app).parseArgs(args) + req.packages.condaOpts == new CondaOpts(mambaImage: CondaOpts.DEFAULT_MAMBA_IMAGE, basePackages: CondaOpts.DEFAULT_PACKAGES) + req.packages.channels == ['seqera', 'conda-forge', 'bioconda', 'defaults'] and: - def req = app.createRequest() - then: - new String(req.containerFile.decodeBase64()) == '''\ - BootStrap: docker - From: mambaorg/micromamba:1.5.5 - %files - {{wave_context_dir}}/conda.yml /scratch/conda.yml - %post - micromamba install -y -n base -f /scratch/conda.yml - micromamba install -y -n base conda-forge::procps-ng - micromamba clean -a -y - %environment - export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - '''.stripIndent() + !req.packages.environment and: - new String(req.condaFile.decodeBase64()) == '''\ - channels: - - seqera - - conda-forge - - bioconda - - defaults - dependencies: - - foo - '''.stripIndent(true) + !req.condaFile } - def 'should create singularity file from conda package and custom options' () { + def 'should create docker file from conda package and custom options' () { given: def app = new App() String[] args = [ - '--singularity', "--conda-package", "foo", "--conda-package", "bar", "--conda-base-image", "my/mamba:latest", @@ -328,75 +211,15 @@ class AppCondaOptsTest extends Specification { and: def req = app.createRequest() then: - new String(req.containerFile.decodeBase64()) == '''\ - BootStrap: docker - From: my/mamba:latest - %files - {{wave_context_dir}}/conda.yml /scratch/conda.yml - %post - micromamba install -y -n base -f /scratch/conda.yml - micromamba install -y -n base conda-forge::procps-ng - micromamba clean -a -y - %environment - export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - %post - RUN one - RUN two - '''.stripIndent(true) - - and: - new String(req.condaFile.decodeBase64()) == '''\ - channels: - - alpha - - beta - dependencies: - - foo - - bar - '''.stripIndent(true) - } - - def 'should create singularity file from conda lock file' () { - given: - def app = new App() - String[] args = ["--conda-package", "https://host.com/file-lock.yml", '--singularity'] - - when: - new CommandLine(app).parseArgs(args) - and: - def req = app.createRequest() - then: - new String(req.containerFile.decodeBase64()) == '''\ - BootStrap: docker - From: mambaorg/micromamba:1.5.5 - %post - micromamba install -y -n base -c seqera -c conda-forge -c bioconda -c defaults -f https://host.com/file-lock.yml - micromamba install -y -n base conda-forge::procps-ng - micromamba clean -a -y - %environment - export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" - '''.stripIndent() + req.packages.type == PackagesSpec.Type.CONDA + req.packages.entries == ['foo','bar'] + req.packages.channels == ['alpha','beta'] and: - req.condaFile == null - } - - - def 'should get conda lock file' () { - expect: - new App(condaPackages: ['https://foo.com/lock.yml']) - .condaLock() == 'https://foo.com/lock.yml' - + req.packages.condaOpts == new CondaOpts(mambaImage: 'my/mamba:latest', basePackages: CondaOpts.DEFAULT_PACKAGES, commands: ['RUN one','RUN two']) and: - new App(condaPackages: ['foo', 'bar']) - .condaLock() == null - + !req.packages.environment and: - new App(condaPackages: null) - .condaLock() == null - - when: - new App(condaPackages: ['foo', 'http://foo.com']) .condaLock() - then: - thrown(IllegalCliArgumentException) + !req.condaFile } diff --git a/app/src/test/groovy/io/seqera/wave/cli/AppConfigOptsTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/AppConfigOptsTest.groovy index 0d208c0..c7889a1 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/AppConfigOptsTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/AppConfigOptsTest.groovy @@ -19,14 +19,13 @@ package io.seqera.wave.cli import java.nio.file.Files -import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.client.WireMock -import com.github.tomakehurst.wiremock.core.WireMockConfiguration +import com.sun.net.httpserver.HttpExchange +import com.sun.net.httpserver.HttpHandler +import com.sun.net.httpserver.HttpServer import io.seqera.wave.api.ContainerConfig import io.seqera.wave.cli.exception.IllegalCliArgumentException import picocli.CommandLine import spock.lang.Specification - /** * Test App config prefixed options * @@ -49,26 +48,6 @@ class AppConfigOptsTest extends Specification { } ''' - WireMockServer wireMockServer - def setup() { - wireMockServer = new WireMockServer(WireMockConfiguration.wireMockConfig().port(8080)) - wireMockServer.start() - - WireMock.stubFor( - WireMock.get(WireMock.urlEqualTo("/api/data")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(CONFIG_JSON) - ) - ) - } - - def cleanup() { - wireMockServer.stop() - } - def "test valid entrypoint"() { given: @@ -218,13 +197,27 @@ class AppConfigOptsTest extends Specification { def "test valid config file from a URL"() { given: + HttpHandler handler = { HttpExchange exchange -> + String body = CONFIG_JSON + exchange.getResponseHeaders().add("Content-Type", "text/json") + exchange.sendResponseHeaders(200, body.size()) + exchange.getResponseBody() << body + exchange.getResponseBody().close() + + } + + HttpServer server = HttpServer.create(new InetSocketAddress(9901), 0); + server.createContext("/", handler); + server.start() + + def app = new App() - String[] args = ["--config-file", "http://localhost:8080/api/data"] + String[] args = ["--config-file", "http://localhost:9901/api/data"] when: new CommandLine(app).parseArgs(args) then: - app.@configFile == "http://localhost:8080/api/data" + app.@configFile == "http://localhost:9901/api/data" when: def config = app.prepareConfig() @@ -235,5 +228,8 @@ class AppConfigOptsTest extends Specification { layer.gzipDigest == "sha256:gzipDigest" layer.tarDigest == "sha256:tarDigest" layer.gzipSize == 100 + + cleanup: + server?.stop(0) } } diff --git a/app/src/test/groovy/io/seqera/wave/cli/AppSpackOptsTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/AppSpackOptsTest.groovy index d2a6d60..d51ed54 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/AppSpackOptsTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/AppSpackOptsTest.groovy @@ -19,7 +19,9 @@ package io.seqera.wave.cli import java.nio.file.Files +import io.seqera.wave.api.PackagesSpec import io.seqera.wave.cli.exception.IllegalCliArgumentException +import io.seqera.wave.config.SpackOpts import picocli.CommandLine import spock.lang.Specification /** @@ -128,10 +130,19 @@ class AppSpackOptsTest extends Specification { and: def req = app.createRequest() then: - new String(req.containerFile.decodeBase64()).startsWith("# Runner image") - + req.packages.type == PackagesSpec.Type.SPACK + and: + new String(req.packages.environment.decodeBase64()) == SPACK_FILE + and: + req.packages.spackOpts == new SpackOpts() + and: + !req.packages.condaOpts + !req.packages.channels + !req.packages.entries + !req.packages.channels and: - new String(req.spackFile.decodeBase64()) == SPACK_FILE + !req.spackFile + !req.containerFile cleanup: folder?.deleteDir() @@ -148,17 +159,18 @@ class AppSpackOptsTest extends Specification { and: def req = app.createRequest() then: - def spec = new String(req.containerFile.decodeBase64()).tokenize('\n') - spec[0] == '# Runner image' - spec[1] == 'FROM {{spack_runner_image}}' - spec[2] == 'COPY --from=builder /opt/spack-env /opt/spack-env' - + req.packages.type == PackagesSpec.Type.SPACK + req.packages.entries == ['foo'] and: - new String(req.spackFile.decodeBase64()) == '''\ - spack: - specs: [foo] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) + !req.packages.environment + and: + req.packages.spackOpts == new SpackOpts() + and: + !req.packages.condaOpts + !req.packages.channels + and: + !req.spackFile + !req.containerFile } def 'should create container file from spack package and custom options' () { @@ -176,39 +188,19 @@ class AppSpackOptsTest extends Specification { and: def req = app.createRequest() then: - new String(req.containerFile.decodeBase64()).startsWith("# Runner image") - new String(req.containerFile.decodeBase64()).contains("RUN one\n") - new String(req.containerFile.decodeBase64()).contains("RUN two\n") - + req.packages.type == PackagesSpec.Type.SPACK + req.packages.entries == ['foo','bar'] and: - new String(req.spackFile.decodeBase64()) == '''\ - spack: - specs: [foo, bar] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - } - - def 'should create container file from spack package with singularity' () { - given: - def app = new App() - String[] args = ["--spack-package", "foo", "--singularity"] - - when: - new CommandLine(app).parseArgs(args) + !req.packages.environment and: - def req = app.createRequest() - then: - def spec = new String(req.containerFile.decodeBase64()).tokenize('\n') - spec[0] == 'Bootstrap: docker' - spec[1] == 'From: {{spack_runner_image}}' - spec[2] == 'stage: final' - + req.packages.spackOpts == new SpackOpts(commands: ['RUN one','RUN two']) and: - new String(req.spackFile.decodeBase64()) == '''\ - spack: - specs: [foo] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) + !req.packages.condaOpts + !req.packages.channels + !req.packages.channels + and: + !req.spackFile + !req.containerFile } } diff --git a/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy index 8cd647d..eeb5c9f 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy @@ -17,14 +17,17 @@ package io.seqera.wave.cli +import io.seqera.wave.cli.util.DurationConverter + import java.nio.file.Files +import java.time.Duration import java.time.Instant -import io.seqera.wave.api.ContainerInspectResponse import io.seqera.wave.api.SubmitContainerTokenResponse +import io.seqera.wave.cli.exception.IllegalCliArgumentException +import io.seqera.wave.cli.model.ContainerInspectResponseEx import io.seqera.wave.core.spec.ContainerSpec import io.seqera.wave.util.TarUtils -import io.seqera.wave.cli.exception.IllegalCliArgumentException import picocli.CommandLine import spock.lang.Specification import spock.lang.Unroll @@ -59,7 +62,8 @@ class AppTest extends Specification { targetImage: 'docker.io/some/repo', containerImage: 'docker.io/some/container', expiration: Instant.ofEpochMilli(1691839913), - buildId: '98765' + buildId: '98765', + cached: true ) when: @@ -68,9 +72,11 @@ class AppTest extends Specification { then: result == '''\ buildId: '98765' + cached: true containerImage: docker.io/some/container containerToken: '12345' expiration: '1970-01-20T13:57:19.913Z' + freeze: null targetImage: docker.io/some/repo '''.stripIndent(true) } @@ -100,13 +106,13 @@ class AppTest extends Specification { def app = new App() String[] args = ["--output", "json"] and: - def resp = new ContainerInspectResponse( new ContainerSpec('docker.io', 'busybox', 'latest', 'sha:12345', null, null, null) ) + def resp = new ContainerInspectResponseEx( new ContainerSpec('docker.io', 'https://docker.io', 'busybox', 'latest', 'sha:12345', null, null) ) when: new CommandLine(app).parseArgs(args) def result = app.dumpOutput(resp) then: - result == '{"container":{"digest":"sha:12345","imageName":"busybox","reference":"latest","registry":"docker.io"}}' + result == '{"container":{"digest":"sha:12345","hostName":"https://docker.io","imageName":"busybox","reference":"latest","registry":"docker.io"}}' } def 'should dump inspect to yaml' () { @@ -114,7 +120,7 @@ class AppTest extends Specification { def app = new App() String[] args = ["--output", "yaml"] and: - def resp = new ContainerInspectResponse( new ContainerSpec('docker.io', 'busybox', 'latest', 'sha:12345', null, null, null) ) + def resp = new ContainerInspectResponseEx( new ContainerSpec('docker.io', 'https://docker.io', 'busybox', 'latest', 'sha:12345', null, null) ) when: new CommandLine(app).parseArgs(args) @@ -124,6 +130,7 @@ class AppTest extends Specification { container: config: null digest: sha:12345 + hostName: https://docker.io imageName: busybox manifest: null reference: latest @@ -188,7 +195,9 @@ class AppTest extends Specification { String[] args = ["-i", "ubuntu:latest","--dry-run", '--await'] when: - new CommandLine(app).parseArgs(args) + def cli = new CommandLine(app) + cli.registerConverter(Duration.class, new DurationConverter()) + cli.parseArgs(args) and: app.validateArgs() then: @@ -269,4 +278,54 @@ class AppTest extends Specification { app.@towerToken == 'xyz' } + def 'should get the correct await duration in minutes'(){ + given: + def app = new App() + String[] args = ["-i", "ubuntu:latest", '--await', '10m'] + + when: + def cli = new CommandLine(app) + cli.registerConverter(Duration.class, new DurationConverter()) + cli.parseArgs(args) + and: + app.validateArgs() + then: + noExceptionThrown() + and: + app.@await == Duration.ofMinutes(10) + } + + def 'should get the correct await duration in seconds'(){ + given: + def app = new App() + String[] args = ["-i", "ubuntu:latest", '--await', '10s'] + + when: + def cli = new CommandLine(app) + cli.registerConverter(Duration.class, new DurationConverter()) + cli.parseArgs(args) + and: + app.validateArgs() + then: + noExceptionThrown() + and: + app.@await == Duration.ofSeconds(10) + } + + def 'should get the default await duration'(){ + given: + def app = new App() + String[] args = ["-i", "ubuntu:latest", '--await'] + + when: + def cli = new CommandLine(app) + cli.registerConverter(Duration.class, new DurationConverter()) + cli.parseArgs(args) + and: + app.validateArgs() + then: + noExceptionThrown() + and: + app.@await == Duration.ofMinutes(15) + } } diff --git a/app/src/test/groovy/io/seqera/wave/cli/json/JsonHelperTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/json/JsonHelperTest.groovy index 5fed98e..4785e74 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/json/JsonHelperTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/json/JsonHelperTest.groovy @@ -18,8 +18,9 @@ package io.seqera.wave.cli.json import io.seqera.wave.api.SubmitContainerTokenRequest -import spock.lang.Specification; - +import io.seqera.wave.cli.model.ContainerInspectResponseEx +import io.seqera.wave.core.spec.ContainerSpec +import spock.lang.Specification /** * @author Paolo Di Tommaso */ @@ -43,5 +44,15 @@ class JsonHelperTest extends Specification { result.containerImage == 'quay.io/nextflow/bash:latest' } + def 'should convert response to json' () { + given: + def spec = new ContainerSpec('docker.io', 'https://docker.io', 'ubuntu', '22.04', 'sha:12345', null, null) + def resp = new ContainerInspectResponseEx(spec) + + when: + def result = JsonHelper.toJson(resp) + then: + result == '{"container":{"digest":"sha:12345","hostName":"https://docker.io","imageName":"ubuntu","reference":"22.04","registry":"docker.io"}}' + } } diff --git a/app/src/test/groovy/io/seqera/wave/cli/util/YamlHelperTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/util/YamlHelperTest.groovy index 5aa8879..94ea941 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/util/YamlHelperTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/util/YamlHelperTest.groovy @@ -17,12 +17,13 @@ package io.seqera.wave.cli.util - import java.time.Instant -import io.seqera.wave.api.ContainerInspectResponse import io.seqera.wave.api.SubmitContainerTokenResponse +import io.seqera.wave.cli.model.ContainerInspectResponseEx import io.seqera.wave.core.spec.ContainerSpec +import io.seqera.wave.core.spec.ManifestSpec +import io.seqera.wave.core.spec.ObjectRef import spock.lang.Specification /** * @@ -37,7 +38,9 @@ class YamlHelperTest extends Specification { targetImage: 'docker.io/some/repo', containerImage: 'docker.io/some/container', expiration: Instant.ofEpochMilli(1691839913), - buildId: '98765' + buildId: '98765', + cached: false, + freeze: false ) when: @@ -45,17 +48,21 @@ class YamlHelperTest extends Specification { then: result == '''\ buildId: '98765' + cached: false containerImage: docker.io/some/container containerToken: '12345' expiration: '1970-01-20T13:57:19.913Z' + freeze: false targetImage: docker.io/some/repo '''.stripIndent(true) } def 'should convert response to yaml' () { given: - def spec = new ContainerSpec('docker.io','ubuntu','22.04','sha:12345', null, null, null) - def resp = new ContainerInspectResponse(spec) + def layers = [ new ObjectRef('text', 'sha256:12345', 100, null), new ObjectRef('text', 'sha256:67890', 200, null) ] + def manifest = new ManifestSpec(2, 'some/media', null, layers, [one: '1', two:'2']) + def spec = new ContainerSpec('docker.io', 'https://docker.io', 'ubuntu','22.04','sha:12345', null, manifest) + def resp = new ContainerInspectResponseEx(spec) when: def result = YamlHelper.toYaml(resp) @@ -64,8 +71,26 @@ class YamlHelperTest extends Specification { container: config: null digest: sha:12345 + hostName: https://docker.io imageName: ubuntu - manifest: null + manifest: + annotations: + one: '1' + two: '2' + config: null + layers: + - annotations: null + digest: sha256:12345 + mediaType: text + size: 100 + uri: https://docker.io/v2/ubuntu/blobs/sha256:12345 + - annotations: null + digest: sha256:67890 + mediaType: text + size: 200 + uri: https://docker.io/v2/ubuntu/blobs/sha256:67890 + mediaType: some/media + schemaVersion: 2 reference: '22.04' registry: docker.io '''.stripIndent(true) From 26bb08de873c66a4bb4e6c48cf8d26850737a3e3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 11 Apr 2024 12:01:35 +0200 Subject: [PATCH 04/12] Add generative containers Signed-off-by: Paolo Di Tommaso --- app/build.gradle | 1 + app/conf/proxy-config.json | 3 + app/conf/reflect-config.json | 85 ++++++++++++++ app/conf/resource-config.json | 6 + app/src/main/java/io/seqera/wave/cli/App.java | 27 ++++- .../io/seqera/wave/cli/util/GptHelper.java | 107 ++++++++++++++++++ .../io/seqera/wave/cli/util/StreamHelper.java | 56 +++++++++ .../groovy/io/seqera/wave/cli/AppTest.groovy | 12 ++ .../seqera/wave/cli/util/GptHelperTest.groovy | 50 ++++++++ .../wave/cli/util/StreamHelperTest.groovy | 34 ++++++ 10 files changed, 378 insertions(+), 3 deletions(-) create mode 100644 app/src/main/java/io/seqera/wave/cli/util/GptHelper.java create mode 100644 app/src/main/java/io/seqera/wave/cli/util/StreamHelper.java create mode 100644 app/src/test/groovy/io/seqera/wave/cli/util/GptHelperTest.groovy create mode 100644 app/src/test/groovy/io/seqera/wave/cli/util/StreamHelperTest.groovy diff --git a/app/build.gradle b/app/build.gradle index f6aa595..f719947 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -29,6 +29,7 @@ dependencies { implementation 'dev.failsafe:failsafe:3.1.0' implementation 'org.apache.commons:commons-lang3:3.12.0' implementation 'org.yaml:snakeyaml:2.1' + implementation 'dev.langchain4j:langchain4j-open-ai:0.29.0' annotationProcessor 'info.picocli:picocli-codegen:4.6.1' testImplementation "org.codehaus.groovy:groovy:3.0.15" diff --git a/app/conf/proxy-config.json b/app/conf/proxy-config.json index 0d4f101..f24b201 100644 --- a/app/conf/proxy-config.json +++ b/app/conf/proxy-config.json @@ -1,2 +1,5 @@ [ + { + "interfaces":["dev.ai4j.openai4j.OpenAiApi"] + } ] diff --git a/app/conf/reflect-config.json b/app/conf/reflect-config.json index 29f267c..2bda58f 100644 --- a/app/conf/reflect-config.json +++ b/app/conf/reflect-config.json @@ -140,6 +140,71 @@ "name":"com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "name":"dev.ai4j.openai4j.chat.AssistantMessage", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.ChatCompletionChoice", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.ChatCompletionRequest", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.ChatCompletionResponse", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.Delta", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.Function", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.FunctionCall", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.Parameters", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.ResponseFormat", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.Tool", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.ToolCall", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.chat.ToolChoice", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"dev.ai4j.openai4j.shared.Usage", + "allDeclaredFields":true, + "methods":[{"name":"","parameterTypes":[] }] +}, { "name":"groovy.lang.Closure" }, @@ -310,6 +375,10 @@ { "name":"java.io.FilePermission" }, +{ + "name":"java.lang.Class", + "methods":[{"name":"getRecordComponents","parameterTypes":[] }, {"name":"isRecord","parameterTypes":[] }] +}, { "name":"java.lang.Object", "allDeclaredFields":true, @@ -336,6 +405,14 @@ "name":"java.lang.Thread", "fields":[{"name":"threadLocalRandomProbe"}] }, +{ + "name":"java.lang.invoke.MethodHandles$Lookup", + "methods":[{"name":"","parameterTypes":["java.lang.Class","int"] }] +}, +{ + "name":"java.lang.reflect.RecordComponent", + "methods":[{"name":"getName","parameterTypes":[] }, {"name":"getType","parameterTypes":[] }] +}, { "name":"java.net.NetPermission" }, @@ -463,6 +540,10 @@ { "name":"java.util.Date" }, +{ + "name":"java.util.ImmutableCollections$MapN", + "methods":[{"name":"","parameterTypes":[] }] +}, { "name":"java.util.PropertyPermission" }, @@ -601,6 +682,10 @@ "name":"sun.security.ssl.SSLContextImpl$DefaultSSLContext", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "name":"sun.security.ssl.SSLContextImpl$TLSContext", + "methods":[{"name":"","parameterTypes":[] }] +}, { "name":"sun.security.ssl.TrustManagerFactoryImpl$PKIXFactory", "methods":[{"name":"","parameterTypes":[] }] diff --git a/app/conf/resource-config.json b/app/conf/resource-config.json index 1bce942..c9ede61 100644 --- a/app/conf/resource-config.json +++ b/app/conf/resource-config.json @@ -4,6 +4,10 @@ "pattern":"\\QMETA-INF/build-info.properties\\E" }, { "pattern":"\\QMETA-INF/services/ch.qos.logback.classic.spi.Configurator\\E" + }, { + "pattern":"\\QMETA-INF/services/dev.ai4j.openai4j.spi.OpenAiClientBuilderFactory\\E" + }, { + "pattern":"\\QMETA-INF/services/dev.langchain4j.model.openai.spi.OpenAiChatModelBuilderFactory\\E" }, { "pattern":"\\QMETA-INF/services/java.lang.System$LoggerFinder\\E" }, { @@ -18,6 +22,8 @@ "pattern":"\\QMETA-INF/services/javax.xml.parsers.SAXParserFactory\\E" }, { "pattern":"\\QMETA-INF/services/org.slf4j.spi.SLF4JServiceProvider\\E" + }, { + "pattern":"\\Qcom/knuddels/jtokkit/cl100k_base.tiktoken\\E" }, { "pattern":"\\Qio/seqera/wave/cli/usage-examples.txt\\E" }, { diff --git a/app/src/main/java/io/seqera/wave/cli/App.java b/app/src/main/java/io/seqera/wave/cli/App.java index bfd3b12..56b5884 100644 --- a/app/src/main/java/io/seqera/wave/cli/App.java +++ b/app/src/main/java/io/seqera/wave/cli/App.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.time.Duration; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.List; @@ -57,6 +58,7 @@ import io.seqera.wave.cli.util.BuildInfo; import io.seqera.wave.cli.util.CliVersionProvider; import io.seqera.wave.cli.util.DurationConverter; +import io.seqera.wave.cli.util.GptHelper; import io.seqera.wave.cli.util.YamlHelper; import io.seqera.wave.config.CondaOpts; import io.seqera.wave.config.SpackOpts; @@ -66,6 +68,7 @@ import picocli.CommandLine; import static io.seqera.wave.cli.util.Checkers.isEmpty; import static io.seqera.wave.cli.util.Checkers.isEnvVar; +import static io.seqera.wave.cli.util.StreamHelper.tryReadStdin; import static picocli.CommandLine.Command; import static picocli.CommandLine.Option; @@ -192,7 +195,21 @@ public class App implements Runnable { private boolean inspect; @Option(names = {"--include"}, paramLabel = "false", description = "Include one or more containers in the specified base image") - List includes; + private List includes; + + @CommandLine.Parameters + List prompt; + + static private String[] makeArgs(String[] args) { + String stdin = tryReadStdin(); + if( stdin==null ) + return args; + + List result = new ArrayList<>(Arrays.asList(args)); + result.add("--"); + result.add(stdin); + return result.toArray(new String[args.length+2]); + } public static void main(String[] args) { try { @@ -208,7 +225,7 @@ public static void main(String[] args) { .usageMessage() .footer(readExamples("usage-examples.txt")); - final CommandLine.ParseResult result = cli.parseArgs(args); + final CommandLine.ParseResult result = cli.parseArgs(makeArgs(args)); if( result.matchedArgs().size()==0 || result.isUsageHelpRequested() ) { cli.usage(System.out); } @@ -290,7 +307,7 @@ protected void validateArgs() { if( !isEmpty(image) && !isEmpty(containerFile) ) throw new IllegalCliArgumentException("Argument --image and --containerfile conflict each other - Specify an image name or a container file for the container to be provisioned"); - if( isEmpty(image) && isEmpty(containerFile) && isEmpty(condaFile) && condaPackages==null && isEmpty(spackFile) && spackPackages ==null ) + if( isEmpty(image) && isEmpty(containerFile) && isEmpty(condaFile) && condaPackages==null && isEmpty(spackFile) && spackPackages==null && isEmpty(prompt) ) throw new IllegalCliArgumentException("Provide either a image name or a container file for the Wave container to be provisioned"); if( freeze && isEmpty(buildRepository) ) @@ -621,6 +638,10 @@ protected PackagesSpec packagesSpec() { .withEntries(spackPackages); } + if( !isEmpty(prompt) ) { + return GptHelper.grabPackages(prompt.stream().collect(Collectors.joining(" "))); + } + return null; } diff --git a/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java b/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java new file mode 100644 index 0000000..b2fa17a --- /dev/null +++ b/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java @@ -0,0 +1,107 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.cli.util; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.agent.tool.ToolParameters; +import dev.langchain4j.agent.tool.ToolSpecification; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.model.openai.OpenAiChatModel; +import dev.langchain4j.model.output.Response; +import io.seqera.wave.api.PackagesSpec; +import io.seqera.wave.cli.json.JsonHelper; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Paolo Di Tommaso + */ +public class GptHelper { + + private static Logger log = LoggerFactory.getLogger(GptHelper.class); + + static private OpenAiChatModel client() { + String key = System.getenv("OPENAI_API_KEY"); + if( StringUtils.isEmpty(key) ) + throw new IllegalArgumentException("Missing OPENAI_API_KEY environment variable"); + String model = System.getenv("OPENAI_MODEL"); + if( model==null ) + model = "gpt-3.5-turbo"; + + return OpenAiChatModel.builder() + .apiKey(key) + .modelName(model) + .build(); + } + + + static public PackagesSpec grabPackages(String prompt) { + final Map items = Map.of("type","string", "description", "A Conda package specification provided as the pair name and version, separated by the equals character, for example: foo=1.2.3"); + final Map PACKAGES = Map.of("type", "array", "description", "A list of one more Conda package", "items", items); + final Map CHANNELS = Map.of("type", "array", "description", "A list of one more Conda channels", "items", Map.of("type", "string", "description", "A Conda channel name")); + final Map> properties = Map.of("packages", PACKAGES, "channels", CHANNELS); + final ToolParameters params = ToolParameters + .builder() + .properties(properties) + .required(List.of("packages")) + .build(); + final ToolSpecification toolSpec = ToolSpecification + .builder() + .name("wave_container") + .description(prompt) + .parameters(params) + .build(); + final AiMessage msg = AiMessage.from(prompt); + + final OpenAiChatModel client = client(); + final Response resp = client.generate(List.of(msg), toolSpec); + if( Checkers.isEmpty(resp.content().toolExecutionRequests()) ) + throw new IllegalArgumentException("Unable to resolve container for prompt: " + prompt); + ToolExecutionRequest tool = resp.content().toolExecutionRequests().get(0); + String json = tool.arguments(); + log.debug("GPT response: {}", json); + + return jsonToPackageSpec(json); + } + + static protected PackagesSpec jsonToPackageSpec(String json) { + try { + Map object = JsonHelper.fromJson(json, Map.class); + List packages = (List) object.get("packages"); + if( Checkers.isEmpty(packages) ) + throw new IllegalArgumentException("Unable to resolve packages from json response: " + json); + List channels = (List) object.get("channels"); + if( Checkers.isEmpty(channels) ) + channels = List.of("bioconda","conda-forge","defaults"); + return new PackagesSpec() + .withType(PackagesSpec.Type.CONDA) + .withEntries(packages) + .withChannels(channels); + } + catch (IOException e) { + throw new IllegalArgumentException("Unable to parse json object: " + json); + } + } + +} diff --git a/app/src/main/java/io/seqera/wave/cli/util/StreamHelper.java b/app/src/main/java/io/seqera/wave/cli/util/StreamHelper.java new file mode 100644 index 0000000..0beedfb --- /dev/null +++ b/app/src/main/java/io/seqera/wave/cli/util/StreamHelper.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.cli.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Paolo Di Tommaso + */ +public class StreamHelper { + + private static final Logger log = LoggerFactory.getLogger(StreamHelper.class); + + static public String tryReadStdin() { + return tryReadStream(System.in); + } + + static public String tryReadStream(InputStream stream) { + try { + if( stream.available()==0 ) + return null; + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int len; + while( (len=stream.read(buffer))!=-1 ) { + result.write(buffer,0,len); + } + return new String(result.toByteArray()); + } + catch (IOException e){ + log.debug("Unable to read system.in", e); + } + return null; + } + +} diff --git a/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy index eeb5c9f..ec5128b 100644 --- a/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy +++ b/app/src/test/groovy/io/seqera/wave/cli/AppTest.groovy @@ -328,4 +328,16 @@ class AppTest extends Specification { and: app.@await == Duration.ofMinutes(15) } + + def 'should generate a container' () { + given: + def app = new App() + String[] args = [ 'Get a docker container'] + + when: + new CommandLine(app).parseArgs(args) + then: + app.prompt == ['Get a docker container'] + } + } diff --git a/app/src/test/groovy/io/seqera/wave/cli/util/GptHelperTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/util/GptHelperTest.groovy new file mode 100644 index 0000000..8f03686 --- /dev/null +++ b/app/src/test/groovy/io/seqera/wave/cli/util/GptHelperTest.groovy @@ -0,0 +1,50 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.cli.util + +import io.seqera.wave.api.PackagesSpec +import spock.lang.Requires +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class GptHelperTest extends Specification { + + + def 'should map json to spec'() { + given: + def JSON = ''' + {"packages":["multiqc=1.17","samtools"],"channels":["conda-forge"]} + ''' + when: + def spec = GptHelper.jsonToPackageSpec(JSON) + then: + spec.entries == ['multiqc=1.17', 'samtools'] + spec.channels == ['conda-forge'] + } + + @Requires({ System.getenv('OPENAI_API_KEY') }) + def 'should get a package spec from a prompt' () { + when: + def spec = GptHelper.grabPackages("Give me a container image for multiqc 1.15") + then: + spec == new PackagesSpec(type: PackagesSpec.Type.CONDA, entries: ['multiqc=1.15'], channels: ['bioconda','conda-forge','defaults']) + } +} diff --git a/app/src/test/groovy/io/seqera/wave/cli/util/StreamHelperTest.groovy b/app/src/test/groovy/io/seqera/wave/cli/util/StreamHelperTest.groovy new file mode 100644 index 0000000..b4e28ed --- /dev/null +++ b/app/src/test/groovy/io/seqera/wave/cli/util/StreamHelperTest.groovy @@ -0,0 +1,34 @@ +/* + * Copyright 2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.cli.util + + +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class StreamHelperTest extends Specification { + + def 'should read from stream' () { + expect: + StreamHelper.tryReadStream(new ByteArrayInputStream('Hello\nworld!'.bytes)) == 'Hello\nworld!' + StreamHelper.tryReadStream(new ByteArrayInputStream('Hello\nworld!\n\n'.bytes)) == 'Hello\nworld!\n\n' + } + +} From f35a86d83c2507a0e62ffd1e3e41d74dbd26a39a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 11 Apr 2024 12:08:58 +0200 Subject: [PATCH 05/12] [release] bump version 1.3.0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 26aaba0..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/changelog.txt b/changelog.txt index 2b72a23..e1a6eb1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,13 @@ WAVE CLI CHANGE-LOG =================== +1.3.0 - 11 Apr 2024 +- Add generative containers [26bb08de] +- Use container-alpha1v2 endpoint (#62) [a923aacf] +- change gc to g1gc for linux distribution of cli (#63) [9decb2a4] +- Update default endpoint to api.cloud.seqera.io (#61) [416005cf] +- bump github actions to v4 (#58) [2d0e2672] +- Handle client connection errors [f5d6d9a5] + 1.2.0 - 12 Feb 2024 - Add support for container includes option [01d5904d] - Add support for inspect option [a29f905f] From e119be2a481daad75b0d76cec23b87842c731165 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Thu, 11 Apr 2024 16:05:35 +0200 Subject: [PATCH 06/12] fix release and bump actions/download-artifact@v4 (#64) --- .github/workflows/build.yml | 4 ++-- jreleaser.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b672e1e..b0f2ca1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,7 +62,7 @@ jobs: if: ${{ matrix.fatjar }} uses: actions/upload-artifact@v4 with: - name: wave_jar + name: wave-jar path: ./app/build/libs/wave.jar - name: Build static native @@ -128,7 +128,7 @@ jobs: fetch-depth: 0 - name: Download all build artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 - name: Setup Java for JReleaser uses: actions/setup-java@v4 diff --git a/jreleaser.yml b/jreleaser.yml index abdff82..c273e25 100644 --- a/jreleaser.yml +++ b/jreleaser.yml @@ -104,7 +104,7 @@ distributions: wave-cli-jar: type: SINGLE_JAR artifacts: - - path: "wave_jar/wave.jar" + - path: "wave-jar/wave.jar" transform: "wave-{{projectEffectiveVersion}}.jar" packagers: From 4b081c602e102197b162784735913c40474154ac Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 11 Apr 2024 16:41:36 +0200 Subject: [PATCH 07/12] [release] bump version 1.3.0 Signed-off-by: Paolo Di Tommaso From b2cfdd3f37c5a6087deb624f2ca8b1af61ee5a6e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 20 Apr 2024 18:43:14 +0200 Subject: [PATCH 08/12] Improve error handling Signed-off-by: Paolo Di Tommaso --- .../java/io/seqera/wave/cli/util/GptHelper.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java b/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java index b2fa17a..d1f1e50 100644 --- a/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java +++ b/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java @@ -29,6 +29,7 @@ import dev.langchain4j.model.openai.OpenAiChatModel; import dev.langchain4j.model.output.Response; import io.seqera.wave.api.PackagesSpec; +import io.seqera.wave.cli.exception.BadClientResponseException; import io.seqera.wave.cli.json.JsonHelper; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -39,7 +40,7 @@ */ public class GptHelper { - private static Logger log = LoggerFactory.getLogger(GptHelper.class); + private static final Logger log = LoggerFactory.getLogger(GptHelper.class); static private OpenAiChatModel client() { String key = System.getenv("OPENAI_API_KEY"); @@ -52,11 +53,21 @@ static private OpenAiChatModel client() { return OpenAiChatModel.builder() .apiKey(key) .modelName(model) + .maxRetries(1) .build(); } - static public PackagesSpec grabPackages(String prompt) { + try { + return grabPackages0(prompt); + } + catch (RuntimeException e) { + String msg = "Unexpected OpenAI response - cause: " + e.getMessage(); + throw new BadClientResponseException(msg, e); + } + } + + static PackagesSpec grabPackages0(String prompt) { final Map items = Map.of("type","string", "description", "A Conda package specification provided as the pair name and version, separated by the equals character, for example: foo=1.2.3"); final Map PACKAGES = Map.of("type", "array", "description", "A list of one more Conda package", "items", items); final Map CHANNELS = Map.of("type", "array", "description", "A list of one more Conda channels", "items", Map.of("type", "string", "description", "A Conda channel name")); From 110ad94d6e0d13098b02e8c0e6e066961a79f898 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 22 Apr 2024 18:31:42 +0200 Subject: [PATCH 09/12] Update test dependecies Signed-off-by: Paolo Di Tommaso --- app/build.gradle | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index f719947..701650c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -32,9 +32,10 @@ dependencies { implementation 'dev.langchain4j:langchain4j-open-ai:0.29.0' annotationProcessor 'info.picocli:picocli-codegen:4.6.1' - testImplementation "org.codehaus.groovy:groovy:3.0.15" - testImplementation "org.codehaus.groovy:groovy-nio:3.0.15" - testImplementation ("org.codehaus.groovy:groovy-test:3.0.17") + testImplementation "org.codehaus.groovy:groovy:3.0.21" + testImplementation "org.codehaus.groovy:groovy-nio:3.0.21" + testImplementation ("org.codehaus.groovy:groovy-test:3.0.21") + testImplementation ("org.codehaus.groovy:groovy-json:3.0.21") testImplementation ("cglib:cglib-nodep:3.3.0") testImplementation ("org.objenesis:objenesis:3.2") testImplementation ("org.spockframework:spock-core:2.3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } From 85a01baa7cf297d92c59a8cf450b0c685a33bfee Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 22 Apr 2024 19:56:17 +0200 Subject: [PATCH 10/12] Improve Gpt helper class Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/cli/util/GptHelper.java | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java b/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java index d1f1e50..860b628 100644 --- a/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java +++ b/app/src/main/java/io/seqera/wave/cli/util/GptHelper.java @@ -32,6 +32,7 @@ import io.seqera.wave.cli.exception.BadClientResponseException; import io.seqera.wave.cli.json.JsonHelper; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,20 +69,11 @@ static public PackagesSpec grabPackages(String prompt) { } static PackagesSpec grabPackages0(String prompt) { - final Map items = Map.of("type","string", "description", "A Conda package specification provided as the pair name and version, separated by the equals character, for example: foo=1.2.3"); - final Map PACKAGES = Map.of("type", "array", "description", "A list of one more Conda package", "items", items); - final Map CHANNELS = Map.of("type", "array", "description", "A list of one more Conda channels", "items", Map.of("type", "string", "description", "A Conda channel name")); - final Map> properties = Map.of("packages", PACKAGES, "channels", CHANNELS); - final ToolParameters params = ToolParameters - .builder() - .properties(properties) - .required(List.of("packages")) - .build(); final ToolSpecification toolSpec = ToolSpecification .builder() .name("wave_container") - .description(prompt) - .parameters(params) + .description("This function get a container with one or more tools specified via Conda packages. If the container image does not yet exists it does create it to fulfill the requirement") + .parameters(getToolParameters()) .build(); final AiMessage msg = AiMessage.from(prompt); @@ -96,6 +88,27 @@ static PackagesSpec grabPackages0(String prompt) { return jsonToPackageSpec(json); } + protected static ToolParameters getToolParameters() { + return ToolParameters + .builder() + .properties(getToolProperties()) + .required(List.of("packages")) + .build(); + } + + @NotNull + protected static Map> getToolProperties() { + final Map PACKAGES = Map.of( + "type", "array", + "description", "A list of one more Conda package", + "items", Map.of("type","string", "description", "A Conda package specification provided as the pair name and version, separated by the equals character, for example: foo=1.2.3")); + final Map CHANNELS = Map.of( + "type", "array", + "description", "A list of one more Conda channels", + "items", Map.of("type", "string", "description", "A Conda channel name")); + return Map.of("packages", PACKAGES, "channels", CHANNELS); + } + static protected PackagesSpec jsonToPackageSpec(String json) { try { Map object = JsonHelper.fromJson(json, Map.class); From 76c927664cf79d359f4e538b66b26e98471e0f7b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 22 Apr 2024 20:01:22 +0200 Subject: [PATCH 11/12] Remove build repository check Signed-off-by: Paolo Di Tommaso --- app/src/main/java/io/seqera/wave/cli/App.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/src/main/java/io/seqera/wave/cli/App.java b/app/src/main/java/io/seqera/wave/cli/App.java index 56b5884..cb7843a 100644 --- a/app/src/main/java/io/seqera/wave/cli/App.java +++ b/app/src/main/java/io/seqera/wave/cli/App.java @@ -310,9 +310,6 @@ protected void validateArgs() { if( isEmpty(image) && isEmpty(containerFile) && isEmpty(condaFile) && condaPackages==null && isEmpty(spackFile) && spackPackages==null && isEmpty(prompt) ) throw new IllegalCliArgumentException("Provide either a image name or a container file for the Wave container to be provisioned"); - if( freeze && isEmpty(buildRepository) ) - throw new IllegalCliArgumentException("Specify the build repository where the freeze container will be pushed by using the --build-repo option"); - if( isEmpty(towerToken) && !isEmpty(buildRepository) ) throw new IllegalCliArgumentException("Specify the Tower access token required to authenticate the access to the build repository either by using the --tower-token option or the TOWER_ACCESS_TOKEN environment variable"); From f7823d7bd49d79079e165b85d0c272897a9e47ec Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 22 Apr 2024 20:03:00 +0200 Subject: [PATCH 12/12] [release] bump version 1.3.1 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f0bb29e..3a3cd8c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1 diff --git a/changelog.txt b/changelog.txt index e1a6eb1..52301f3 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,11 @@ WAVE CLI CHANGE-LOG =================== +1.3.1 - 22 Apr 2024 +- Remove build repository check [76c92766] +- Improve error handling [b2cfdd3f] +- Improve Gpt helper class [85a01baa] +- Bump groovy 3.0.21 for tests [110ad94d] + 1.3.0 - 11 Apr 2024 - Add generative containers [26bb08de] - Use container-alpha1v2 endpoint (#62) [a923aacf]