Skip to content

Commit

Permalink
Add support for Container mirror and status
Browse files Browse the repository at this point in the history
Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
  • Loading branch information
pditommaso committed Oct 1, 2024
1 parent 1dec71c commit 270a412
Show file tree
Hide file tree
Showing 9 changed files with 296 additions and 34 deletions.
2 changes: 1 addition & 1 deletion app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ repositories {
}

dependencies {
implementation 'io.seqera:wave-api:0.12.0'
implementation 'io.seqera:wave-api:0.13.0-B1'
implementation 'io.seqera:wave-utils:0.12.0'
implementation 'info.picocli:picocli:4.6.1'
implementation 'com.squareup.moshi:moshi:1.15.0'
Expand Down
30 changes: 29 additions & 1 deletion app/conf/reflect-config.json
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@
},
{
"name":"io.seqera.wave.api.BuildStatusResponse$Status",
"fields":[{"name":"COMPLETED"}, {"name":"PENDING"}]
"fields":[{"name":"BUILDING"}, {"name":"COMPLETED"}, {"name":"PENDING"}, {"name":"SCANNING"}]
},
{
"name":"io.seqera.wave.api.ContainerConfig",
Expand Down Expand Up @@ -250,6 +250,15 @@
"allDeclaredFields":true,
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"name":"io.seqera.wave.api.ContainerStatus",
"fields":[{"name":"BUILDING"}, {"name":"PENDING"}, {"name":"READY"}, {"name":"SCANNING"}]
},
{
"name":"io.seqera.wave.api.ContainerStatusResponse",
"allDeclaredFields":true,
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"name":"io.seqera.wave.api.PackagesSpec",
"allDeclaredFields":true,
Expand All @@ -259,6 +268,14 @@
"name":"io.seqera.wave.api.PackagesSpec$Type",
"fields":[{"name":"CONDA"}, {"name":"SPACK"}]
},
{
"name":"io.seqera.wave.api.ScanLevel",
"fields":[{"name":"critical"}, {"name":"high"}, {"name":"low"}, {"name":"medium"}]
},
{
"name":"io.seqera.wave.api.ScanMode",
"fields":[{"name":"async"}, {"name":"lazy"}, {"name":"none"}, {"name":"off"}, {"name":"sync"}]
},
{
"name":"io.seqera.wave.api.ServiceInfo",
"allDeclaredFields":true,
Expand Down Expand Up @@ -326,6 +343,17 @@
"name":"io.seqera.wave.cli.model.LayerRef",
"allDeclaredFields":true
},
{
"name":"io.seqera.wave.cli.model.SubmitContainerTokenResponseEx",
"allDeclaredFields":true,
"queryAllPublicMethods":true
},
{
"name":"io.seqera.wave.cli.model.SubmitContainerTokenResponseExBeanInfo"
},
{
"name":"io.seqera.wave.cli.model.SubmitContainerTokenResponseExCustomizer"
},
{
"name":"io.seqera.wave.cli.util.CliVersionProvider",
"allDeclaredFields":true,
Expand Down
26 changes: 21 additions & 5 deletions app/src/main/java/io/seqera/wave/cli/App.java
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,11 @@
import io.seqera.wave.cli.exception.BadClientResponseException;
import io.seqera.wave.cli.exception.ClientConnectionException;
import io.seqera.wave.cli.exception.IllegalCliArgumentException;
import io.seqera.wave.cli.exception.ReadyTimeoutException;
import io.seqera.wave.cli.json.JsonHelper;
import io.seqera.wave.cli.model.ContainerInspectResponseEx;
import io.seqera.wave.cli.model.ContainerSpecEx;
import io.seqera.wave.cli.model.SubmitContainerTokenResponseEx;
import io.seqera.wave.cli.util.BuildInfo;
import io.seqera.wave.cli.util.CliVersionProvider;
import io.seqera.wave.cli.util.DurationConverter;
Expand Down Expand Up @@ -197,6 +199,12 @@ public class App implements Runnable {
@Option(names = {"--mirror-to"}, paramLabel = "false", description = "Specify registry where the container should be mirrored e.g. 'docker.io'")
private String mirrorToRegistry;

@Option(names = {"--scan-mode"}, paramLabel = "false", description = "Specify container security scan mode, it can be 'none', 'lazy' 'async' or 'sync'")
private ScanMode scanMode;

@Option(names = {"--scan-level"}, paramLabel = "false", description = "Specify one or more security scan vulnerabilities level allowed in the container e.g. low,medium,high,critical")
private List<ScanLevel> scanLevels;

@CommandLine.Parameters
List<String> prompt;

Expand Down Expand Up @@ -247,7 +255,7 @@ else if( app.inspect ) {
}
}
catch (IllegalCliArgumentException | CommandLine.ParameterException | BadClientResponseException |
ClientConnectionException e) {
ReadyTimeoutException | ClientConnectionException e) {
System.err.println(e.getMessage());
System.exit(1);
}
Expand Down Expand Up @@ -440,6 +448,8 @@ protected SubmitContainerTokenRequest createRequest() {
.withContainerIncludes(includes)
.withNameStrategy(nameStrategy)
.withMirrorRegistry(mirrorToRegistry)
.withScanMode(scanMode)
.withScanLevels(scanLevels)
;
}

Expand Down Expand Up @@ -471,10 +481,16 @@ public void run() {
// submit it
SubmitContainerTokenResponse resp = client.submit(request);
// await build to be completed
if( await != null && resp.buildId!=null && !resp.cached )
client.awaitCompletion(resp.buildId, await);
// print the wave container name
System.out.println(dumpOutput(resp));
if( await != null && resp.status!=null && resp.status!=ContainerStatus.READY ) {
ContainerStatusResponse status = client.awaitReadiness(resp.requestId, await);
// print the wave container name
System.out.println(dumpOutput(new SubmitContainerTokenResponseEx(resp, status)));
}
else {
// print the wave container name
System.out.println(dumpOutput(resp));
}

}

private String encodePathBase64(String value) {
Expand Down
38 changes: 25 additions & 13 deletions app/src/main/java/io/seqera/wave/cli/Client.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,18 @@
import dev.failsafe.event.EventListener;
import dev.failsafe.event.ExecutionAttemptedEvent;
import dev.failsafe.function.CheckedSupplier;
import io.seqera.wave.api.BuildStatusResponse;
import io.seqera.wave.api.ContainerInspectRequest;
import io.seqera.wave.api.ContainerInspectResponse;
import io.seqera.wave.api.ContainerStatus;
import io.seqera.wave.api.ContainerStatusResponse;
import io.seqera.wave.api.ServiceInfo;
import io.seqera.wave.api.ServiceInfoResponse;
import io.seqera.wave.api.SubmitContainerTokenRequest;
import io.seqera.wave.api.SubmitContainerTokenResponse;
import io.seqera.wave.cli.config.RetryOpts;
import io.seqera.wave.cli.exception.BadClientResponseException;
import io.seqera.wave.cli.exception.ClientConnectionException;
import io.seqera.wave.cli.exception.ReadyTimeoutException;
import io.seqera.wave.cli.json.JsonHelper;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -197,40 +199,50 @@ protected URI imageToManifestUri(String image) {
return URI.create(result);
}

void awaitCompletion(String buildId, Duration await) {
log.debug("Waiting for build completion: {} - timeout: {} Seconds", buildId, await.toSeconds());
ContainerStatusResponse awaitReadiness(String requestId, Duration await) {
if( StringUtils.isEmpty(requestId) )
throw new IllegalArgumentException("Argument 'requestId' cannot be empty");
log.debug("Waiting for build completion: {} - timeout: {} Seconds", requestId, await.toSeconds());
final long startTime = Instant.now().toEpochMilli();
while (!isComplete(buildId)) {
while ( true ) {
final ContainerStatusResponse response = checkStatus(requestId);
if( response.status==ContainerStatus.READY )
return response;

if (System.currentTimeMillis() - startTime > await.toMillis()) {
break;
String msg = String.format("Container did not ready status within the max await time (%s)", await.toString());
throw new ReadyTimeoutException(msg);
}
// await
try {
TimeUnit.SECONDS.sleep(10);
}
catch (InterruptedException e) {
throw new RuntimeException("Execution interrupted", e);
}
}
}

protected boolean isComplete(String buildId) {
final String statusEndpoint = endpoint + "/v1alpha1/builds/"+buildId+"/status";
protected ContainerStatusResponse checkStatus(String requestId) {
final String statusEndpoint = endpoint + "/v1alpha2/container/"+requestId+"/status";
final HttpRequest req = HttpRequest.newBuilder()
.uri(URI.create(statusEndpoint))
.headers("Content-Type","application/json")
.GET()
.build();

try {
//interval of 10 seconds
TimeUnit.SECONDS.sleep(10);

final HttpResponse<String> resp = httpSend(req);
log.debug("Wave response: statusCode={}; body={}", resp.statusCode(), resp.body());
if( resp.statusCode()==200 ) {
BuildStatusResponse result = JsonHelper.fromJson(resp.body(), BuildStatusResponse.class);
return result.status == BuildStatusResponse.Status.COMPLETED;
return JsonHelper.fromJson(resp.body(), ContainerStatusResponse.class);
}
else {
String msg = String.format("Wave invalid response: [%s] %s", resp.statusCode(), resp.body());
throw new BadClientResponseException(msg);
}
}
catch (IOException | FailsafeException | InterruptedException e) {
catch (IOException | FailsafeException e) {
throw new ClientConnectionException("Unable to connect Wave service: " + endpoint, e);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright 2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package io.seqera.wave.cli.exception;

/**
* Exception thrown when a container do not reach a ready status with the max expected time
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
public class ReadyTimeoutException extends RuntimeException {

public ReadyTimeoutException(String message) {
super(message);
}

public ReadyTimeoutException(String message, Throwable cause) {
super(message, cause);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Copyright 2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package io.seqera.wave.cli.model;

import java.time.Duration;
import java.util.Map;

import io.seqera.wave.api.ContainerStatusResponse;
import io.seqera.wave.api.SubmitContainerTokenResponse;

/**
* Extend the {@link SubmitContainerTokenResponse} object with extra fields
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
public class SubmitContainerTokenResponseEx extends SubmitContainerTokenResponse {

/**
* The request duration
*/
public Duration duration;

/**
* The found vulnerabilities
*/
public Map<String,Integer> vulnerabilities;

/**
* Whenever the request is succeeded or not
*/
public Boolean succeeded;

/**
* Descriptive reason for returned status, used for failures
*/
public String reason;

/**
* Link to detail page
*/
public String detailsUri;

public SubmitContainerTokenResponseEx(SubmitContainerTokenResponse resp1, ContainerStatusResponse resp2) {
super(resp1);
this.status = resp2.status;
this.duration = resp2.duration;
this.vulnerabilities = resp2.vulnerabilities;
this.succeeded = resp2.succeeded;
this.reason = resp2.reason;
this.detailsUri = resp2.detailsUri;
}

}
25 changes: 25 additions & 0 deletions app/src/main/java/io/seqera/wave/cli/util/YamlHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,23 @@

package io.seqera.wave.cli.util;

import java.time.Duration;
import java.time.Instant;

import io.seqera.wave.api.ContainerInspectResponse;
import io.seqera.wave.api.SubmitContainerTokenResponse;
import io.seqera.wave.cli.model.ContainerInspectResponseEx;
import io.seqera.wave.cli.model.ContainerSpecEx;
import io.seqera.wave.cli.model.LayerRef;
import io.seqera.wave.cli.model.SubmitContainerTokenResponseEx;
import io.seqera.wave.core.spec.ConfigSpec;
import io.seqera.wave.core.spec.ContainerSpec;
import io.seqera.wave.core.spec.ManifestSpec;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.introspector.BeanAccess;
import org.yaml.snakeyaml.introspector.Property;
import org.yaml.snakeyaml.nodes.NodeTuple;
import org.yaml.snakeyaml.nodes.Tag;
import org.yaml.snakeyaml.representer.Representer;

Expand All @@ -46,7 +50,18 @@ public static String toYaml(SubmitContainerTokenResponse resp) {
final Representer representer = new Representer(opts) {
{
addClassTag(SubmitContainerTokenResponse.class, Tag.MAP);
addClassTag(SubmitContainerTokenResponseEx.class, Tag.MAP);
representers.put(Instant.class, data -> representScalar(Tag.STR, data.toString()));
representers.put(Duration.class, data -> representScalar(Tag.STR, data.toString()));
}

// skip null values in the resulting yaml
@Override
protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) {
if (propertyValue == null) {
return null;
}
return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag);
}
};

Expand All @@ -70,6 +85,16 @@ public static String toYaml(ContainerInspectResponseEx resp) {
addClassTag(LayerRef.class, Tag.MAP);
representers.put(Instant.class, data -> representScalar(Tag.STR, data.toString()));
}

// skip null values in the resulting yaml
@Override
protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) {
// Skip null values
if (propertyValue == null) {
return null;
}
return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag);
}
};

representer.getPropertyUtils().setSkipMissingProperties(true);
Expand Down
Loading

0 comments on commit 270a412

Please sign in to comment.