diff --git a/osgi-tests/src/test/java/io/siddhi/distribution/test/osgi/SiddhiStoreAPITestcase.java b/osgi-tests/src/test/java/io/siddhi/distribution/test/osgi/SiddhiStoreAPITestcase.java index a11a18eed4..5f72d49b01 100644 --- a/osgi-tests/src/test/java/io/siddhi/distribution/test/osgi/SiddhiStoreAPITestcase.java +++ b/osgi-tests/src/test/java/io/siddhi/distribution/test/osgi/SiddhiStoreAPITestcase.java @@ -184,7 +184,7 @@ private void testHttpResponse(String body, Event[] inputEvents, int expectedResp @Test(dependsOnMethods = "testStoreApiBundle") public void testSelectAllWithSuccessResponse() throws InterruptedException { - Thread.sleep(15000); + Thread.sleep(20000); Event[] events = new Event[]{ new Event(System.currentTimeMillis(), new Object[]{ "recordId1", 10.34f, false, 1200, 300, 400, "2017-11-22"}), diff --git a/pom.xml b/pom.xml index 417945f7e4..0028ea6198 100644 --- a/pom.xml +++ b/pom.xml @@ -1072,6 +1072,68 @@ testcontainers ${org.testcontainers.version} + + com.spotify + docker-client + shaded + ${com.spotify.docker.client.version} + + + commons-lang + commons-lang + ${commons.lang.commons.lang.version} + + + org.glassfish.jersey.connectors + jersey-apache-connector + ${org.glassfish.jersey.connectors.jersey.apache.connector.version} + + + org.apache.httpcomponents + httpclient + ${org.apache.httpcomponents.httpclient.version} + + + commons-logging + commons-logging + + + + + org.glassfish.jersey.inject + jersey-hk2 + ${org.glassfish.jersey.inject.jersey.hk2.version} + + + org.glassfish.jersey.core + jersey-client + ${org.glassfish.jersey.core.jersey.client.version} + + + org.glassfish.jersey.core + jersey-common + ${org.glassfish.jersey.core.jersey.common.version} + + + org.glassfish.hk2 + hk2-api + ${org.glassfish.hk2.hk2.api.version} + + + org.glassfish.jersey.bundles.repackaged + jersey-guava + ${org.glassfish.jersey.bundles.repackaged.jersey.guava.version} + + + org.glassfish.hk2.external + javax.inject + ${org.glassfish.hk2.external.javax.inject.version} + + + com.kohlschutter.junixsocket + junixsocket-core + ${com.kohlschutter.junixsocket.junixsocket.core.version} + @@ -1243,6 +1305,18 @@ 3.1.4 3.1.0 1.1.1 + 8.16.0 + 2.22.2 + 4.5.6 + 2.6 + 2.27 + 2.22.2 + 2.22.2 + 2.4.0-b34 + 2.22.2 + 2.4.0-b34 + 2.2.0 + 2.22.2 1.2.4 2.4.8 diff --git a/tooling/components/io.siddhi.distribution.editor.core/pom.xml b/tooling/components/io.siddhi.distribution.editor.core/pom.xml index 57679a7273..e22836cc56 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/pom.xml +++ b/tooling/components/io.siddhi.distribution.editor.core/pom.xml @@ -136,30 +136,96 @@ org.yaml snakeyaml + + com.spotify + docker-client + shaded + + + + extract-dependency + + true + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-dependencies + compile + + unpack + + + ${project.build.directory}/dependencies/dependency-maven-plugin-markers + + + com.spotify + docker-client + shaded + ${project.build.directory}/dependencies/docker-client + + + + + + + + org.apache.felix + maven-bundle-plugin + true + + + ${project.artifactId} + ${project.artifactId} + + web=target/classes/web, + target/classes/mime-map.properties, + META-INF=target/classes/META-INF, + META-INF/services=${project.build.directory}/dependencies/docker-client/META-INF/services + + + com.spotify.docker.*, + com.kenai.*, + org.apache.commons.*, + jni.*, + jersey.repackaged.com.*, + io.siddhi.distribution.editor.core.internal, + io.siddhi.distribution.editor.core.internal.* + + + io.siddhi.distribution.editor.core.*, + org.apache.commons.* + + + org.osgi.framework.*;version="${osgi.framework.import.version.range}", + org.wso2.carbon.kernel;version="${carbon.kernel.package.import.version.range}", + org.yaml.snakeyaml.*; version="${org.yaml.snakeyaml.version.range}", + io.siddhi.distribution.common.*;version="${io.siddhi.distribution.version.range}", + io.siddhi.distribution.editor.log.appender.*, + org.wso2.msf4j.*;version="${msf4j.import.version.range}", + javax.management.*, + javax.ws.rs.*;version="0.0.0", + com.fasterxml.jackson.core.*;version="${com.fasterxml.jackson.core.version.range}", + com.fasterxml.jackson.databind.*;version="${com.fasterxml.jackson.core.version.range}", + *;resolution:=optional + + * + + + + + + + + findbugs-exclude.xml - - io.siddhi.distribution.editor.core.internal, - io.siddhi.distribution.editor.core.internal.*, - - - io.siddhi.distribution.editor.core.* - - - org.osgi.framework.*;version="${osgi.framework.import.version.range}", - org.wso2.carbon.kernel;version="${carbon.kernel.package.import.version.range}", - org.yaml.snakeyaml.*; version="${org.yaml.snakeyaml.version.range}", - io.siddhi.distribution.common.*;version="${io.siddhi.distribution.version.range}", - io.siddhi.distribution.editor.log.appender.*, - org.wso2.msf4j.*;version="${msf4j.import.version.range}", - javax.management.*, - javax.ws.rs.*;version="0.0.0", - com.fasterxml.jackson.core.*;version="${com.fasterxml.jackson.core.version.range}", - com.fasterxml.jackson.databind.*;version="${com.fasterxml.jackson.core.version.range}", - *;resolution:=optional - osgi.service; objectClass="org.wso2.msf4j.Microservice"; serviceCount="1", osgi.service; objectClass="org.wso2.carbon.deployment.engine.Deployer";serviceCount="1" diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/configs/DockerBuildConfig.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/configs/DockerBuildConfig.java new file mode 100644 index 0000000000..145d2e8e1d --- /dev/null +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/configs/DockerBuildConfig.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.siddhi.distribution.editor.core.commons.configs; + +/** + * This class holds Docker build and push process related data. + */ +public class DockerBuildConfig { + private String userName; + private String email; + private String password; + private String imageName; + + public String getUserName() { + return userName; + } + + public String getEmail() { + return email; + } + + public String getPassword() { + return password; + } + + public String getImageName() { + return imageName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public void setEmail(String email) { + this.email = email; + } + + public void setPassword(String password) { + this.password = password; + } + + public void setImageName(String imageName) { + this.imageName = imageName; + } +} diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/kubernetes/SiddhiProcessContainer.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/kubernetes/SiddhiProcessContainer.java index 972cdae8e5..89de6fce2e 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/kubernetes/SiddhiProcessContainer.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/kubernetes/SiddhiProcessContainer.java @@ -25,6 +25,7 @@ */ public class SiddhiProcessContainer { private ArrayList env; + private String image; public ArrayList getEnv() { return env; @@ -33,4 +34,12 @@ public ArrayList getEnv() { public void setEnv(ArrayList env) { this.env = env; } + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } } diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/AppStartRequest.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/AppStartRequest.java new file mode 100644 index 0000000000..821b00ca10 --- /dev/null +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/AppStartRequest.java @@ -0,0 +1,23 @@ +package io.siddhi.distribution.editor.core.commons.request; + +import java.util.Map; + +/** + * Bean class to represent the Siddhi application start request. + */ +public class AppStartRequest { + private String siddhiAppName; + private Map variables; + + public String getSiddhiAppName() { + return siddhiAppName; + } + + public Map getVariables() { + return variables; + } + + public void setVariables(Map variables) { + this.variables = variables; + } +} diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ExportAppsRequest.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ExportAppsRequest.java index 7b87a1d4fb..8b4395e5ff 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ExportAppsRequest.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ExportAppsRequest.java @@ -18,6 +18,8 @@ package io.siddhi.distribution.editor.core.commons.request; +import io.siddhi.distribution.editor.core.commons.configs.DockerBuildConfig; + import java.util.List; import java.util.Map; @@ -31,6 +33,7 @@ public class ExportAppsRequest { private List bundles; private List jars; private String kubernetesConfiguration; + private DockerBuildConfig dockerConfiguration; public List> getTemplatedSiddhiApps() { return templatedSiddhiApps; @@ -55,4 +58,8 @@ public List getJars() { public String getKubernetesConfiguration() { return kubernetesConfiguration; } + + public DockerBuildConfig getDockerConfiguration() { + return dockerConfiguration; + } } diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ValidationRequest.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ValidationRequest.java index b1958a0ce6..49c1c7d0e9 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ValidationRequest.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/commons/request/ValidationRequest.java @@ -18,6 +18,7 @@ package io.siddhi.distribution.editor.core.commons.request; import java.util.List; +import java.util.Map; /** * Request wrapper for Validation Request. @@ -28,6 +29,7 @@ public class ValidationRequest { private List missingStreams; private List missingAggregationDefinitions; private List> missingInnerStreams; + private Map variables; public String getSiddhiApp() { @@ -68,4 +70,12 @@ public void setMissingInnerStreams(List> missingInnerStreams) { this.missingInnerStreams = missingInnerStreams; } + + public Map getVariables() { + return variables; + } + + public void setVariables(Map variables) { + this.variables = variables; + } } diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DebugRuntime.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DebugRuntime.java index 2502fd1cee..d753fff587 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DebugRuntime.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DebugRuntime.java @@ -71,6 +71,10 @@ public SiddhiDebugger getDebugger() { return debugger; } + public String getSiddhiApp() { + return siddhiApp; + } + public void start() { if (Mode.STOP.equals(mode)) { diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DockerBuilder.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DockerBuilder.java new file mode 100644 index 0000000000..c77db6630b --- /dev/null +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DockerBuilder.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.siddhi.distribution.editor.core.internal; + +import com.spotify.docker.client.DefaultDockerClient; +import com.spotify.docker.client.DockerClient; +import com.spotify.docker.client.exceptions.DockerException; +import com.spotify.docker.client.messages.RegistryAuth; +import io.siddhi.distribution.editor.core.exception.DockerGenerationException; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicReference; + +/** + * This class build and push Docker images using a given Dockerfile. + */ +public class DockerBuilder extends Thread { + private Thread t; + private static final Logger log = LoggerFactory.getLogger(DockerBuilder.class); + private static final String UNIX_DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"; + private static final String WINDOWS_DEFAULT_DOCKER_HOST = "tcp://localhost:2375"; + private static final String STEP_BUILDING = "Building"; + private static final String STEP_PUSHING = "Pushing"; + private static final String STEP_COMPLETED = "Completed"; + private static final String STEP_ERROR = "Error"; + private DockerClient dockerClient; + private DockerBuilderStatus dockerBuilderStatus; + private String imageId; + private String dockerImageName; + private String dockerUserName; + private String dockerEmail; + private String dockerPassword; + private Path dockerFilePath; + + public DockerBuilder( + String dockerImageName, + String dockerUserName, + String dockerEmail, + String dockerPassword, + Path dockerFilePath, + DockerBuilderStatus dockerBuilderStatus + ) { + this.dockerImageName = dockerImageName; + this.dockerUserName = dockerUserName; + this.dockerEmail = dockerEmail; + this.dockerPassword = dockerPassword; + this.dockerFilePath = dockerFilePath; + this.dockerBuilderStatus = dockerBuilderStatus; + } + + public void run() { + boolean isDockerBuilt; + boolean isDockerPushed; + try { + isDockerBuilt = buildDockerImage(); + if (isDockerBuilt) { + isDockerPushed = pushDockerImage(); + if (isDockerPushed) { + dockerBuilderStatus.setStep(STEP_COMPLETED); + } else { + dockerBuilderStatus.setStep(STEP_ERROR); + } + } + removeSampleDockerDir(); + } catch (DockerGenerationException e) { + log.error("Failed to build and push Docker artifacts automatically ", e); + } + } + + public void start () { + if (t == null) { + t = new Thread (this); + t.start (); + } + } + + private void createDockerClient() throws DockerGenerationException { + String dockerHost; + if (SystemUtils.IS_OS_WINDOWS) { + dockerHost = WINDOWS_DEFAULT_DOCKER_HOST; + } else { + dockerHost = UNIX_DEFAULT_DOCKER_HOST; + } + dockerClient = DefaultDockerClient.builder().uri(dockerHost).build(); + try { + dockerClient.ping(); + } catch (DockerException | InterruptedException e) { + dockerBuilderStatus.setStatus(e.getMessage()); + throw new DockerGenerationException( + "Failed to connect to the Docker host using provided host URI " + dockerHost, e + ); + } + } + + private boolean buildDockerImage() throws DockerGenerationException { + boolean isDockerBuilt = false; + dockerBuilderStatus.setStep(STEP_BUILDING); + if (dockerClient == null) { + createDockerClient(); + } + + final AtomicReference imageIdFromMessage = new AtomicReference<>(); + try { + // build the image + imageId = dockerClient.build(dockerFilePath, dockerImageName, message -> { + final String imageId = message.buildImageId(); + if (message.stream() != null && !message.stream().equals("\n") && !message.stream().equals("")) { + dockerBuilderStatus.setStatus(message.stream().replace("\n", "")); + log.info(message.stream().replace("\n", "")); + } + // Image creation successful + if (imageId != null) { + imageIdFromMessage.set(imageId); + } + }, DockerClient.BuildParam.noCache(), DockerClient.BuildParam.forceRm()); + if (imageIdFromMessage.get() != null) { + isDockerBuilt = true; + } + } catch (DockerException | InterruptedException | IOException e) { + dockerBuilderStatus.setStatus(e.getMessage()); + throw new DockerGenerationException( + "Failed to build the Docker image in directory " + dockerFilePath.toString() + + " and Docker image name " + dockerImageName, e + ); + } + + return isDockerBuilt; + } + + private boolean pushDockerImage() throws DockerGenerationException { + boolean isPushed = false; + dockerBuilderStatus.setStep(STEP_PUSHING); + if (dockerClient == null) { + createDockerClient(); + } + + //push built docker image to the docker hub registry + final RegistryAuth registryAuth = RegistryAuth.builder() + .email(dockerEmail) + .username(dockerUserName) + .password(dockerPassword) + .build(); + try { + final int statusCode = dockerClient.auth(registryAuth); + if (statusCode == 200) { + dockerClient.push(dockerImageName, message -> { + if (message != null && message.status() != null && message.id() != null) { + dockerBuilderStatus.setStatus(message.status()); + log.info(message.id() + ": " + message.status()); + } + }, registryAuth); + isPushed = true; + } else { + dockerBuilderStatus.setStatus("Authentication failed to connect to the Docker registry using UserName: " + + dockerUserName + " and Email: " + dockerEmail); + throw new DockerGenerationException( + "Authentication failed to connect to the Docker registry using UserName: " + + dockerUserName + " and Email: " + dockerEmail + ); + } + } catch (DockerException | InterruptedException e) { + dockerBuilderStatus.setStatus(e.getMessage()); + throw new DockerGenerationException( + "Failed to push the Docker image " + dockerImageName + + " with UserName: " + dockerUserName + " and Email: " + dockerEmail, e + ); + } + return isPushed; + } + + private void removeSampleDockerDir() { + if (dockerFilePath != null && Files.exists(dockerFilePath)) { + try { + FileUtils.deleteDirectory(new File(dockerFilePath.toString())); + } catch (IOException e) { + log.error("Failed automatic deletion of the temporary Docker directory " + + dockerFilePath.toString(), e); + } + } + } +} + diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DockerBuilderStatus.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DockerBuilderStatus.java new file mode 100644 index 0000000000..8bdc25a6af --- /dev/null +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/DockerBuilderStatus.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.siddhi.distribution.editor.core.internal; + +/** + * This class store the status of the DockerBuilder. + */ +class DockerBuilderStatus { + private String status; + private String step; + + DockerBuilderStatus(String status, String step) { + this.status = status; + this.step = step; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getStatus() { + return status; + } + + public String getStep() { + return step; + } + + public void setStep(String step) { + this.step = step; + } +} diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/EditorMicroservice.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/EditorMicroservice.java index 3df40eb10a..063e10ff0a 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/EditorMicroservice.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/EditorMicroservice.java @@ -37,7 +37,9 @@ import io.siddhi.distribution.common.common.utils.config.FileConfigManager; import io.siddhi.distribution.editor.core.EditorSiddhiAppRuntimeService; import io.siddhi.distribution.editor.core.Workspace; +import io.siddhi.distribution.editor.core.commons.configs.DockerBuildConfig; import io.siddhi.distribution.editor.core.commons.metadata.MetaData; +import io.siddhi.distribution.editor.core.commons.request.AppStartRequest; import io.siddhi.distribution.editor.core.commons.request.ExportAppsRequest; import io.siddhi.distribution.editor.core.commons.request.ValidationRequest; import io.siddhi.distribution.editor.core.commons.response.DebugRuntimeResponse; @@ -104,6 +106,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -139,6 +142,9 @@ public class EditorMicroservice implements Microservice { private static final String STATUS = "status"; private static final String SUCCESS = "success"; private static final String EXPORT_TYPE_KUBERNETES = "kubernetes"; + private static final String EXPORT_REQUEST_TYPE_DOWNLOAD_ONLY = "downloadOnly"; + private static final String EXPORT_REQUEST_TYPE_BUILD_ONLY = "buildOnly"; + private static final String EXPORT_REQUEST_GET_STATUS_HEADER = "Siddhi-Docker-Key"; private ServiceRegistration serviceRegistration; private Workspace workspace; private ExecutorService executorService = Executors @@ -149,6 +155,7 @@ public class EditorMicroservice implements Microservice { private ConfigProvider configProvider; private ServiceRegistration siddhiAppRuntimeServiceRegistration; private StoreQueryAPIHelper storeQueryAPIHelper; + private Map dockerBuilderStatusMap = new HashMap<>(); public EditorMicroservice() { @@ -216,8 +223,12 @@ public Response validateSiddhiApp(String validationRequestString) { ValidationRequest validationRequest = new Gson().fromJson(validationRequestString, ValidationRequest.class); String jsonString; try { + String siddhiApp = validationRequest.getSiddhiApp(); + if (validationRequest.getVariables().size() != 0) { + siddhiApp = SourceEditorUtils.populateSiddhiAppWithVars(validationRequest.getVariables(), siddhiApp); + } SiddhiAppRuntime siddhiAppRuntime = - EditorDataHolder.getSiddhiManager().createSiddhiAppRuntime(validationRequest.getSiddhiApp()); + EditorDataHolder.getSiddhiManager().createSiddhiAppRuntime(siddhiApp); // Status SUCCESS to indicate that the siddhi app is valid ValidationSuccessResponse response = new ValidationSuccessResponse(Status.SUCCESS); @@ -725,7 +736,6 @@ public Response getMetaData() { @Produces(MediaType.APPLICATION_JSON) @Path("/{siddhiAppName}/start") public Response start(@PathParam("siddhiAppName") String siddhiAppName) { - List streams = EditorDataHolder .getDebugProcessorService() .getSiddhiAppRuntimeHolder(siddhiAppName) @@ -743,6 +753,23 @@ public Response start(@PathParam("siddhiAppName") String siddhiAppName) { .entity(new DebugRuntimeResponse(Status.SUCCESS, null, siddhiAppName, streams, queries)).build(); } + @POST + @Produces(MediaType.APPLICATION_JSON) + @Path("/start") + public Response startWithVariables(String appStartRequestString) { + AppStartRequest appStartRequest = new Gson().fromJson(appStartRequestString, AppStartRequest.class); + String siddhiAppName = appStartRequest.getSiddhiAppName(); + if (appStartRequest.getVariables().size() > 0) { + DebugRuntime existingRuntime = EditorDataHolder.getSiddhiAppMap().get(siddhiAppName); + String siddhiApp = existingRuntime.getSiddhiApp(); + siddhiApp = SourceEditorUtils.populateSiddhiAppWithVars(appStartRequest.getVariables(), siddhiApp); + DebugRuntime runtimeHolder = new DebugRuntime(siddhiAppName, siddhiApp); + EditorDataHolder.getSiddhiAppMap().put(siddhiAppName, runtimeHolder); + + } + return start(siddhiAppName); + } + @GET @Produces(MediaType.APPLICATION_JSON) @Path("/{siddhiAppName}/debug") @@ -1114,21 +1141,83 @@ public Response getToolTips(String encodedSiddhiAppConfigJson) { @POST @Path("/export") @Consumes(MediaType.APPLICATION_FORM_URLENCODED) - public Response exportApps(@QueryParam("type") String exportType, @FormParam("payload") String payload) { + public Response exportApps( + @QueryParam("exportType") String exportType, + @QueryParam("requestType") String requestType, + @FormParam("payload") String payload + ) { + String dockerBuilderStatusKey = ""; try { + if (payload == null) { + log.error("Form parameter payload cannot be null. Expected payload format: " + + "{'templatedSiddhiApps': [''], " + + "'configuration': '', " + + "'templatedVariables': [''], " + + "'kubernetesConfiguration': '', " + + "'dockerConfiguration': ''}"); + return Response + .status(Response.Status.BAD_REQUEST) + .build(); + } ExportAppsRequest exportAppsRequest = new Gson().fromJson(payload, ExportAppsRequest.class); ExportUtils exportUtils = new ExportUtils(configProvider, exportAppsRequest, exportType); File zipFile = exportUtils.createZipFile(); String fileName = "siddhi-docker.zip"; + boolean kubernetesEnabled = false; if (exportType != null) { if (exportType.equals(EXPORT_TYPE_KUBERNETES)) { fileName = "siddhi-kubernetes.zip"; + kubernetesEnabled = true; + } + } + + if (requestType != null && requestType.equals(EXPORT_REQUEST_TYPE_DOWNLOAD_ONLY)) { + return Response + .status(Response.Status.OK) + .entity(zipFile) + .header("Content-Disposition", ("attachment; filename=" + fileName)) + .build(); + } + DockerBuilderStatus dockerBuilderStatus = new DockerBuilderStatus("", ""); + if (exportAppsRequest != null && exportAppsRequest.getDockerConfiguration() != null) { + DockerBuildConfig dockerBuildConfig = exportAppsRequest.getDockerConfiguration(); + if ((dockerBuildConfig.getImageName() == null) || + (dockerBuildConfig.getUserName() == null) || + (dockerBuildConfig.getEmail() == null) || + (dockerBuildConfig.getPassword() == null) + ) { + log.error("Missing required Docker build configuration " + + "of (DockerImageName|UserName|Email|Password)"); + return Response + .status(Response.Status.BAD_REQUEST) + .build(); } + DockerBuilder dockerBuilder = new DockerBuilder( + dockerBuildConfig.getImageName(), + dockerBuildConfig.getUserName(), + dockerBuildConfig.getEmail(), + dockerBuildConfig.getPassword(), + exportUtils.getTempDockerPath(), + dockerBuilderStatus + ); + dockerBuilder.start(); + UUID uuid = UUID.randomUUID(); + dockerBuilderStatusKey = uuid.toString(); + dockerBuilderStatusMap.clear(); + dockerBuilderStatusMap.put(dockerBuilderStatusKey, dockerBuilderStatus); + } + + if (requestType != null && requestType.equals(EXPORT_REQUEST_TYPE_BUILD_ONLY) && !kubernetesEnabled) { + return Response + .status(Response.Status.OK) + .header(EXPORT_REQUEST_GET_STATUS_HEADER, dockerBuilderStatusKey) + .build(); } return Response .status(Response.Status.OK) .entity(zipFile) .header("Content-Disposition", ("attachment; filename=" + fileName)) + .header("Siddhi-Docker-Key", dockerBuilderStatusKey) .build(); } catch (JsonSyntaxException e) { log.error("Incorrect configuration format.", e); @@ -1143,6 +1232,42 @@ public Response exportApps(@QueryParam("type") String exportType, @FormParam("pa } } + @GET + @Path("/dockerBuildStatus") + @Produces(MediaType.APPLICATION_JSON) + public Response get(@Context Request request) { + JsonObject dockerBuilderStatus = new JsonObject(); + if (request.getHeader(EXPORT_REQUEST_GET_STATUS_HEADER) != null) { + if (dockerBuilderStatusMap.get(request.getHeader(EXPORT_REQUEST_GET_STATUS_HEADER)) != null) { + DockerBuilderStatus currentStatus = dockerBuilderStatusMap.get( + request.getHeader(EXPORT_REQUEST_GET_STATUS_HEADER) + ); + dockerBuilderStatus.addProperty( + "Step", + currentStatus.getStep() + ); + dockerBuilderStatus.addProperty( + "Status", + currentStatus.getStatus() + ); + } else { + dockerBuilderStatus.addProperty( + "Step", + "" + ); + dockerBuilderStatus.addProperty( + "Status", + "" + ); + } + } + return Response + .status(Response.Status.OK) + .entity(dockerBuilderStatus) + .type(MediaType.APPLICATION_JSON) + .build(); + } + /** * Export given Siddhi apps and other configurations to docker or kubernetes artifacts. * diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/ExportUtils.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/ExportUtils.java index 0a790f33b9..fd2940ac2e 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/ExportUtils.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/internal/ExportUtils.java @@ -18,6 +18,7 @@ package io.siddhi.distribution.editor.core.internal; +import io.siddhi.core.stream.input.source.Source; import io.siddhi.distribution.editor.core.commons.configs.DockerConfigs; import io.siddhi.distribution.editor.core.commons.kubernetes.Env; import io.siddhi.distribution.editor.core.commons.kubernetes.KubernetesConfig; @@ -29,6 +30,7 @@ import io.siddhi.distribution.editor.core.exception.DockerGenerationException; import io.siddhi.distribution.editor.core.exception.KubernetesGenerationException; import io.siddhi.distribution.editor.core.util.Constants; +import io.siddhi.distribution.editor.core.util.SourceEditorUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.wso2.carbon.config.ConfigurationException; @@ -49,7 +51,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -65,7 +72,9 @@ public class ExportUtils { private static final String BUNDLES_BLOCK_TEMPLATE = "\\{\\{BUNDLES_BLOCK}}"; private static final String ENV_BLOCK_TEMPLATE = "\\{\\{ENV_BLOCK}}"; private static final String APPS_BLOCK_TEMPLATE = "\\{\\{APPS_BLOCK}}"; - private static final String PRODUCT_VERSION_TEMPLATE = "\\{\\{PRODUCT_VERSION}}"; + private static final String EXPOSE_PORTS_BLOCK_TEMPLATE = "\\{\\{EXPORT_PORTS_BLOCK}}"; + private static final String DOCKER_BASE_IMAGE_TEMPLATE = "\\{\\{SIDDHI_RUNNER_BASE_IMAGE}}"; + private static final String PORT_BIND_TEMPLATE = "\\{\\{BIND_PORTS}}"; private static final String CONFIG_BLOCK_VALUE = "COPY --chown=siddhi_user:siddhi_io \\$\\{CONFIG_FILE}/ \\$\\{USER_HOME}"; private static final String CONFIG_PARAMETER_VALUE = @@ -85,6 +94,9 @@ public class ExportUtils { private static final String SIDDHI_TEMPLATED_VAR_VALUE_ENTRY = "value"; private static final String RESOURCES_DIR = "resources/docker-export"; private static final String DOCKER_FILE_NAME = "Dockerfile"; + private static final String DOCKER_README_FILE_NAME = "DOCKER-README.md"; + private static final String KUBERNETES_README_FILE_NAME = "K8S-README.md"; + private static final String GENERIC_README_FILE_NAME = "README.md"; private static final String KUBERNETES_FILE_NAME = "siddhi-process.yaml"; private static final String JARS_DIR = "jars/"; private static final String BUNDLE_DIR = "bundles/"; @@ -101,18 +113,22 @@ public class ExportUtils { private DockerConfigs dockerConfigs; private ExportAppsRequest exportAppsRequest; private String exportType; + Path tempDockerDirectoryPath; + private List exposePorts = new ArrayList<>(); ExportUtils( ConfigProvider configProvider, ExportAppsRequest exportAppsRequest, String exportType ) { + this.configProvider = configProvider; this.exportAppsRequest = exportAppsRequest; this.exportType = exportType; } ExportUtils(ConfigProvider configProvider) { + this.configProvider = configProvider; } @@ -123,10 +139,12 @@ public class ExportUtils { * @throws DockerGenerationException if docker generation fails */ public File createZipFile() throws DockerGenerationException, KubernetesGenerationException { + boolean jarsAdded = false; boolean bundlesAdded = false; boolean configChanged = false; boolean envChanged = false; + boolean buildDocker = false; String zipFileName = "siddhi-docker.zip"; String zipFileRoot = "siddhi-docker/"; if (exportType != null && exportType.equals(EXPORT_TYPE_KUBERNETES)) { @@ -134,12 +152,28 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati zipFileRoot = "siddhi-kubernetes/"; } Path dockerFilePath = Paths.get(Constants.RUNTIME_PATH, RESOURCES_DIR, DOCKER_FILE_NAME); + Path dockerReadmeFilePath = Paths.get(Constants.RUNTIME_PATH, RESOURCES_DIR, DOCKER_README_FILE_NAME); + Path kubernetesReadmeFilePath = Paths.get(Constants.RUNTIME_PATH, RESOURCES_DIR, KUBERNETES_README_FILE_NAME); File zipFile = new File(zipFileName); - StringBuilder stringBuilder = new StringBuilder(); ZipOutputStream zipOutputStream = null; - ZipEntry dockerFileEntry = new ZipEntry( - Paths.get(zipFileRoot, DOCKER_FILE_NAME).toString() - ); + ZipEntry dockerFileEntry = new ZipEntry(Paths.get(zipFileRoot, DOCKER_FILE_NAME).toString()); + + if (exportAppsRequest.getDockerConfiguration() != null) { + UUID uuid = UUID.randomUUID(); + tempDockerDirectoryPath = Paths.get(RESOURCES_DIR, uuid.toString()); + if (!Files.exists(tempDockerDirectoryPath)) { + if (!new File(tempDockerDirectoryPath.toString()).mkdir()) { + throw new DockerGenerationException( + "Failed to create the sample directory " + + tempDockerDirectoryPath.toString() + ); + } + } + if (Files.isWritable(tempDockerDirectoryPath)) { + buildDocker = true; + } + } + try { zipOutputStream = new ZipOutputStream(new FileOutputStream(zipFile)); @@ -148,6 +182,16 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati jarsAdded = true; String jarRootDir = Paths.get(Constants.CARBON_HOME, JARS_DIR).toString(); String jarEntryRootDir = Paths.get(zipFileRoot, JARS_DIR).toString(); + Path tempDockerJarDirPath = Paths.get(tempDockerDirectoryPath.toString(), JARS_DIR); + if (buildDocker && !Files.exists(tempDockerJarDirPath)) { + if (!new File(tempDockerJarDirPath.toString()).mkdir()) { + throw new DockerGenerationException( + "Failed to create the sample jars directory " + + tempDockerJarDirPath.toString() + ); + } + } + for (String jar : exportAppsRequest.getJars()) { Path jarPath = Paths.get(jarRootDir, jar); ZipEntry jarEntry = new ZipEntry(Paths.get(jarEntryRootDir, jar).toString()); @@ -156,6 +200,9 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati byte[] jarData = Files.readAllBytes(jarPath); zipOutputStream.write(jarData, 0, jarData.length); zipOutputStream.closeEntry(); + if (buildDocker) { + Files.write(Paths.get(tempDockerJarDirPath.toString(), jar), jarData); + } } else { log.error("JAR file" + jarPath.toString() + " is not readable."); } @@ -168,6 +215,16 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati bundlesAdded = true; String bundleRootDir = Paths.get(Constants.CARBON_HOME, BUNDLE_DIR).toString(); String bundleEntryRootDir = Paths.get(zipFileRoot, BUNDLE_DIR).toString(); + Path tempDockerBundleDirPath = Paths.get(tempDockerDirectoryPath.toString(), BUNDLE_DIR); + if (buildDocker && !Files.exists(tempDockerBundleDirPath)) { + if (!new File(tempDockerBundleDirPath.toString()).mkdir()) { + throw new DockerGenerationException( + "Failed to create the sample bundles directory " + + tempDockerBundleDirPath.toString() + ); + } + } + for (String bundle : exportAppsRequest.getBundles()) { Path bundlePath = Paths.get(bundleRootDir, bundle); ZipEntry bundleEntry = new ZipEntry( @@ -179,6 +236,9 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati byte[] bundleData = Files.readAllBytes(bundlePath); zipOutputStream.write(bundleData, 0, bundleData.length); zipOutputStream.closeEntry(); + if (buildDocker) { + Files.write(Paths.get(tempDockerBundleDirPath.toString(), bundle), bundleData); + } } else { log.error("Bundle file" + bundlePath.toString() + " is not readable."); } @@ -186,7 +246,17 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati } // Write Siddhi apps to the zip file + List userGivenSiddhiApps = new ArrayList<>(); String appsEntryRootDir = Paths.get(zipFileRoot, APPS_DIR).toString(); + Path tempDockerAppDirPath = Paths.get(tempDockerDirectoryPath.toString(), APPS_DIR); + if (buildDocker && !Files.exists(tempDockerAppDirPath)) { + if (!new File(tempDockerAppDirPath.toString()).mkdir()) { + throw new DockerGenerationException( + "Failed to create the sample apps directory " + + tempDockerAppDirPath.toString() + ); + } + } if (exportAppsRequest.getTemplatedSiddhiApps() != null) { for (Map app : exportAppsRequest.getTemplatedSiddhiApps()) { String appName = app.get(SIDDHI_APP_NAME_ENTRY); @@ -194,9 +264,14 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati Paths.get(appsEntryRootDir, appName).toString() ); zipOutputStream.putNextEntry(appEntry); - byte[] appData = app.get(SIDDHI_APP_CONTENT_ENTRY).getBytes(StandardCharsets.UTF_8); + String siddhiAppContent = app.get(SIDDHI_APP_CONTENT_ENTRY); + userGivenSiddhiApps.add(siddhiAppContent); + byte[] appData = siddhiAppContent.getBytes(StandardCharsets.UTF_8); zipOutputStream.write(appData, 0, appData.length); zipOutputStream.closeEntry(); + if (buildDocker) { + Files.write(Paths.get(tempDockerAppDirPath.toString(), appName), appData); + } } } @@ -213,38 +288,107 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati .getBytes(StandardCharsets.UTF_8); zipOutputStream.write(configData, 0, configData.length); zipOutputStream.closeEntry(); + if (buildDocker) { + Path tempDockerConfigPath = Paths.get(tempDockerDirectoryPath.toString(), CONFIG_FILE); + Files.write(tempDockerConfigPath, configData); + } } // Write ENVs to the docker file + StringBuilder envStringBuilder = new StringBuilder(); + Map envMap = new HashMap<>(); if (exportAppsRequest.getTemplatedVariables() != null && !exportAppsRequest.getTemplatedVariables().isEmpty()) { envChanged = true; for (Map env : exportAppsRequest.getTemplatedVariables()) { - stringBuilder + String envKey = env.get(SIDDHI_TEMPLATED_VAR_KEY_ENTRY); + String envVal = env.get(SIDDHI_TEMPLATED_VAR_VALUE_ENTRY); + envMap.put(envKey, envVal); + envStringBuilder .append("ENV ") - .append(env.get(SIDDHI_TEMPLATED_VAR_KEY_ENTRY)) + .append(envKey) .append(" ") - .append(env.get(SIDDHI_TEMPLATED_VAR_VALUE_ENTRY)) + .append(envVal) .append("\n"); } } + //add default expose ports + exposePorts.addAll(Arrays.asList(9090, 9443)); + List siddhiApps = userGivenSiddhiApps; + if (envMap.size() != 0 && !userGivenSiddhiApps.isEmpty()) { + List populatedSiddhiApps = new ArrayList<>(); + for (String siddhiApp : siddhiApps) { + String populatedSiddhiApp = SourceEditorUtils.populateSiddhiAppWithVars(envMap, siddhiApp); + populatedSiddhiApps.add(populatedSiddhiApp); + } + siddhiApps = populatedSiddhiApps; + } + for (String app : siddhiApps) { + try { + Collection> sources = EditorDataHolder.getSiddhiManager() + .createSiddhiAppRuntime(app).getSources(); + for (List sourceList : sources) { + for (Source source : sourceList) { + if (source.getServiceDeploymentInfo() != null) { + exposePorts.add(source.getServiceDeploymentInfo().getPort()); + } + } + } + } catch (Exception ignored) { + //ignoring exception since Siddhi app parsing might fail due to unset variables + if (log.isDebugEnabled()) { + log.error("Exception caught while parsing the exported Siddhi applications.", ignored); + } + } + } + + // Write Expose Ports to the docker file + StringBuilder exposePortsStr = new StringBuilder(); + if (!exposePorts.isEmpty()) { + exposePortsStr.append("EXPOSE "); + for (int port : exposePorts) { + exposePortsStr + .append(port) + .append(" "); + } + exposePortsStr.append("\n"); + } + // Write the docker file to the zip file zipOutputStream.putNextEntry(dockerFileEntry); - byte[] data = this.getDockerFile( + byte[] dockerContent = this.getDockerFile( dockerFilePath, jarsAdded, bundlesAdded, configChanged, envChanged, - stringBuilder.toString() + envStringBuilder.toString(), + exposePortsStr.toString() ); - zipOutputStream.write(data, 0, data.length); + zipOutputStream.write(dockerContent, 0, dockerContent.length); zipOutputStream.closeEntry(); + if (buildDocker) { + Path tempDockerFilePath = Paths.get(tempDockerDirectoryPath.toString(), DOCKER_FILE_NAME); + Files.write(tempDockerFilePath, dockerContent); + } - // Write the kubernetes file to the zip file + // Write the kubernetes file to the zip file and add README.md + ZipEntry readmeEntry = new ZipEntry(Paths.get(zipFileRoot, GENERIC_README_FILE_NAME).toString()); + zipOutputStream.putNextEntry(readmeEntry); if (exportType != null && exportType.equals(EXPORT_TYPE_KUBERNETES)) { + // Add K8s README.md + if (!Files.isReadable(kubernetesReadmeFilePath)) { + throw new KubernetesGenerationException( + "Readme file " + kubernetesReadmeFilePath.toString() + " is not readable." + ); + } + byte[] readmeContent = Files.readAllBytes(kubernetesReadmeFilePath); + zipOutputStream.write(readmeContent, 0, readmeContent.length); + zipOutputStream.closeEntry(); + + // Add K8s YAML ZipEntry kubernetesFileEntry = new ZipEntry( Paths.get(zipFileRoot, KUBERNETES_FILE_NAME).toString() ); @@ -254,6 +398,27 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati ); zipOutputStream.write(kubernetesFileData, 0, kubernetesFileData.length); zipOutputStream.closeEntry(); + } else { + // Add Docker README.md + StringBuilder portBindingStr = new StringBuilder(); + for (int port: exposePorts) { + portBindingStr.append("-p "); + portBindingStr.append(port); + portBindingStr.append(":"); + portBindingStr.append(port); + portBindingStr.append(" "); + } + if (!Files.isReadable(dockerReadmeFilePath)) { + throw new DockerGenerationException( + "Readme file " + dockerReadmeFilePath.toString() + " is not readable." + ); + } + byte[] data = Files.readAllBytes(dockerReadmeFilePath); + String content = new String(data, StandardCharsets.UTF_8); + content = content.replaceAll(PORT_BIND_TEMPLATE, portBindingStr.toString()); + byte[] readmeContent = content.getBytes(StandardCharsets.UTF_8); + zipOutputStream.write(readmeContent, 0, readmeContent.length); + zipOutputStream.closeEntry(); } } catch (IOException e) { throw new DockerGenerationException( @@ -275,14 +440,28 @@ public File createZipFile() throws DockerGenerationException, KubernetesGenerati return zipFile; } + /** + * Return the path of the temporary directory path that created for docker build. + * + * @return Path + */ + public Path getTempDockerPath() { + if (tempDockerDirectoryPath != null) { + return tempDockerDirectoryPath; + } else { + return Paths.get(""); + } + + } + /** * Read Dockerfile and replace the string tokens with valid values read from configurations. * * @param dockerFilePath Path to the Dockerfile - * @param jarsAdded True if user specified custom JARs in the request - * @param bundlesAdded True if user specified custom JARs in the request - * @param configChanged True if user changed the existing deployment.yaml - * @param envList String that contained environment variable list + * @param jarsAdded True if user specified custom JARs in the request + * @param bundlesAdded True if user specified custom JARs in the request + * @param configChanged True if user changed the existing deployment.yaml + * @param envList String that contained environment variable list * @return Content * @throws IOException */ @@ -292,8 +471,10 @@ private byte[] getDockerFile( boolean bundlesAdded, boolean configChanged, boolean envChanged, - String envList + String envList, + String exportPortList ) throws IOException, DockerGenerationException, ConfigurationException { + byte[] data; if (!Files.isReadable(dockerFilePath)) { throw new DockerGenerationException( @@ -302,8 +483,13 @@ private byte[] getDockerFile( } data = Files.readAllBytes(dockerFilePath); String content = new String(data, StandardCharsets.UTF_8); - String productVersion = this.getConfigurations().getProductVersion(); - content = content.replaceAll(PRODUCT_VERSION_TEMPLATE, productVersion); + String dockerBaseImgName = Constants.DEFAULT_SI_DOCKER_BASE_IMAGE; + if (configProvider.getConfigurationObject(Constants.EXPORT_PROPERTIES_NAMESPACE) != null) { + dockerBaseImgName = (String) ((Map) configProvider + .getConfigurationObject(Constants.EXPORT_PROPERTIES_NAMESPACE)) + .get(Constants.DOCKER_BASE_IMAGE_PROPERTY); + } + content = content.replaceAll(DOCKER_BASE_IMAGE_TEMPLATE, dockerBaseImgName); if (exportType != null && exportType.equals(EXPORT_TYPE_KUBERNETES)) { content = content.replaceAll(APPS_BLOCK_TEMPLATE, ""); @@ -336,6 +522,9 @@ private byte[] getDockerFile( } else { content = content.replaceAll(ENV_BLOCK_TEMPLATE, ""); } + + content = content.replaceAll(EXPOSE_PORTS_BLOCK_TEMPLATE, exportPortList); + return content.getBytes(StandardCharsets.UTF_8); } @@ -349,6 +538,7 @@ private byte[] getDockerFile( */ private byte[] getKubernetesFile(Path kubernetesFilePath) throws KubernetesGenerationException, IOException { + if (!Files.isReadable(kubernetesFilePath)) { throw new KubernetesGenerationException( "Kubernetes file " + kubernetesFilePath.toString() + " is not readable." @@ -368,7 +558,7 @@ private byte[] getKubernetesFile(Path kubernetesFilePath) ); SiddhiProcessSpec siddhiProcessSpec = new SiddhiProcessSpec(); - if (kubernetesConfig != null) { + if (kubernetesConfig != null) { if (kubernetesConfig.getMessagingSystem() != null) { siddhiProcessSpec.setMessagingSystem(kubernetesConfig.getMessagingSystem()); } @@ -380,6 +570,9 @@ private byte[] getKubernetesFile(Path kubernetesFilePath) } } + // Set container spec + SiddhiProcessContainer siddhiProcessContainer = new SiddhiProcessContainer(); + boolean changedContainerSpec = false; if (this.exportAppsRequest.getTemplatedVariables() != null && this.exportAppsRequest.getTemplatedVariables().size() > 0) { ArrayList envs = new ArrayList(); @@ -391,9 +584,21 @@ private byte[] getKubernetesFile(Path kubernetesFilePath) ); envs.add(env); } - SiddhiProcessContainer siddhiProcessContainer = new SiddhiProcessContainer(); siddhiProcessContainer.setEnv(envs); siddhiProcessSpec.setContainer(siddhiProcessContainer); + changedContainerSpec = true; + } + + if (exportAppsRequest.getDockerConfiguration() != null && + exportAppsRequest.getDockerConfiguration().getImageName() != null) { + siddhiProcessContainer.setImage( + exportAppsRequest.getDockerConfiguration().getImageName() + ); + changedContainerSpec = true; + } + + if (changedContainerSpec) { + siddhiProcessSpec.setContainer(siddhiProcessContainer); } if (this.exportAppsRequest.getTemplatedSiddhiApps() != null && @@ -443,7 +648,7 @@ protected NodeTuple representJavaBeanProperty( spec = spec.replaceAll("\\$", "\\\\\\$"); spec = spec.replaceAll("\\{", "\\\\\\{"); content = content.replaceAll(SIDDHI_PROCESS_SPEC_TEMPLATE, spec); - if (kubernetesConfig != null) { + if (kubernetesConfig != null) { if (kubernetesConfig.getSiddhiProcessName() != null) { content = content.replaceAll( SIDDHI_PROCESS_NAME_TEMPLATE, @@ -486,6 +691,7 @@ private DockerConfigs getConfigurations() throws ConfigurationException { * @throws IOException */ public String exportConfigs() throws IOException { + Path toolingConfigFile = Paths.get( Constants.CARBON_HOME, DIRECTORY_CONF, diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/Constants.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/Constants.java index 8484d3b893..33841b875c 100755 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/Constants.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/Constants.java @@ -66,6 +66,9 @@ public class Constants { public static final String ATTR_TYPE_BOOL = "bool"; public static final String EVENT_ATTRIBUTE_VALUE_SEPARATOR = ":"; public static final String FAULT_STREAM_PREFIX = "!"; + public static final String EXPORT_PROPERTIES_NAMESPACE = "export.properties"; + public static final String DOCKER_BASE_IMAGE_PROPERTY = "dockerBaseImage"; + public static final String DEFAULT_SI_DOCKER_BASE_IMAGE = "siddhiio/siddhi-runner-base-alpine:latest"; static final String FUNCTION_EXECUTOR = "FunctionExecutor"; static final String ATTRIBUTE_AGGREGATOR = "AttributeAggregatorExecutor"; static final String INCREMENTAL_AGGREGATOR = "IncrementalAggregator"; diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/SourceEditorUtils.java b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/SourceEditorUtils.java index 483b72ad89..6ca98349bb 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/SourceEditorUtils.java +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/java/io/siddhi/distribution/editor/core/util/SourceEditorUtils.java @@ -46,6 +46,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Utility class for getting the meta data for the in built and extension processors in Siddhi. @@ -513,4 +515,21 @@ private static ProcessorMetaData generateProcessorMetaData(Class processorCla } return processorMetaData; } + + public static String populateSiddhiAppWithVars(Map envMap, String siddhiApp) { + if (siddhiApp.contains("$")) { + String envPattern = "\\$\\{(\\w+)\\}"; + Pattern expr = Pattern.compile(envPattern); + Matcher matcher = expr.matcher(siddhiApp); + while (matcher.find()) { + for (int i = 1; i <= matcher.groupCount(); i++) { + String envValue = envMap.getOrDefault(matcher.group(i), ""); + envValue = envValue.replace("\\", "\\\\"); + Pattern subexpr = Pattern.compile("\\$\\{" + matcher.group(i) + "\\}"); + siddhiApp = subexpr.matcher(siddhiApp).replaceAll(envValue); + } + } + } + return siddhiApp; + } } diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/js/dialog/export-dialog.js b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/js/dialog/export-dialog.js index f579d70bbf..c8ef19934c 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/js/dialog/export-dialog.js +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/js/dialog/export-dialog.js @@ -17,9 +17,10 @@ */ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelectorDialog', 'jarsSelectorDialog', - 'templateAppDialog', 'templateConfigDialog', 'fillTemplateValueDialog', 'kubernetesConfigDialog'], + 'templateAppDialog', 'templateConfigDialog', 'fillTemplateValueDialog', 'kubernetesConfigDialog', + 'dockerConfigDialog'], function (require, $, log, Backbone, smartWizard, SiddhiAppSelectorDialog, JarsSelectorDialog, - TemplateAppDialog, TemplateConfigDialog, FillTemplateValueDialog, KubernetesConfigDialog) { + TemplateAppDialog, TemplateConfigDialog, FillTemplateValueDialog, KubernetesConfigDialog, DockerConfigDialog) { var ExportDialog = Backbone.View.extend( /** @lends ExportDialog.prototype */ @@ -36,6 +37,9 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect var exportDialog = _.cloneDeep(_.get(options.config, 'export_dialog')); this._exportContainer = $(_.get(exportDialog, 'selector')).clone(); + var exportKubeStep = _.cloneDeep(_.get(options.config, 'export_kube_step')); + this._exportKubeStepContainer = $(_.get(exportKubeStep, 'selector')).clone(); + this._isExportDockerFlow = isExportDockerFlow; this._payload = { templatedSiddhiApps: [], @@ -43,7 +47,8 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect templatedVariables: [], bundles: [], jars: [], - kubernetesConfiguration: '' + kubernetesConfiguration: '', + dockerConfiguration: '' }; this._siddhiAppSelector; this._jarsSelectorDialog; @@ -51,18 +56,20 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect this._configTemplateModel; this._kubernetesConfigModel; this._fill_template_value_dialog; + this._dockerConfigModel; + this._exportUrl; + this._exportType; - var type; if (isExportDockerFlow) { - type = 'docker'; + this._exportType = 'docker'; } else { - type = 'kubernetes'; + this._exportType = 'kubernetes'; } - var exportUrl = options.config.baseUrl + "/export?type=" + type; + this._exportUrl = options.config.baseUrl + "/export?exportType=" + this._exportType; this._btnExportForm = $('' + '
' + '' + - '
').attr('action', exportUrl); + ''); }, @@ -84,15 +91,9 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect } else { heading.text('Export Siddhi Apps For Kubernetes CRD'); form.find('#form-steps') - .append('
  • Step 6
    Add Kubernetes Config
    • '); - - form.find('#form-containers') - .append("\n" + - "
    \n" + - "
    \n" + - "
    Configure Kubernetes for Siddhi
    " + - "
    \n" + - "
    "); + .append('
  • Step 7
    Add Kubernetes Config
    • '); + + form.find('#form-containers').append(this._exportKubeStepContainer); } // Toolbar extra buttons @@ -201,6 +202,13 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect self._fill_template_value_dialog = new FillTemplateValueDialog(fillTemplateOptions); self._fill_template_value_dialog.render(); } else if (stepNumber === 5) { + self._dockerConfigModel = new DockerConfigDialog({ + app: self._options, + templateHeader: exportContainer.find('#docker-config-container-id'), + payload: self._payload + }); + self._dockerConfigModel.render(); + } else if (stepNumber === 6) { self._kubernetesConfigModel = new KubernetesConfigDialog({ app: self._options, templateHeader: exportContainer.find('#kubernetes-configuration-step-id') @@ -217,6 +225,7 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect if (!this._isExportDockerFlow) { this._payload.kubernetesConfiguration = this._kubernetesConfigModel.getKubernetesConfigs(); } + this._payload.dockerConfiguration = this._dockerConfigModel.getDockerConfigs(); this._payload.bundles = this._jarsSelectorDialog.getSelected('bundles'); this._payload.jars = this._jarsSelectorDialog.getSelected('jars'); @@ -225,6 +234,47 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect this._btnExportForm.append(payloadInputField); $(document.body).append(this._btnExportForm); + var exportUrl = this._exportUrl + var requestType = "downloadOnly" + + if (this._exportType == "docker") { + if (this._payload.dockerConfiguration.pushDocker && this._payload.dockerConfiguration.downloadDocker) { + requestType = "downloadAndBuild"; + } else if (this._payload.dockerConfiguration.pushDocker) { + requestType = "buildOnly"; + exportUrl = exportUrl + "&requestType=" + requestType; + $.ajax({ + type: "POST", + url: exportUrl, + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + data: {"payload": JSON.stringify(this._payload)}, + async: false, + success: function (response) { + result = {status: "success"}; + }, + error: function (error) { + if (error.responseText) { + result = {status: "fail", errorMessage: error.responseText}; + } else { + result = {status: "fail", errorMessage: "Error Occurred while processing your request"}; + } + } + }); + return; + } else { + requestType = "downloadOnly"; + } + } else if (this._exportType == "kubernetes") { + if (this._payload.dockerConfiguration.pushDocker) { + requestType = "downloadAndBuild"; + } else { + requestType = "downloadOnly"; + } + } + exportUrl = exportUrl + "&requestType=" + requestType; + this._btnExportForm = this._btnExportForm.attr('action', exportUrl) this._btnExportForm.submit(); }, @@ -235,6 +285,9 @@ define(['require', 'jquery', 'log', 'backbone', 'smart_wizard', 'siddhiAppSelect if (!_.isNil(this._btnExportForm)) { this._btnExportForm.remove(); } + if (!_.isNil(this._exportKubeStepContainer)) { + this._exportKubeStepContainer.remove(); + } } }); return ExportDialog; diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/lib/smartWizard/css/smart_wizard.css b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/lib/smartWizard/css/smart_wizard.css index 7855432f02..eaee0e562e 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/lib/smartWizard/css/smart_wizard.css +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/commons/lib/smartWizard/css/smart_wizard.css @@ -100,7 +100,7 @@ background: transparent !important; border: none !important; cursor: not-allowed; - height: 7%; + height: 86px; } .sw-theme-default > ul.step-anchor > li.clickable > a:hover { diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/css/export-docker-kubernetes.css b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/css/export-docker-kubernetes.css index a18dc8a273..d5ea93bb93 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/css/export-docker-kubernetes.css +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/css/export-docker-kubernetes.css @@ -115,6 +115,12 @@ hr.export-dialog-hr { .kubernetes-configuration-step { } + +.template-value-elements { + margin: 10px; + padding: 15px; +} + .sub-template-value-element-div .option { align-items: center; padding: 5px; @@ -125,7 +131,7 @@ hr.export-dialog-hr { .sub-template-value-element-div .option input { color: #fff !important; width: 75%; - background-color: #2C2C2C; + background-color: #222222; border-style: none; height: 35px; padding-left: 10px; @@ -135,7 +141,6 @@ hr.export-dialog-hr { .step-description { padding: 10px; background-color: transparent; - transition: background-color 500ms linear; } .export-siddhi-app-list { @@ -151,10 +156,95 @@ hr.export-dialog-hr { height: 98% !important; } -.sw-theme-default > ul.step-anchor.k8_export_header_item li { - width: 16%; +.k8_export_header { + margin-left: 10px; + margin-right: 10px; +} + +.sw-theme-default > ul.step-anchor.k8_export_header li { + max-width: 14.28%; + height: 86px; } .link-disabled { pointer-events: none; } + +.docker-config-container { + padding-top: 40px; + padding-left: 50px; +} + +.label-description { + padding-top: 2px; + padding-left: 8px; +} + +.docker-config-container input { + margin: 0 0 0 !important; +} + +.docker-details-form { + margin-top: 10px; + margin-left: 50px; + padding-left: 10px; + padding-top: 10px; +} + +.docker-details-row { + padding: 5px; +} + +.export-form-input { + padding-top: 2px; +} + +.docker-details-row input { + height: 30px !important; + width: 80% !important; + background-color: #222222 !important; +} + +.export-kube-config-container { + display: block; + overflow: auto; + width: 100%; + background-color: #2C2C2C !important; + padding: 10px; +} + +.kube-config-form { + padding-left: 10px; + padding-right: 10px; + overflow: auto; + width: 100%; + background-color: transparent; + margin: auto; +} + +.kube-config-row { + padding-top: 10px; + padding-left: 10px; + margin-top: 10px; +} + +.kube-config-sub-row { + padding: 10px 10px 5px; +} + +.kube-input-group { + padding: 5px 5px 5px 10px; +} + +.siddhi-process-name-input { + height: 30px !important; + background-color: #222222 !important; + margin-top: 2px; + width: 80%; +} + +.kubernetes-config-editor { + height: 150px !important; + width: 80%; !important; + padding-top: 2px !important; +} \ No newline at end of file diff --git a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/index.html b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/index.html index 83f88dac1f..b9ceeadcaf 100644 --- a/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/index.html +++ b/tooling/components/io.siddhi.distribution.editor.core/src/main/resources/web/index.html @@ -2294,6 +2294,7 @@

    Templated Variables

    @@ -2481,23 +2482,14 @@
    +