v4.0.2

build.gradle

@@ -1,11 +1,14 @@
-import groovy.json.JsonParserType
 import groovy.json.JsonSlurper
+import groovy.json.JsonParserType
+
 import java.nio.file.Files
 import java.nio.file.Paths
 import java.nio.file.StandardOpenOption
+
 description = "Mongoose is a high-load storage performance testing tool"
 
 ext {
+
 	depVersion = [
 		confuse            : "1.1.4",
 		confuseIoJson      : "1.0.4",
@@ -28,7 +31,6 @@ ext {
 		nettyTcNative      : "2.0.12.Final",
 	]
 
-
 	leafProjects = subprojects.findAll {
 		!it.path.equals(":load") &&
 			!it.path.equals(":load:step") &&
@@ -43,6 +45,7 @@ ext {
 }
 
 allprojects {
+
 	buildscript {
 		repositories {
 			mavenCentral()
@@ -51,9 +54,6 @@ allprojects {
 
 	repositories {
 		mavenCentral()
-		maven {
-			url "https://oss.sonatype.org/service/local/repositories/releases/content"
-		}
 	}
 
 	apply plugin: "java"
@@ -105,6 +105,7 @@ allprojects {
 	}
 
 	task perfTest(type: Test) {
+
 		include "com/emc/mongoose/perf/*"
 
 		jvmArgs "-XX:MaxDirectMemorySize=2g"
@@ -131,7 +132,9 @@ allprojects {
 	}
 
 	task systemTest(type: Test) {
+
 		include "com/emc/mongoose/system/*"
+
 		maxHeapSize "1g"
 		testLogging {
 			events "passed", "skipped", "failed", "standardOut"
@@ -195,6 +198,7 @@ allprojects {
 				}
 
 				pom {
+
 					groupId = rootProject.group
 					name = nameParts.join("-")
 					artifactId = name
@@ -230,20 +234,23 @@ allprojects {
 		}
 	}
 }
+
 String defineVersion() {
-	def configFilePath = [rootDir, "src", "main", "resources", "config", "defaults.json",]
+	def configFilePath = [ rootDir, "src", "main", "resources", "config", "defaults.json", ]
 		.join(File.separator);
 	def inputFile = new File(configFilePath)
 	def json = new JsonSlurper().setType(JsonParserType.LAX).parseText(inputFile.text)
 	json.run.version
 }
+
 task printVersion {
 	group = "versioning"
 	description = "Prints Mongoose version."
 	doLast { logger.quiet "Mongoose version: $version" }
 }
 
 dependencies {
+
 	provided(
 		"commons-codec:commons-codec:${depVersion.commonsCodec}",
 		"commons-lang:commons-lang:${depVersion.commonsLang}",
@@ -261,12 +268,12 @@ dependencies {
 		"io.prometheus:simpleclient_hotspot:${depVersion.tsdbMetrics}",
 		"org.apache.logging.log4j:log4j-api:${depVersion.log4j}",
 		"org.apache.logging.log4j:log4j-core:${depVersion.log4j}",
+		"org.eclipse.jetty:jetty-server:${depVersion.jetty}",
+		"org.eclipse.jetty:jetty-servlet:${depVersion.jetty}",
 		// actually the deps below are the runtime deps
 		"org.apache.logging.log4j:log4j-jul:${depVersion.log4j}",
 		"org.apache.logging.log4j:log4j-slf4j-impl:${depVersion.log4j}",
-		"org.slf4j:slf4j-api:${depVersion.slf4j}",
-		"org.eclipse.jetty:jetty-server:${depVersion.jetty}",
-		"org.eclipse.jetty:jetty-servlet:${depVersion.jetty}",
+		"org.slf4j:slf4j-api:${depVersion.slf4j}"
 	)
 
 	testCompile(
@@ -281,13 +288,16 @@ dependencies {
 		"org.mockito:mockito-core:2.23.0",
 	)
 }
+
 void collectName(Project p, LinkedList<String> names) {
 	names.addFirst p.name
 	if(p.parent != null) {
 		collectName(p.parent, names)
 	}
 }
+
 subprojects {
+
 	task sourcesJar(type: Jar, dependsOn: classes) {
 		LinkedList names = new LinkedList()
 		collectName(project, names)
@@ -314,6 +324,7 @@ subprojects {
 task wrapper(type: Wrapper) {
 	gradleVersion = "4.7"
 }
+
 static def isTestsProject(final Project p) {
 	if(p == null) {
 		return false
@@ -323,46 +334,49 @@ static def isTestsProject(final Project p) {
 		return isTestsProject(p.parent)
 	}
 }
+
 task mergeLog4j2PluginsDat(dependsOn: compileJava) {
 	doLast {
+
 		byte[] pluginsData
 		byte[] customPluginsData
 
 		configurations.provided
 			.each {
-			if(!it.isDirectory()) {
-				zipTree(it)
-					.filter {
-					it.toString().endsWith(log4j2PluginsDatPath.toString())
-				}
-				.each {
-					pluginsData = Files.readAllBytes(Paths.get(it.toString()))
+				if(!it.isDirectory()) {
+					zipTree(it)
+						.filter {
+							it.toString().endsWith(log4j2PluginsDatPath.toString())
+						}
+						.each {
+							pluginsData = Files.readAllBytes(Paths.get(it.toString()))
+						}
 				}
 			}
-		}
 
 		sourceSets.main.output.classesDirs
 			.any {
-			customPluginsData = Files.readAllBytes(
-				Paths.get(it.toString(), log4j2PluginsDatPath.toString(),)
-			)
-		}
+				customPluginsData = Files.readAllBytes(
+					Paths.get(it.toString(), log4j2PluginsDatPath.toString(),)
+				)
+			}
 
 		sourceSets.main.output.classesDirs
 			.any {
-			Files.write(
-				Paths.get(it.toString(), log4j2PluginsDatPath.toString(),),
-				pluginsData, StandardOpenOption.TRUNCATE_EXISTING
-			)
-			Files.write(
-				Paths.get(it.toString(), log4j2PluginsDatPath.toString(),),
-				customPluginsData, StandardOpenOption.APPEND
-			)
-		}
+				Files.write(
+					Paths.get(it.toString(), log4j2PluginsDatPath.toString(),),
+					pluginsData, StandardOpenOption.TRUNCATE_EXISTING
+				)
+				Files.write(
+					Paths.get(it.toString(), log4j2PluginsDatPath.toString(),),
+					customPluginsData, StandardOpenOption.APPEND
+				)
+			}
 	}
 }
 
 task dist(type: Jar) {
+
 	dependsOn += compileJava
 	dependsOn += mergeLog4j2PluginsDat
 
@@ -389,14 +403,14 @@ task dist(type: Jar) {
 
 	leafProjects
 		.each {
-		final Project p = it
-		from(p.jar.outputs.files) {
-			into "${File.separator}ext"
-			rename {
-				fileName -> "${p.description}.jar"
+			final Project p = it
+			from(p.jar.outputs.files) {
+				into "${File.separator}ext"
+				rename {
+					fileName -> "${p.description}.jar"
+				}
 			}
 		}
-	}
 }
 
 task sourcesJar(type: Jar, dependsOn: classes) {

doc/roadmap/README.md

@@ -0,0 +1,13 @@
+* Core
+    * Tests Solidification
+    * Measurements Solidification
+        * Fix the measurement issues like negative actual concurrency
+        * Generate the baselines
+    * Distributed Mode Complete Implementation
+        Make the monitoring/control API available from any node of the Mongoose cluster, e.g. the metrics should be able
+        to be aggregated from any node, the step may be stopped in the absence of the entry node, etc
+* GUI
+* Storage Drivers Implementation
+    * Pravega
+    * JDBC
+        Required to prove that Mongoose is abstract enough to work against any kind of data storage

src/main/java/com/emc/mongoose/Main.java

@@ -5,6 +5,7 @@
 import com.emc.mongoose.config.CliArgUtil;
 import com.emc.mongoose.config.ConfigUtil;
 import com.emc.mongoose.config.IllegalArgumentNameException;
+import com.emc.mongoose.control.ConfigServlet;
 import com.emc.mongoose.env.CoreResourcesToInstall;
 import com.emc.mongoose.env.Extension;
 import com.emc.mongoose.exception.InterruptRunException;
@@ -16,16 +17,31 @@
 import com.emc.mongoose.metrics.MetricsManager;
 import com.emc.mongoose.metrics.MetricsManagerImpl;
 import com.emc.mongoose.svc.Service;
+import static com.emc.mongoose.Constants.APP_NAME;
+import static com.emc.mongoose.Constants.DIR_EXAMPLE_SCENARIO;
+import static com.emc.mongoose.Constants.DIR_EXT;
+import static com.emc.mongoose.Constants.PATH_DEFAULTS;
+import static com.emc.mongoose.config.CliArgUtil.allCliArgs;
+import static com.emc.mongoose.load.step.Constants.ATTR_CONFIG;
+
 import com.github.akurilov.confuse.Config;
 import com.github.akurilov.confuse.SchemaProvider;
 import com.github.akurilov.confuse.exceptions.InvalidValuePathException;
 import com.github.akurilov.confuse.exceptions.InvalidValueTypeException;
+
 import io.prometheus.client.exporter.MetricsServlet;
+
 import org.apache.commons.lang.StringUtils;
+
 import org.apache.logging.log4j.Level;
+
+import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ThreadPool;
 
 import javax.script.ScriptEngine;
 import javax.script.ScriptException;
@@ -39,17 +55,12 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
-
-import static com.emc.mongoose.Constants.APP_NAME;
-import static com.emc.mongoose.Constants.DIR_EXAMPLE_SCENARIO;
-import static com.emc.mongoose.Constants.DIR_EXT;
-import static com.emc.mongoose.Constants.PATH_DEFAULTS;
-import static com.emc.mongoose.config.CliArgUtil.allCliArgs;
-import static com.emc.mongoose.load.step.Constants.ATTR_CONFIG;
 import static javax.script.ScriptContext.ENGINE_SCOPE;
 
 public final class Main {
 
+	private static final int JETTY_THREAD_COUNT = 4;
+
 	public static void main(final String... args)
 	throws Exception {
 		final CoreResourcesToInstall coreResources = new CoreResourcesToInstall();
@@ -71,11 +82,12 @@ public static void main(final String... args)
 				// install the extensions
 				installExtensions(extensions, appHomePath);
 				final Config configWithArgs;
+				final Config fullDefaultConfig;
 				try {
 					// apply the extensions defaults
-					final Config config = collectDefaults(extensions, defaultConfig, appHomePath);
+					fullDefaultConfig = collectDefaults(extensions, defaultConfig, appHomePath);
 					// parse the CLI args and apply them to the config instance
-					configWithArgs = applyArgsToConfig(args, config, initialStepId);
+					configWithArgs = applyArgsToConfig(args, fullDefaultConfig, initialStepId);
 				} catch(final Exception e) {
 					LogUtil.exception(Level.ERROR, e, "Failed to load the defaults");
 					throw e;
@@ -86,8 +98,12 @@ public static void main(final String... args)
 				if(configWithArgs.boolVal("run-node")) {
 					runNode(configWithArgs, extensions, metricsMgr);
 				} else {
-					final Server server = new Server(port);
-					addMetricsService(server);
+					final ThreadPool tp = new QueuedThreadPool(JETTY_THREAD_COUNT, 1);
+					final Server server = new Server(tp);
+					final ServerConnector connector = new ServerConnector(server);
+					connector.setPort(port);
+					server.setConnectors(new Connector[] {connector});
+					addServices(server, fullDefaultConfig);
 					server.start();
 					try {
 						runScenario(configWithArgs, extensions, extClsLoader, metricsMgr, appHomePath);
@@ -103,6 +119,14 @@ public static void main(final String... args)
 		}
 	}
 
+	private static void addServices(final Server server, final Config defaultConfig) {
+		final ServletContextHandler context = new ServletContextHandler();
+		context.setContextPath("/");
+		server.setHandler(context);
+		context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
+		context.addServlet(new ServletHolder(new ConfigServlet(defaultConfig)), "/config/*");
+	}
+
 	private static Config loadDefaultConfig(final Path appHomePath)
 	throws Exception {
 		final Map<String, Object> mainConfigSchema = SchemaProvider
@@ -186,20 +210,6 @@ private static Map<String, String> argsWithAliases(final String[] args, final Co
 		return AliasingUtil.apply(parsedArgs, aliasingConfig);
 	}
 
-	private static void addMetricsService(final Server server) {
-		final ServletContextHandler context = new ServletContextHandler();
-		context.setContextPath("/");
-		server.setHandler(context);
-		context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
-	}
-
-	private static void addControlService(final Server server) {
-		final ServletContextHandler context = new ServletContextHandler();
-		context.setContextPath("/");
-		server.setHandler(context);
-		//context.addServlet(new ServletHolder(new MetricsServlet()), "/control");
-	}
-
 	private static void runNode(final Config config, final List<Extension> extensions, final MetricsManager metricsMgr)
 	throws InterruptRunException, InterruptedException {
 		final int listenPort = config.intVal("load-step-node-port");