Skip to content

Commit

Permalink
Allow parallel generation of service execution artifacts (finos#669)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevin-m-knight-gs
kevin-m-knight-gs authored Jun 21, 2023
1 parent 0ae54f3 commit e87037c
Show file tree
Hide file tree
Showing 8 changed files with 850 additions and 210 deletions.
17 changes: 10 additions & 7 deletions legend-sdlc-generation-service-maven-plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,6 @@
<name>Legend SDLC Service Generation Maven Plugin</name>

<dependencies>
<!-- PURE -->
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-code-compiled-core</artifactId>
</dependency>
<!-- PURE -->

<!-- ENGINE -->
<dependency>
<groupId>org.finos.legend.engine</groupId>
Expand Down Expand Up @@ -141,6 +134,12 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-grammar</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
Expand All @@ -156,6 +155,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>*</artifactId>
Expand Down
245 changes: 193 additions & 52 deletions ...plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java
View file

Large diffs are not rendered by default.

91 changes: 77 additions & 14 deletions ...in/src/test/java/org/finos/legend/sdlc/generation/service/TestServicesGenerationMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.finos.legend.sdlc.serialization.EntitySerializers;
import org.finos.legend.sdlc.tools.entity.EntityPaths;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
Expand Down Expand Up @@ -58,17 +59,17 @@ public class TestServicesGenerationMojo
private static final String GOAL = "generate-service-executions";
private static final String SERVICE_CLASSIFIER = "meta::legend::service::metamodel::Service";

@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
@ClassRule
public static TemporaryFolder TMP_FOLDER = new TemporaryFolder();

@Rule
public MojoRule mojoRule = new MojoRule();

@Test
public void testEmptyEntitiesDirectory() throws Exception
{
File[] emptyEntityDirs = {this.tempFolder.newFolder("empty1"), this.tempFolder.newFolder("empty2")};
File projectDir = buildSingleModuleProject("project", "org.finos.test", "test-project", "1.0.0", null, null, emptyEntityDirs, null, null, null, "org.finos.test.test_project", null, null);
File[] emptyEntityDirs = {TMP_FOLDER.newFolder(), TMP_FOLDER.newFolder()};
File projectDir = buildSingleModuleProject("org.finos.test", "test-project", "1.0.0", null, null, emptyEntityDirs, null, null, null, "org.finos.test.test_project", null, null);

MavenProject mavenProject = this.mojoRule.readMavenProject(projectDir);

Expand All @@ -81,9 +82,9 @@ public void testEmptyEntitiesDirectory() throws Exception
@Test
public void testNonExistentEntitiesDirectory() throws Exception
{
File tempPath = this.tempFolder.getRoot();
File tempPath = TMP_FOLDER.newFolder();
File[] nonExistentEntityDirectories = {new File(tempPath, "nonexistent1"), new File(tempPath, "nonexistent2")};
File projectDir = buildSingleModuleProject("project", "org.finos.test", "test-project", "1.0.0", null, null, nonExistentEntityDirectories, null, null, null, "org.finos.test.test_project", null, null);
File projectDir = buildSingleModuleProject("org.finos.test", "test-project", "1.0.0", null, null, nonExistentEntityDirectories, null, null, null, "org.finos.test.test_project", null, null);

MavenProject mavenProject = this.mojoRule.readMavenProject(projectDir);

Expand All @@ -96,14 +97,14 @@ public void testNonExistentEntitiesDirectory() throws Exception
@Test
public void testNoServices() throws Exception
{
File entitiesDir = this.tempFolder.newFolder("testEntities");
File entitiesDir = TMP_FOLDER.newFolder();
try (EntityLoader testEntities = getTestEntities())
{
testEntities.getAllEntities()
.filter(e -> !isServiceEntity(e))
.forEach(e -> writeEntityToDirectory(entitiesDir.toPath(), e));
}
File projectDir = buildSingleModuleProject("project", "org.finos.test", "test-project", "1.0.0", null, null, new File[]{entitiesDir}, null, null, null, "org.finos.test.test_project", null, null);
File projectDir = buildSingleModuleProject("org.finos.test", "test-project", "1.0.0", null, null, new File[]{entitiesDir}, null, null, null, "org.finos.test.test_project", null, null);

MavenProject mavenProject = this.mojoRule.readMavenProject(projectDir);

Expand All @@ -117,15 +118,15 @@ public void testNoServices() throws Exception
public void testWithServices() throws Exception
{
String packagePrefix = "org.finos.test.test_project";
File entitiesDir = this.tempFolder.newFolder("testEntities");
File entitiesDir = TMP_FOLDER.newFolder();
List<Entity> entities;
try (EntityLoader testEntities = getTestEntities())
{
entities = testEntities.getAllEntities().collect(Collectors.toList());
}
Assert.assertEquals(5, entities.stream().filter(this::isServiceEntity).count());
entities.forEach(e -> writeEntityToDirectory(entitiesDir.toPath(), e));
File projectDir = buildSingleModuleProject("project", "org.finos.test", "test-project", "1.0.0", null, null, new File[]{entitiesDir}, null, null, null, packagePrefix, null, null);
File projectDir = buildSingleModuleProject("org.finos.test", "test-project", "1.0.0", null, null, new File[]{entitiesDir}, null, null, null, packagePrefix, null, null);

MavenProject mavenProject = this.mojoRule.readMavenProject(projectDir);

Expand All @@ -145,6 +146,68 @@ public void testWithServices() throws Exception
Assert.assertEquals(Collections.emptyList(), expectedServiceClassJavaPaths.reject(actualGeneratedSourceFiles::contains));
}

@Test
public void testParseParallel()
{
int procCount = Runtime.getRuntime().availableProcessors();

// Null, empty, whitespace
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel(null));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel(""));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel(" \r\t\n \r\n \n\r "));

// Integers
for (int i = 0; i < 256; i++)
{
String s = Integer.toString(i);
Assert.assertEquals(s, i, ServicesGenerationMojo.parseParallel(s));
Assert.assertEquals(s, i, ServicesGenerationMojo.parseParallel("+" + s));
Assert.assertEquals(s, -i, ServicesGenerationMojo.parseParallel("-" + s));
Assert.assertEquals(s, i, ServicesGenerationMojo.parseParallel("00000000" + s));
Assert.assertEquals(s, i, ServicesGenerationMojo.parseParallel(" " + s + " "));
}

// False
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel("false"));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel("FALSE"));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel("False"));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel("FaLsE"));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel(" false "));
Assert.assertEquals(1, ServicesGenerationMojo.parseParallel(" false "));

// True
int defaultParallelism = procCount - 1;
Assert.assertEquals(defaultParallelism, ServicesGenerationMojo.parseParallel("true"));
Assert.assertEquals(defaultParallelism, ServicesGenerationMojo.parseParallel("TRUE"));
Assert.assertEquals(defaultParallelism, ServicesGenerationMojo.parseParallel("True"));
Assert.assertEquals(defaultParallelism, ServicesGenerationMojo.parseParallel("TrUe"));
Assert.assertEquals(defaultParallelism, ServicesGenerationMojo.parseParallel(" true "));
Assert.assertEquals(defaultParallelism, ServicesGenerationMojo.parseParallel(" true "));

// Processor based
Assert.assertEquals(procCount, ServicesGenerationMojo.parseParallel("C"));
Assert.assertEquals(procCount, ServicesGenerationMojo.parseParallel("c"));
Assert.assertEquals(procCount, ServicesGenerationMojo.parseParallel(" C "));
Assert.assertEquals(procCount, ServicesGenerationMojo.parseParallel("1.0C"));
Assert.assertEquals(procCount, ServicesGenerationMojo.parseParallel("1C"));
Assert.assertEquals(2 * procCount, ServicesGenerationMojo.parseParallel("2.0C"));
Assert.assertEquals(2 * procCount, ServicesGenerationMojo.parseParallel("2\tC"));
Assert.assertEquals(Math.round(2.5 * procCount), ServicesGenerationMojo.parseParallel("2.5C"));
Assert.assertEquals(Math.round(12.3 * procCount), ServicesGenerationMojo.parseParallel("12.3 C"));
Assert.assertEquals(procCount - 1, ServicesGenerationMojo.parseParallel("C-1"));
Assert.assertEquals(procCount - 1, ServicesGenerationMojo.parseParallel("C - 1"));
Assert.assertEquals(procCount + 1, ServicesGenerationMojo.parseParallel("C + 1"));
Assert.assertEquals(procCount + 2, ServicesGenerationMojo.parseParallel("C + 2"));
Assert.assertEquals(Math.round(2.5 * procCount) + 2, ServicesGenerationMojo.parseParallel("2.5C + 2"));
Assert.assertEquals(Math.round(3.5 * procCount) - 1, ServicesGenerationMojo.parseParallel("3.5c-1"));

for (String invalid : new String[]{"blah", "trueee", "null", "123.123", "2.5*C", "C / 5"})
{
RuntimeException e = Assert.assertThrows(RuntimeException.class, () -> ServicesGenerationMojo.parseParallel(invalid));
Assert.assertEquals(invalid, "Could not parse parallel value: \"" + invalid + "\"", e.getMessage());
}
}

private boolean isServiceEntity(Entity entity)
{
return SERVICE_CLASSIFIER.equals(entity.getClassifierPath());
Expand Down Expand Up @@ -209,15 +272,15 @@ private void writeEntityToDirectory(Path directory, Entity entity)
}
}

private File buildSingleModuleProject(String projectDirName, String groupId, String artifactId, String version, Set<String> includePaths, Set<String> includePackages, File[] includeDirectories, Set<String> excludePaths, Set<String> excludePackages, File[] excludeDirectories, String packagePrefix, File javaSourceOutputDirectory, File resourceOutputDirectory) throws IOException
private File buildSingleModuleProject(String groupId, String artifactId, String version, Set<String> includePaths, Set<String> includePackages, File[] includeDirectories, Set<String> excludePaths, Set<String> excludePackages, File[] excludeDirectories, String packagePrefix, File javaSourceOutputDirectory, File resourceOutputDirectory) throws IOException
{
Model mavenModel = buildMavenModelWithPlugin(groupId, artifactId, version, includePaths, includePackages, includeDirectories, excludePaths, excludePackages, excludeDirectories, packagePrefix, javaSourceOutputDirectory, resourceOutputDirectory);
return buildProject(projectDirName, mavenModel);
return buildProject(mavenModel);
}

private File buildProject(String projectDirName, Model mainModel, Model... childModels) throws IOException
private File buildProject(Model mainModel, Model... childModels) throws IOException
{
File projectDir = this.tempFolder.newFolder(projectDirName);
File projectDir = TMP_FOLDER.newFolder();
serializeMavenModel(projectDir, mainModel);
for (Model childModel : childModels)
{
Expand Down
5 changes: 5 additions & 0 deletions legend-sdlc-generation-service/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@
</dependency>
<!-- SDLC -->

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
Expand Down
Loading

0 comments on commit e87037c

Please sign in to comment.