From 0e4447e23655c9712dbb47349670968a92f47f96 Mon Sep 17 00:00:00 2001 From: Yas Okada Date: Thu, 19 Dec 2024 09:44:32 +0000 Subject: [PATCH] bump embulk to 0.10.39 --- .github/workflows/gem-push.yml | 2 +- build.gradle | 96 ++++++------------- .../AbstractCloudwatchLogsInputPlugin.java | 21 ++-- .../CloudwatchLogsInputPlugin.java | 7 +- .../aws/AwsCredentialsConfig.java | 2 +- .../aws/AwsCredentialsTask.java | 6 +- .../cloudwatch_logs/TestAwsCredentials.java | 8 +- .../TestCloudwatchLogsInputPlugin.java | 5 +- 8 files changed, 59 insertions(+), 88 deletions(-) diff --git a/.github/workflows/gem-push.yml b/.github/workflows/gem-push.yml index 5345b86..f7bb584 100644 --- a/.github/workflows/gem-push.yml +++ b/.github/workflows/gem-push.yml @@ -23,5 +23,5 @@ jobs: uses: trocco-io/push-gem-to-gpr-action@v1 with: language: java - gem-path: "./pkg/*.gem" + gem-path: "./build/gems/*.gem" github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/build.gradle b/build.gradle index fbda876..11fe253 100644 --- a/build.gradle +++ b/build.gradle @@ -1,12 +1,11 @@ plugins { - id "com.jfrog.bintray" version "1.1" id "com.github.jruby-gradle.base" version "1.5.0" id "java" id "checkstyle" + id "org.embulk.embulk-plugins" version "0.5.5" id "com.palantir.git-version" version "3.0.0" } -import com.github.jrubygradle.JRubyExec repositories { mavenCentral() jcenter() @@ -28,19 +27,22 @@ sourceCompatibility = 1.8 targetCompatibility = 1.8 dependencies { - compile "org.embulk:embulk-core:0.10.1" - provided "org.embulk:embulk-core:0.10.1" + def embulkVersion = "0.10.39" + compileOnly "org.embulk:embulk-api:${embulkVersion}" + compileOnly "org.embulk:embulk-spi:${embulkVersion}" + implementation "org.embulk:embulk-util-config:0.3.4" + implementation "org.embulk:embulk-util-timestamp:0.2.2" implementation "com.google.guava:guava:28.2-jre" - compile group: 'com.amazonaws', name: 'aws-java-sdk-logs', version: '1.11.749' - compile group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.11.749' - // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION" - testCompile "junit:junit:4.+" - testCompile "org.mockito:mockito-core:1.+" - testCompile "org.embulk:embulk-core:0.10.1:tests" - testCompile "org.embulk:embulk-standards:0.10.1" - testCompile "org.embulk:embulk-junit4:0.10.1" - testCompile "org.embulk:embulk-deps-buffer:0.10.1" - testCompile "org.embulk:embulk-deps-config:0.10.1" + implementation group: 'com.amazonaws', name: 'aws-java-sdk-logs', version: '1.11.749' + implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.11.749' + + testImplementation "junit:junit:4.+" + testImplementation "org.mockito:mockito-core:1.+" + testImplementation "org.embulk:embulk-api:${embulkVersion}" + testImplementation "org.embulk:embulk-spi:${embulkVersion}" + testImplementation "org.embulk:embulk-core:${embulkVersion}" + testImplementation "org.embulk:embulk-deps:${embulkVersion}" + testImplementation "org.embulk:embulk-junit4:${embulkVersion}" } // add tests/javadoc/source jar tasks as artifacts to be released @@ -57,11 +59,12 @@ task javadocJar(type: Jar, dependsOn: javadoc) { from javadoc.destinationDir } -task classpath(type: Copy, dependsOn: ["jar"]) { - doFirst { file("classpath").deleteDir() } - from (configurations.runtime - configurations.provided + files(jar.archivePath)) - into "classpath" +embulkPlugin { + mainClass = "org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin" + category = "input" + type = "cloudwatch_logs" } + clean { delete "classpath" } checkstyle { @@ -81,54 +84,11 @@ task checkstyle(type: Checkstyle) { source = sourceSets.main.allJava + sourceSets.test.allJava } -task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) { - jrubyArgs "-S" - script "gem" - scriptArgs "build", "${project.name}.gemspec" - doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") } -} - -task gemPush(type: JRubyExec, dependsOn: ["gem"]) { - jrubyArgs "-S" - script "gem" - scriptArgs "push", "pkg/${project.name}-${project.version}.gem" -} - -task "package"(dependsOn: ["gemspec", "classpath"]) { - doLast { - println "> Build succeeded." - println "> You can run embulk with '-L ${file(".").absolutePath}' argument." - } -} - -artifacts { - archives testsJar, sourcesJar, javadocJar -} - -task gemspec { - ext.gemspecFile = file("${project.name}.gemspec") - inputs.file "build.gradle" - outputs.file gemspecFile - doLast { gemspecFile.write($/ -Gem::Specification.new do |spec| - spec.name = "${project.name}" - spec.version = "${project.version}" - spec.authors = ["Hiroshi Hatake"] - spec.summary = %[Cloudwatch Logs input plugin for Embulk] - spec.description = %[Loads records from Cloudwatch Logs.] - spec.email = ["cosmo0920.wp@gmail.com"] - spec.licenses = ["MIT"] - spec.homepage = "https://github.com/cosmo0920/embulk-input-cloudwatch_logs" - - spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"] - spec.test_files = spec.files.grep(%r"^(test|spec)/") - spec.require_paths = ["lib"] - - #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION'] - spec.add_development_dependency 'bundler', ['~> 1.0'] - spec.add_development_dependency 'rake', ['~> 12.0'] -end -/$) - } +gem { + from("LICENSE") // Optional -- if you need other files in the gem. + authors = [ "d-hrs" ] + email = [ "s1150199@gmail.com" ] + summary = "output plugin for Embulk" + homepage = "https://github.com/primenumber-dev/embulk-output-" + licenses = [ "Apache-2.0" ] } -clean { delete "${project.name}.gemspec" } diff --git a/src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java b/src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java index 99548aa..e0fc480 100644 --- a/src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java +++ b/src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java @@ -1,22 +1,24 @@ package org.embulk.input.cloudwatch_logs; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; - -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; +import java.util.Optional; import org.embulk.config.ConfigDiff; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; -import org.embulk.config.Task; import org.embulk.config.TaskReport; import org.embulk.config.TaskSource; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; +import org.embulk.util.config.ConfigMapper; +import org.embulk.util.config.ConfigMapperFactory; +import org.embulk.util.config.Task; +import org.embulk.util.config.TaskMapper; import org.embulk.spi.Exec; import org.embulk.spi.InputPlugin; import org.embulk.spi.PageBuilder; @@ -46,6 +48,11 @@ public abstract class AbstractCloudwatchLogsInputPlugin implements InputPlugin { + static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = + ConfigMapperFactory.builder().addDefaultModules().build(); + private static final ConfigMapper CONFIG_MAPPER = CONFIG_MAPPER_FACTORY.createConfigMapper(); + private static final TaskMapper TASK_MAPPER = CONFIG_MAPPER_FACTORY.createTaskMapper(); + private static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; public interface PluginTask @@ -85,7 +92,7 @@ public interface PluginTask public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control) { - PluginTask task = config.loadConfig(getTaskClass()); + PluginTask task = CONFIG_MAPPER.map(config, getTaskClass()); Schema schema = new Schema.Builder() .add("timestamp", Types.TIMESTAMP) @@ -133,7 +140,7 @@ public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) { - PluginTask task = taskSource.loadTask(getTaskClass()); + PluginTask task = TASK_MAPPER.map(taskSource, getTaskClass()); AWSLogs client = newLogsClient(task); CloudWatchLogsDrainer drainer = new CloudWatchLogsDrainer(task, client); diff --git a/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java b/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java index 874ea3a..a44a084 100644 --- a/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java +++ b/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java @@ -2,11 +2,12 @@ import com.amazonaws.services.logs.AWSLogsClientBuilder; import com.amazonaws.services.logs.AWSLogs; -import com.google.common.base.Optional; -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; import org.embulk.config.ConfigException; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; + +import java.util.Optional; public class CloudwatchLogsInputPlugin extends AbstractCloudwatchLogsInputPlugin diff --git a/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java b/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java index 14fb20a..a566664 100644 --- a/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java +++ b/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java @@ -1,6 +1,6 @@ package org.embulk.input.cloudwatch_logs.aws; -import org.embulk.spi.unit.LocalFile; +import org.embulk.util.config.units.LocalFile; import java.util.Optional; diff --git a/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java b/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java index 50c072a..422bb6d 100644 --- a/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java +++ b/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java @@ -1,8 +1,8 @@ package org.embulk.input.cloudwatch_logs.aws; -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; -import org.embulk.spi.unit.LocalFile; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; +import org.embulk.util.config.units.LocalFile; import java.util.Optional; diff --git a/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java b/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java index 8f68b59..b5cb5c6 100644 --- a/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java +++ b/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java @@ -31,6 +31,7 @@ import static org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin.CloudWatchLogsPluginTask; import static org.junit.Assume.assumeNotNull; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -44,6 +45,7 @@ public class TestAwsCredentials .registerPlugin(InputPlugin.class, "cloudwatch_logs", CloudwatchLogsInputPlugin.class) .build(); + private final org.embulk.util.config.ConfigMapper configMapper = CloudwatchLogsInputPlugin.CONFIG_MAPPER_FACTORY.createConfigMapper(); private CloudwatchLogsInputPlugin plugin; private ConfigSource config; @@ -81,7 +83,7 @@ public void setUp() throws IOException logStreamName = TestHelpers.getLogStreamName(); if (plugin == null) { - plugin = Mockito.spy(new CloudwatchLogsInputPlugin()); + plugin = spy(new CloudwatchLogsInputPlugin()); config = runtime.getExec().newConfigSource() .set("type", "cloudwatch_logs") .set("log_group_name", logGroupName) @@ -102,8 +104,8 @@ public void tearDown() throws IOException private void doTest(ConfigSource config) throws IOException { - CloudWatchLogsPluginTask task = config.loadConfig(CloudWatchLogsPluginTask.class); - CloudwatchLogsInputPlugin plugin = runtime.getInstance(CloudwatchLogsInputPlugin.class); + CloudWatchLogsPluginTask task = configMapper.map(config, CloudWatchLogsPluginTask.class); + CloudwatchLogsInputPlugin plugin = spy(new CloudwatchLogsInputPlugin()); logsClient = plugin.newLogsClient(task); testUtils = new CloudWatchLogsTestUtils(logsClient, logGroupName, logStreamName); diff --git a/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java b/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java index 2e3a4a1..6c29ade 100644 --- a/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java +++ b/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java @@ -31,6 +31,7 @@ import static org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin.CloudWatchLogsPluginTask; import static org.junit.Assume.assumeNotNull; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -93,7 +94,7 @@ public void setUp() throws IOException } doReturn(pageBuilder).when(plugin).getPageBuilder(Mockito.any(), Mockito.any()); CloudWatchLogsPluginTask task = config.loadConfig(CloudWatchLogsPluginTask.class); - CloudwatchLogsInputPlugin plugin = runtime.getInstance(CloudwatchLogsInputPlugin.class); + CloudwatchLogsInputPlugin plugin = spy(new CloudwatchLogsInputPlugin()); logsClient = plugin.newLogsClient(task); testUtils = new CloudWatchLogsTestUtils(logsClient, logGroupName, logStreamName); } @@ -135,7 +136,7 @@ public void configuredRegion() .set("region", "ap-southeast-2") .remove("endpoint") .loadConfig(CloudWatchLogsPluginTask.class); - CloudwatchLogsInputPlugin plugin = runtime.getInstance(CloudwatchLogsInputPlugin.class); + CloudwatchLogsInputPlugin plugin = spy(new CloudwatchLogsInputPlugin()); AWSLogs logsClient = plugin.newLogsClient(task); // Should not be null