Merge branch 'master' into json

.github/workflows/jre.yml

@@ -12,5 +12,5 @@ jobs:
     - uses: docker/build-push-action@v4
       with:
         context: "docker/jre"
-        tags: "neowu/jre:21"
+        tags: "neowu/jre:21.0.1"
         push: true

CHANGELOG.md

@@ -1,6 +1,11 @@
 ## Change log
 
-### 9.0.4 (12/20/2023 - )
+### 9.0.4 (12/20/2023 - 1/9/2024)
+
+* jre: published neowu/jre:21.0.1
+* mysql: aggressively simplified mysql jdbc driver, removed unused features
+  > add slow query support, decoupled core-ng and mysql classes
+  > must use "core.framework.mysql:mysql-connector-j:8.2.0-r3"
 
 ### 9.0.3 (12/12/2023 - 12/19/2023)
 

build.gradle.kts

@@ -7,7 +7,7 @@ apply(plugin = "project")
 
 subprojects {
     group = "core.framework"
-    version = "9.0.3"
+    version = "9.0.4"
 
     repositories {
         maven {
@@ -22,9 +22,9 @@ subprojects {
 val elasticVersion = "8.11.1"
 val kafkaVersion = "3.6.1"
 val jacksonVersion = "2.16.1"
-val junitVersion = "5.10.0"
-val mockitoVersion = "5.6.0"
-val assertjVersion = "3.24.2"
+val junitVersion = "5.10.1"
+val mockitoVersion = "5.8.0"
+val assertjVersion = "3.25.1"
 
 project("core-ng-api") {
     apply(plugin = "lib")
@@ -42,7 +42,6 @@ project("core-ng") {
         implementation("io.undertow:undertow-core:2.3.10.Final")
         implementation("org.apache.kafka:kafka-clients:${kafkaVersion}@jar")
         implementation("org.xerial.snappy:snappy-java:1.1.10.5")      // used by kafka message compression
-        compileOnly("com.mysql:mysql-connector-j:8.2.0@jar")
         compileOnly("org.jboss.logging:jboss-logging-annotations:2.2.1.Final")
         compileOnly("com.github.spotbugs:spotbugs-annotations:4.8.0")
         testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")

buildSrc/build.gradle.kts

@@ -14,7 +14,7 @@ java {
 }
 
 dependencies {
-    implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.1.5")
+    implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.2.5")
     implementation("org.flywaydb:flyway-gradle-plugin:10.1.0")
     implementation("org.flywaydb:flyway-mysql:10.1.0")
 }

buildSrc/src/main/check/spotbugs.xml

@@ -419,7 +419,6 @@
             NIR_NEEDLESS_INSTANCE_RETRIEVAL,
             DDC_DOUBLE_DATE_COMPARISON,
             SWCO_SUSPICIOUS_WAIT_ON_CONCURRENT_OBJECT,
-            JVR_JDBC_VENDOR_RELIANCE,
             PMB_POSSIBLE_MEMORY_BLOAT,
             PMB_INSTANCE_BASED_THREAD_LOCAL,
             LSYC_LOCAL_SYNCHRONIZED_COLLECTION,

core-ng/src/main/java/core/framework/db/QueryDiagnostic.java

@@ -0,0 +1,9 @@
+package core.framework.db;
+
+public interface QueryDiagnostic {
+    String sql();
+
+    boolean noGoodIndexUsed();
+
+    boolean noIndexUsed();
+}

core-ng/src/main/java/core/framework/internal/db/DatabaseImpl.java

@@ -121,7 +121,6 @@ Properties driverProperties(String url) {
             // force to UTC, generally on cloud it defaults to UTC, this setting is to make local match cloud
             properties.setProperty("connectionTimeZone", "UTC");
             properties.setProperty("rewriteBatchedStatements", "true");
-            properties.setProperty("queryInterceptors", MySQLQueryInterceptor.class.getName());
             properties.setProperty("logger", "Slf4JLogger");
             properties.setProperty("cachePrepStmts", "true");
 

core-ng/src/main/java/core/framework/internal/db/DatabaseOperation.java

@@ -1,9 +1,14 @@
 package core.framework.internal.db;
 
+import core.framework.db.QueryDiagnostic;
 import core.framework.db.UncheckedSQLException;
+import core.framework.internal.log.ActionLog;
+import core.framework.internal.log.LogManager;
 import core.framework.internal.resource.Pool;
 import core.framework.internal.resource.PoolItem;
 import core.framework.util.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.math.BigDecimal;
 import java.sql.Connection;
@@ -19,12 +24,15 @@
 import java.util.Optional;
 import java.util.OptionalLong;
 
+import static core.framework.log.Markers.errorCode;
 import static core.framework.util.Strings.format;
 
 /**
  * @author neo
  */
 public class DatabaseOperation {
+    private final Logger logger = LoggerFactory.getLogger(DatabaseOperation.class);
+
     public final TransactionManager transactionManager;
     final EnumDBMapper enumMapper = new EnumDBMapper();
     int queryTimeoutInSeconds;
@@ -40,7 +48,9 @@ int update(String sql, Object... params) {
         try (PreparedStatement statement = connection.resource.prepareStatement(sql)) {
             statement.setQueryTimeout(queryTimeoutInSeconds);
             setParams(statement, params);
-            return statement.executeUpdate();
+            int result = statement.executeUpdate();
+            logSlowQuery(statement);
+            return result;
         } catch (SQLException e) {
             Connections.checkConnectionState(connection, e);
             throw new UncheckedSQLException(e);
@@ -59,7 +69,9 @@ int[] batchUpdate(String sql, List<Object[]> params) {
                 setParams(statement, batchParams);
                 statement.addBatch();
             }
-            return statement.executeBatch();
+            int[] results = statement.executeBatch();
+            logSlowQuery(statement);
+            return results;
         } catch (SQLException e) {
             Connections.checkConnectionState(connection, e);
             throw new UncheckedSQLException(e);
@@ -141,6 +153,8 @@ private PreparedStatement insertStatement(Connection connection, String sql, Str
 
     private <T> Optional<T> fetchOne(PreparedStatement statement, RowMapper<T> mapper) throws SQLException {
         try (ResultSet resultSet = statement.executeQuery()) {
+            logSlowQuery(statement);
+
             T result = null;
             if (resultSet.next()) {
                 result = mapper.map(new ResultSetWrapper(resultSet));
@@ -153,6 +167,8 @@ private <T> Optional<T> fetchOne(PreparedStatement statement, RowMapper<T> mappe
 
     private <T> List<T> fetch(PreparedStatement statement, RowMapper<T> mapper) throws SQLException {
         try (ResultSet resultSet = statement.executeQuery()) {
+            logSlowQuery(statement);
+
             var wrapper = new ResultSetWrapper(resultSet);
             List<T> results = Lists.newArrayList();
             while (resultSet.next()) {
@@ -228,4 +244,22 @@ private void setParam(PreparedStatement statement, int index, Object param) thro
             default -> throw new Error(format("unsupported param type, type={}, value={}", param.getClass().getCanonicalName(), param));
         }
     }
+
+    void logSlowQuery(PreparedStatement statement) {
+        if (statement instanceof QueryDiagnostic diagnostic) {
+            boolean noIndexUsed = diagnostic.noIndexUsed();
+            boolean badIndexUsed = diagnostic.noGoodIndexUsed();
+            if (!noIndexUsed && !badIndexUsed) return;
+
+            ActionLog actionLog = LogManager.CURRENT_ACTION_LOG.get();
+            boolean warning = actionLog == null || !actionLog.warningContext.suppressSlowSQLWarning;
+            String message = noIndexUsed ? "no index used" : "bad index used";
+            String sqlValue = diagnostic.sql();
+            if (warning) {
+                logger.warn(errorCode("SLOW_SQL"), "{}, sql={}", message, sqlValue);
+            } else {
+                logger.debug("{}, sql={}", message, sqlValue);
+            }
+        }
+    }
 }

core-ng/src/test/java/core/framework/internal/db/MySQLQueryInterceptorTest.java → core-ng/src/test/java/core/framework/internal/db/DatabaseOperationTest.java

@@ -1,8 +1,7 @@
 package core.framework.internal.db;
 
-import com.mysql.cj.protocol.Resultset;
-import com.mysql.cj.protocol.ServerSession;
 import core.framework.db.Database;
+import core.framework.db.QueryDiagnostic;
 import core.framework.internal.log.ActionLog;
 import core.framework.internal.log.LogManager;
 import org.junit.jupiter.api.AfterEach;
@@ -12,25 +11,24 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import java.sql.PreparedStatement;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
 
-/**
- * @author neo
- */
 @ExtendWith(MockitoExtension.class)
-class MySQLQueryInterceptorTest {
-    @Mock
-    ServerSession session;
-    private MySQLQueryInterceptor interceptor;
+class DatabaseOperationTest {
+    @Mock(extraInterfaces = QueryDiagnostic.class)
+    PreparedStatement statement;
+    private DatabaseOperation operation;
     private LogManager logManager;
     private ActionLog actionLog;
 
     @BeforeEach
-    void createMySQLQueryInterceptor() {
-        logManager = new LogManager();
-        interceptor = new MySQLQueryInterceptor();
+    void createDatabaseOperation() {
+        operation = new DatabaseOperation(null);
 
+        logManager = new LogManager();
         actionLog = logManager.begin("begin", null);
     }
 
@@ -40,24 +38,28 @@ void cleanup() {
     }
 
     @Test
-    void defaultBehavior() {
-        assertThat(interceptor.init(null, null, null)).isSameAs(interceptor);
-        Resultset result = interceptor.preProcess(null, null);
-        assertThat(result).isNull();
-        assertThat(interceptor.executeTopLevelOnly()).isTrue();
-        interceptor.destroy();
+    void logSlowQuery() {
+        QueryDiagnostic diagnostic = (QueryDiagnostic) statement;
+        when(diagnostic.sql()).thenReturn("sql");
+        when(diagnostic.noIndexUsed()).thenReturn(Boolean.FALSE);
+        when(diagnostic.noGoodIndexUsed()).thenReturn(Boolean.TRUE);
+
+        operation.logSlowQuery(statement);
     }
 
     @Test
-    void postProcess() {
-        when(session.noGoodIndexUsed()).thenReturn(Boolean.TRUE);
+    void suppressSlowSQLWarning() {
+        QueryDiagnostic diagnostic = (QueryDiagnostic) statement;
+        when(diagnostic.sql()).thenReturn("sql");
+        when(diagnostic.noIndexUsed()).thenReturn(Boolean.TRUE);
+        when(diagnostic.noGoodIndexUsed()).thenReturn(Boolean.FALSE);
 
         Database.suppressSlowSQLWarning(true);
-        interceptor.postProcess(() -> "sql", null, null, session);
+        operation.logSlowQuery(statement);
         Database.suppressSlowSQLWarning(false);
         assertThat(actionLog.errorCode()).isNull();
 
-        interceptor.postProcess(() -> "sql", null, null, session);
+        operation.logSlowQuery(statement);
         assertThat(actionLog.errorCode()).isEqualTo("SLOW_SQL");
     }
 }

docker/jre/Dockerfile

@@ -1,4 +1,4 @@
-FROM        eclipse-temurin:21_35-jdk-alpine as build
+FROM        eclipse-temurin:21.0.1_12-jdk-alpine as build
 RUN         jlink --add-modules java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management.rmi,java.naming,java.security.jgss,java.security.sasl,java.sql,java.xml \
                   --add-modules jdk.charsets,jdk.crypto.cryptoki,jdk.jdi,jdk.localedata,jdk.management.jfr,jdk.naming.dns,jdk.naming.rmi,jdk.net,jdk.unsupported \
                   --output /opt/jre --strip-java-debug-attributes --no-man-pages --no-header-files

ext/log-collector/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM        neowu/jre:21
+FROM        neowu/jre:21.0.1
 LABEL       app=log-collector
 RUN         addgroup --gid 1000 --system app && adduser --system --no-create-home --ingroup app --uid 1000 app
 USER        1000

ext/log-exporter/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM        neowu/jre:21
+FROM        neowu/jre:21.0.1
 LABEL       app=log-exporter
 RUN         apk add --no-cache python3 curl
 # gcloud iam requires user home, so must not put --no-create-home

ext/log-processor/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM        neowu/jre:21
+FROM        neowu/jre:21.0.1
 LABEL       app=log-processor
 RUN         addgroup --gid 1000 --system app && adduser --system --no-create-home --ingroup app --uid 1000 app
 USER        1000

ext/monitor/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM        neowu/jre:21
+FROM        neowu/jre:21.0.1
 LABEL       app=monitor
 RUN         addgroup --gid 1000 --system app && adduser --system --no-create-home --ingroup app --uid 1000 app
 USER        1000