Merge branch 'main' into nightly/sm-sort-order

.github/workflows/nightly-check-ci.yml

@@ -7,7 +7,7 @@ on:
     # 2AM EST == 6AM UTC
     - cron:  '0 6 * * *'
   push:
-    branches: [ 'nightly/**', 'release/v*', 'dependabot/**' ]
+    branches: [ 'nightly/**', 'release/v*', 'dependabot/**', 'coverage/**']
 
 jobs:
   nightly:
@@ -21,6 +21,8 @@ jobs:
     concurrency:
       group: ${{ matrix.gradle-task }}-${{ matrix.test-jvm-version }}-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: true
+    env:
+      COVER: ${{ (github.event_name == 'schedule' || startsWith(github.ref_name, 'coverage/')) && matrix.test-jvm-version == '21' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -66,8 +68,16 @@ jobs:
           .github/scripts/gradle-properties.sh >> gradle.properties
           cat gradle.properties
 
-      - name: Run gradle ${{ matrix.gradle-task }} on java ${{ matrix.test-jvm-version }}
-        run: ./gradlew --scan --continue --rerun-tasks ${{ matrix.gradle-task }} -PtestRuntimeVersion=${{ matrix.test-jvm-version }}
+      - name: Run gradle ${{ matrix.gradle-task }} on java ${{ matrix.test-jvm-version }} with coverage=${{ env.COVER }}
+        run: ./gradlew --scan --continue --rerun-tasks ${{ matrix.gradle-task }} -PtestRuntimeVersion=${{ matrix.test-jvm-version }} -Pcoverage.enabled=$COVER
+
+      - name: Upload Coverage Results
+        if: ${{ env.COVER == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.gradle-task }}-results
+          path: |
+            **/build/jacoco/*.exec
 
       - name: Upload Test Results
         uses: actions/upload-artifact@v4
@@ -107,7 +117,61 @@ jobs:
           webhook-type: webhook-trigger
           payload: |
             {
-              "slack_message": "Nightly build failure in ${{ matrix.gradle-task }} on Java ${{ matrix.test-jvm-version }} @ ${{ github.head_ref }} ${{ github.sha }}" ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              "slack_message": "Nightly build failure in ${{ matrix.gradle-task }} on Java ${{ matrix.test-jvm-version }} @ ${{ github.head_ref }} ${{ github.sha }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
             }
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_NIGHTLY_FAILURE }}
+
+  combined-coverage-report:
+    if: ${{ github.event_name == 'schedule' || startsWith(github.ref_name, 'coverage/') }}
+    needs: nightly
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Download Coverage Results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*-results
+          merge-multiple: true
+
+      - name: Setup JDK 21
+        id: setup-java-21
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+
+      - name: Setup JDK 11
+        id: setup-java-11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+
+      - name: Set JAVA_HOME
+        run: echo "JAVA_HOME=${{ steps.setup-java-11.outputs.path }}" >> $GITHUB_ENV
+
+      - name: Setup gradle properties
+        run: |
+          .github/scripts/gradle-properties.sh >> gradle.properties
+          cat gradle.properties
+          
+      - name: Run gradle ${{ matrix.gradle-task }}
+        run: |
+          ./gradlew -Pcoverage.enabled=true jacocoTestReport
+          ./gradlew -Pcoverage.enabled=true coverage:coverage-merge
+          
+      - name: Upload Combined Coverage Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: combined-coverage-results
+          path: |
+            coverage/build/reports/jacoco/**
+            coverage/build/reports/coverage/**

.github/workflows/publish-ci.yml

@@ -81,36 +81,86 @@ jobs:
             web/client-api/types/build/*.tgz
 
       - name: Publish deephaven-core to PyPi
+        id: publish-deephaven-core
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/server/build/wheel/
         continue-on-error: true
 
       - name: Publish deephaven-server to PyPi
+        id: publish-deephaven-server
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/embedded-server/build/wheel/
         continue-on-error: true
 
       - name: Publish pydeephaven to PyPi
+        id: publish-pydeephaven
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/client/build/wheel/
         continue-on-error: true
 
       - name: Publish pydeephaven-ticking to PyPi
+        id: publish-pydeephaven-ticking
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/client-ticking/build/wheel/
         continue-on-error: true
 
       - name: Publish @deephaven/jsapi-types to npmjs
+        id: publish-deephaven-jsapi-types
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         env:
           NODE_AUTH_TOKEN: ${{ secrets.DEEPHAVENBOT_NPM_TOKEN }}
         run: npm publish --provenance --tag latest web/client-api/types/build/deephaven-jsapi-types-*.tgz
         continue-on-error: true
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-deephaven-core.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-deephaven-core.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-deephaven-server.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-deephaven-server.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-pydeephaven.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-pydeephaven.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-pydeephaven-ticking.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-pydeephaven-ticking.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-deephaven-jsapi-types.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-deephaven-jsapi-types.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

Integrations/build.gradle

@@ -17,6 +17,7 @@ dependencies {
     implementation project(':log-factory')
     implementation project(":util-thread")
     implementation libs.commons.lang3
+    implementation libs.google.findbugs.jsr305
 
     testImplementation project(':engine-test-utils')
     testImplementation project(path: ':Base', configuration: 'tests')

authentication/build.gradle

@@ -6,7 +6,6 @@ plugins {
 description 'authentication: Deephaven authentication and identity'
 
 dependencies {
-    api project(':proto:proto-backplane-grpc')
     implementation project(':log-factory')
     implementation project(':Configuration')
 

authentication/example-providers/mtls/build.gradle

@@ -5,4 +5,6 @@ plugins {
 
 dependencies {
     shadow project(':grpc-java:grpc-mtls')
+    shadow platform(libs.grpc.bom)
+    shadow libs.grpc.api
 }

authorization-codegen/build.gradle

@@ -14,4 +14,6 @@ dependencies {
     implementation libs.grpc.services
 
     implementation libs.squareup.javapoet
+
+    implementation libs.protobuf.java
 }

buildSrc/src/main/groovy/io.deephaven.java-shadow-publishing-conventions.gradle

@@ -34,14 +34,20 @@ PublishingTools.setupPublications(project) { publication ->
       def dependenciesNode = root.appendNode('dependencies')
 
       project.configurations.shadow.allDependencies.each {
-        if ((it instanceof ProjectDependency) || ! (it instanceof SelfResolvingDependency)) {
+        if ((it instanceof ProjectDependency)) {
           def dependencyNode = dependenciesNode.appendNode('dependency')
           dependencyNode.appendNode('groupId', it.group)
           BasePluginConvention base = it.dependencyProject.convention.getPlugin(BasePluginConvention)
 
           dependencyNode.appendNode('artifactId', base.archivesBaseName)
           dependencyNode.appendNode('version', it.version)
           dependencyNode.appendNode('scope', 'runtime')
+        } else if (! (it instanceof SelfResolvingDependency)) {
+          def dependencyNode = dependenciesNode.appendNode('dependency')
+          dependencyNode.appendNode('groupId', it.group)
+          dependencyNode.appendNode('artifactId', it.name)
+          dependencyNode.appendNode('version', it.version)
+          dependencyNode.appendNode('scope', 'runtime')
         }
       }
     }

coverage/gather-coverage.py

@@ -19,16 +19,17 @@
 os.makedirs(coverage_dir)
 
 # Aggregate and normalize coverage for java projects
+print("Aggregating Java Coverage")
 input_glob = proj_root_dir + '/build/reports/jacoco/jacoco-merge/jacoco-merge.csv'
 with open(f'{coverage_dir}/java-coverage.csv', 'w', newline='') as outfile:
     csv_writer = csv.writer(outfile)
-    csv_writer.writerow(['Language','Project','Package','Class','Missed','Covered'])
+    csv_writer.writerow(['Language','Package','Class','Missed','Covered'])
     for filename in glob.glob(input_glob, recursive = True):
         with open(filename, 'r') as csv_in:
             csv_reader = csv.reader(csv_in)
             next(csv_reader, None)
             for row in csv_reader:
-                new_row = ['java',row[0],row[1],row[2],row[3],row[4]]
+                new_row = ['java',row[1],row[2],row[3],row[4]]
                 csv_writer.writerow(new_row)
 
 # Load packages to be excluded from the aggregated coverage CSV
@@ -39,8 +40,10 @@
 with open(coverage_output_path, 'w', newline='') as outfile:
     csv_writer = csv.writer(outfile)
     for csv_file in glob.glob(coverage_input_glob):
+        if os.path.basename(csv_file) == "all-coverage.csv": continue
+        print('Merging', os.path.basename(csv_file))
         with open(csv_file, 'r') as csv_in:
             for row in csv.reader(csv_in):
-                if row[2] in excludes: continue
-                new_row = [row[0],row[1],row[2],row[3],row[4],row[5]]
+                if row[1] in excludes: continue
+                new_row = [row[0],row[1],row[2],row[3],row[4]]
                 csv_writer.writerow(new_row)

cpp-client/build.gradle

@@ -114,6 +114,7 @@ def testCppClient = Docker.registerDockerTask(project, 'testCppClient') {
         environmentVariable 'DH_HOST', deephavenDocker.containerName.get()
         environmentVariable 'DH_PORT', '10000'
     }
+    waitTimeMinutes = 1
     containerDependencies.dependsOn = [deephavenDocker.healthyTask]
     containerDependencies.finalizedBy = deephavenDocker.endTask
     network = deephavenDocker.networkName.get()

cpp-client/deephaven/dhclient/CMakeLists.txt

@@ -74,6 +74,7 @@ set(ALL_FILES
     include/private/deephaven/client/impl/util.h
 
     src/arrowutil/arrow_client_table.cc
+    src/arrowutil/arrow_column_source.cc
     include/private/deephaven/client/arrowutil/arrow_client_table.h
     include/private/deephaven/client/arrowutil/arrow_column_source.h
     include/private/deephaven/client/arrowutil/arrow_value_converter.h
@@ -107,6 +108,7 @@ set(ALL_FILES
     src/utility/table_maker.cc
 
     include/public/deephaven/client/utility/arrow_util.h
+    include/public/deephaven/client/utility/internal_types.h
     include/public/deephaven/client/utility/misc_types.h
     include/public/deephaven/client/utility/table_maker.h
 )