Merge branch 'main' into test-gha-branch

.github/workflows/extended.yml

@@ -32,44 +32,21 @@ on:
   push:
     branches:
       - main
+  workflow_dispatch:
+
 
-  issue_comment:
-    types: [created]
-
 jobs:
 
-  check_trigger:
-    runs-on: ubuntu-latest
-    outputs:
-      should_run: ${{ github.event_name == 'push' || (github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, 'Run extended tests')) }}
-      # Custom ref is required since issue_comment event points to the default branch rather than pull request branch
-      ref: ${{ github.event_name == 'push' && github.ref ||format('refs/pull/{0}/head', github.event.issue.number) }}
-    steps:
-      - name: Add reaction to comment
-        if: github.event_name == 'issue_comment'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            await github.rest.reactions.createForIssueComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              comment_id: context.payload.comment.id,
-              content: 'rocket'
-            });
-
   # Check crate compiles and base cargo check passes
   linux-build-lib:
     name: linux build test
-    needs: [check_trigger]
-    if: needs.check_trigger.outputs.should_run == 'true'
     runs-on: ubuntu-latest
     # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
-          ref: ${{ needs.check_trigger.outputs.ref }}
       - name: Install Rust
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
@@ -85,16 +62,14 @@ jobs:
   # Run extended tests (with feature 'extended_tests')
   linux-test-extended:
     name: cargo test 'extended_tests' (amd64)
-    needs: [linux-build-lib, check_trigger]
-    if: needs.check_trigger.outputs.should_run == 'true'
+    needs: [linux-build-lib]
     runs-on: ubuntu-latest
     # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
-          ref: ${{ needs.check_trigger.outputs.ref }}
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
       - name: Install Rust
@@ -120,8 +95,6 @@ jobs:
   # Check answers are correct when hash values collide
   hash-collisions:
     name: cargo test hash collisions (amd64)
-    needs: [check_trigger]
-    if: needs.check_trigger.outputs.should_run == 'true'
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
@@ -130,7 +103,6 @@ jobs:
         with:
           submodules: true
           fetch-depth: 1
-          ref: ${{ needs.check_trigger.outputs.ref }}
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -143,8 +115,6 @@ jobs:
 
   sqllogictest-sqlite:
     name: "Run sqllogictests with the sqlite test suite"
-    needs: [check_trigger]
-    if: needs.check_trigger.outputs.should_run == 'true'
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
@@ -153,7 +123,6 @@ jobs:
         with:
           submodules: true
           fetch-depth: 1
-          ref: ${{ needs.check_trigger.outputs.ref }}
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:

.github/workflows/pr_comment_commands.yml

@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PR commands
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: read
+  pull-requests: write
+  actions: write
+
+jobs:
+
+  run_extended_tests:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, 'Run extended tests') }}
+    outputs:
+      # Custom ref is required since issue_comment event points to the default branch rather than pull request branch
+      ref: ${{ format('refs/pull/{0}/head', github.event.issue.number) }}
+    steps:
+
+      - name: Dispatch extended tests for a PR branch with comment
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'extended.yml',
+              ref: 'refs/pull/${context.payload.issue.number}/head',
+            });
+
+      - name: Add reaction to comment 
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.reactions.createForIssueComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: context.payload.comment.id,
+              content: 'rocket'
+            });

.github/workflows/rust.yml

@@ -295,7 +295,7 @@ jobs:
           export RUST_MIN_STACK=20971520
           export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`
           cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1
-          INCLUDE_TPCH=true cargo test --profile ci --package datafusion-sqllogictest --test sqllogictests
+          INCLUDE_TPCH=true cargo test --features backtrace --profile ci --package datafusion-sqllogictest --test sqllogictests
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -331,7 +331,7 @@ jobs:
       - name: Run sqllogictest
         run: |
           cd datafusion/sqllogictest
-          PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --profile ci --features=postgres --test sqllogictests
+          PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests
         env:
           # use postgres for the host here because we have specified a container for the job
           POSTGRES_HOST: postgres

benchmarks/README.md

@@ -329,7 +329,37 @@ Your benchmark should create and use an instance of `BenchmarkRun` defined in `b
 
 # Benchmarks
 
-The output of `dfbench` help includes a description of each benchmark, which is reproduced here for convenience
+The output of `dfbench` help includes a description of each benchmark, which is reproduced here for convenience.
+
+## Cancellation
+
+Test performance of cancelling queries
+Queries in DataFusion should stop executing "quickly" after they are
+cancelled (the output stream is dropped).
+
+The queries are executed on a synthetic dataset generated during
+the benchmark execution that is an anonymized version of a
+real-world data set.
+
+The query is an anonymized version of a real-world query, and the
+test starts the query then cancels it and reports how long it takes
+for the runtime to fully exit.
+
+Example output:
+
+```
+Using 7 files found on disk
+Starting to load data into in-memory object store
+Done loading data into in-memory object store
+in main, sleeping
+Starting spawned
+Creating logical plan...
+Creating physical plan...
+Executing physical plan...
+Getting results...
+cancelling thread
+done dropping runtime in 83.531417ms
+```
 
 ## ClickBench