Merge branch 'main' into with_column_updates

.github/dependabot.yml

@@ -21,14 +21,24 @@ updates:
     directory: "/"
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
     target-branch: main
     labels: [auto-dependencies]
     ignore:
-      # arrow is bumped manually
+      # major version bumps of arrow* and parquet are handled manually
       - dependency-name: "arrow*"
         update-types: ["version-update:semver-major"]
+      - dependency-name: "parquet"
+        update-types: ["version-update:semver-major"]
     groups:
+      # minor and patch bumps of arrow* and parquet are grouped
+      arrow-parquet:
+        applies-to: version-updates
+        patterns:
+          - "arrow*"
+          - "parquet"
+        update-types:
+          - "minor"
+          - "patch"
       proto:
         applies-to: version-updates
         patterns:

.github/workflows/extended.yml

@@ -23,59 +23,67 @@ concurrency:
 
 # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
 #
-# These jobs only run on the `main` branch as they are time consuming
+# These jobs are not run as part of PR checks as they are time-consuming
 # and should not fail often.
 #
 # We still run them as they provide important coverage to ensure correctness
 # in the (very rare) event of a hash failure or sqlite library query failure.
 on:
-  # Run on all commits to main
   push:
-    branches:
-      - main
 
 jobs:
   # Check crate compiles and base cargo check passes
   linux-build-lib:
     name: linux build test
     runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
         with:
-          rust-version: stable
+          submodules: true
+          fetch-depth: 1
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          source $HOME/.cargo/env
+          rustup default stable
+      - name: Install Protobuf Compiler
+        run: sudo apt-get install -y protobuf-compiler
       - name: Prepare cargo build
         run: |
           cargo check --profile ci --all-targets
           cargo clean
 
-#  # Run extended tests (with feature 'extended_tests')
-#  # Disabling as it is running out of disk space
-#  # see https://github.com/apache/datafusion/issues/14576
-#  linux-test-extended:
-#    name: cargo test 'extended_tests' (amd64)
-#    needs: linux-build-lib
-#    runs-on: ubuntu-latest
-#    container:
-#      image: amd64/rust
-#    steps:
-#      - uses: actions/checkout@v4
-#        with:
-#          submodules: true
-#          fetch-depth: 1
-#      - name: Setup Rust toolchain
-#        uses: ./.github/actions/setup-builder
-#        with:
-#          rust-version: stable
-#      - name: Run tests (excluding doctests)
-#        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests
-#      - name: Verify Working Directory Clean
-#        run: git diff --exit-code
-#      - name: Cleanup
-#        run: cargo clean
+  # Run extended tests (with feature 'extended_tests')
+  linux-test-extended:
+    name: cargo test 'extended_tests' (amd64)
+    needs: linux-build-lib
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          source $HOME/.cargo/env
+          rustup default stable
+      - name: Install Protobuf Compiler
+        run: sudo apt-get install -y protobuf-compiler
+      # For debugging, test binaries can be large.
+      - name: Show available disk space
+        run: |
+          df -h
+      - name: Run tests (excluding doctests)
+        env:
+          RUST_BACKTRACE: 1
+        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests
+      - name: Verify Working Directory Clean
+        run: git diff --exit-code
+      - name: Cleanup
+        run: cargo clean
 
   # Check answers are correct when hash values collide
   hash-collisions:
@@ -95,7 +103,7 @@ jobs:
       - name: Run tests
         run: |
           cd datafusion
-          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --features=force_hash_collisions,avro,extended_tests
+          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --features=force_hash_collisions,avro
           cargo clean
 
   sqllogictest-sqlite:

.github/workflows/rust.yml

@@ -389,21 +389,20 @@ jobs:
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)
     needs: linux-build-lib
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     container:
-      image: amd64/rust:bullseye # Workaround https://github.com/actions/setup-python/issues/721
+      image: amd64/rust:bullseye # Use the bullseye tag image which comes with python3.9
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.8"
       - name: Install PyArrow
         run: |
           echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
-          python -m pip install pyarrow
+          apt-get update
+          apt-get install python3-pip -y
+          python3 -m pip install pyarrow
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:

.gitignore

@@ -71,3 +71,6 @@ datafusion/core/benches/data/*
 # rat
 filtered_rat.txt
 rat.txt
+
+# data generated by examples
+datafusion-examples/examples/datafusion-examples/