Merge branch 'main' into feature/metadata_columns

apache · Feb 20, 2025 · ea14a2f · ea14a2f
2 parents 7126d97 + 2f40f6c
commit ea14a2f
Show file tree

Hide file tree

Showing 675 changed files with 24,581 additions and 12,714 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -28,24 +28,15 @@ updates:
       # arrow is bumped manually
       - dependency-name: "arrow*"
         update-types: ["version-update:semver-major"]
-  - package-ecosystem: cargo
-    directory: "datafusion-cli/"
-    schedule:
-      interval: daily
-    open-pull-requests-limit: 10
-    target-branch: main
-    labels: [auto-dependencies]
-    ignore:
-      # arrow is bumped manually
-      - dependency-name: "arrow*"
-        update-types: ["version-update:semver-major"]
-      # datafusion is bumped manually
-      - dependency-name: "datafusion*"
-        update-types: ["version-update:semver-major"]
+    groups:
+      proto:
+        applies-to: version-updates
+        patterns:
+          - "prost*"
+          - "pbjson*"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
-
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -4,7 +4,7 @@
 We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
 -->
 
-Closes #.
+- Closes #.
 
 ## Rationale for this change
 

diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
@@ -25,9 +25,11 @@ on:
   push:
     paths:
       - "**/Cargo.toml"
+      - "**/Cargo.lock"
   pull_request:
     paths:
       - "**/Cargo.toml"
+      - "**/Cargo.lock"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
@@ -50,4 +52,4 @@ jobs:
       - name: Check dependencies
         run: |
           cd dev/depcheck
-          cargo run
+          cargo run
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml
@@ -23,9 +23,11 @@ concurrency:
 
 # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
 #
-# this job is intended to only run only on the main branch as it is time consuming
-# and should not fail often. However, it is important coverage to ensure correctness
-# in the (very rare) event of a hash failure or sqlite query failure.
+# These jobs only run on the `main` branch as they are time consuming
+# and should not fail often.
+#
+# We still run them as they provide important coverage to ensure correctness
+# in the (very rare) event of a hash failure or sqlite library query failure.
 on:
   # Run on all commits to main
   push:
@@ -46,28 +48,34 @@ jobs:
         with:
           rust-version: stable
       - name: Prepare cargo build
-        run: cargo check --profile ci --all-targets
+        run: |
+          cargo check --profile ci --all-targets
+          cargo clean
 
-  # Run extended tests (with feature 'extended_tests')
-  linux-test-extended:
-    name: cargo test 'extended_tests' (amd64)
-    needs: linux-build-lib
-    runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
-      - name: Run tests (excluding doctests)
-        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests
-      - name: Verify Working Directory Clean
-        run: git diff --exit-code
+#  # Run extended tests (with feature 'extended_tests')
+#  # Disabling as it is running out of disk space
+#  # see https://github.com/apache/datafusion/issues/14576
+#  linux-test-extended:
+#    name: cargo test 'extended_tests' (amd64)
+#    needs: linux-build-lib
+#    runs-on: ubuntu-latest
+#    container:
+#      image: amd64/rust
+#    steps:
+#      - uses: actions/checkout@v4
+#        with:
+#          submodules: true
+#          fetch-depth: 1
+#      - name: Setup Rust toolchain
+#        uses: ./.github/actions/setup-builder
+#        with:
+#          rust-version: stable
+#      - name: Run tests (excluding doctests)
+#        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests
+#      - name: Verify Working Directory Clean
+#        run: git diff --exit-code
+#      - name: Cleanup
+#        run: cargo clean
 
   # Check answers are correct when hash values collide
   hash-collisions:
@@ -88,6 +96,7 @@ jobs:
         run: |
           cd datafusion
           cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --features=force_hash_collisions,avro,extended_tests
+          cargo clean
 
   sqllogictest-sqlite:
     name: "Run sqllogictests with the sqlite test suite"
@@ -104,4 +113,8 @@ jobs:
         with:
           rust-version: stable
       - name: Run sqllogictest
-        run: cargo test --profile release-nonlto --test sqllogictests -- --include-sqlite
+        run: |
+          cargo test --profile release-nonlto --test sqllogictests -- --include-sqlite
+          cargo clean
+
+
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -41,7 +41,7 @@ on:
 jobs:
   # Check license header
   license-header-check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: Check License Header
     steps:
       - uses: actions/checkout@v4
@@ -60,7 +60,11 @@ jobs:
         with:
           rust-version: stable
       - name: Prepare cargo build
-        run: cargo check --profile ci --all-targets --features integration-tests
+        run: |
+          # Adding `--locked` here to assert that the `Cargo.lock` file is up to
+          # date with the manifest. When this fails, please make sure to commit
+          # the changes to `Cargo.lock` after building with the updated manifest.
+          cargo check --profile ci --workspace --all-targets --features integration-tests --locked
 
   # cargo check common, functions and substrait with no default features
   linux-cargo-check-no-default-features:
@@ -95,12 +99,6 @@ jobs:
       - name: Check workspace with additional features
         run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests
 
-      - name: Check Cargo.lock for datafusion-cli
-        run: |
-          # If this test fails, try running `cargo update` in the `datafusion-cli` directory
-          # and check in the updated Cargo.lock file.
-          cargo check --profile ci --manifest-path datafusion-cli/Cargo.toml --locked
-
   # cargo check datafusion to ensure that the datafusion crate can be built with only a
   # subset of the function packages enabled.
   linux-cargo-check-datafusion:
@@ -189,28 +187,6 @@ jobs:
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
-  linux-test-datafusion-cli:
-    name: cargo test datafusion-cli (amd64)
-    needs: linux-build-lib
-    runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
-      - name: Run tests (excluding doctests)
-        run: |
-          cd datafusion-cli
-          cargo test --profile ci --lib --tests --bins --all-features
-      - name: Verify Working Directory Clean
-        run: git diff --exit-code
-
   linux-test-example:
     name: cargo examples (amd64)
     needs: linux-build-lib
@@ -252,10 +228,7 @@ jobs:
         with:
           rust-version: stable
       - name: Run doctests
-        run: |
-          cargo test --profile ci --doc --features avro,json
-          cd datafusion-cli
-          cargo test --profile ci --doc --all-features
+        run: cargo test --profile ci --doc --features avro,json
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -364,45 +337,40 @@ jobs:
           POSTGRES_HOST: postgres
           POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
 
-#  Temporarily commenting out the Windows flow, the reason is enormously slow running build
-#  Waiting for new Windows 2025 github runner
-#  Details: https://github.com/apache/datafusion/issues/13726
-#
-#  windows:
-#    name: cargo test (win64)
-#    runs-on: windows-latest
-#    steps:
-#      - uses: actions/checkout@v4
-#        with:
-#          submodules: true
-#      - name: Setup Rust toolchain
-#        uses: ./.github/actions/setup-windows-builder
-#      - name: Run tests (excluding doctests)
-#        shell: bash
-#        run: |
-#          export PATH=$PATH:$HOME/d/protoc/bin
-#          cargo test --lib --tests --bins --features avro,json,backtrace
-#          cd datafusion-cli
-#          cargo test --lib --tests --bins --all-features
-
-# Commenting out intel mac build as so few users would ever use it
-# Details: https://github.com/apache/datafusion/issues/13846
-#  macos:
-#    name: cargo test (macos)
-#    runs-on: macos-latest
-#    steps:
-#      - uses: actions/checkout@v4
-#        with:
-#          submodules: true
-#          fetch-depth: 1
-#      - name: Setup Rust toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#      - name: Run tests (excluding doctests)
-#        shell: bash
-#        run: |
-#          cargo test run --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace
-#          cd datafusion-cli
-#          cargo test run --profile ci --lib --tests --bins --all-features
+  #  Temporarily commenting out the Windows flow, the reason is enormously slow running build
+  #  Waiting for new Windows 2025 github runner
+  #  Details: https://github.com/apache/datafusion/issues/13726
+  #
+  #  windows:
+  #    name: cargo test (win64)
+  #    runs-on: windows-latest
+  #    steps:
+  #      - uses: actions/checkout@v4
+  #        with:
+  #          submodules: true
+  #      - name: Setup Rust toolchain
+  #        uses: ./.github/actions/setup-windows-builder
+  #      - name: Run tests (excluding doctests)
+  #        shell: bash
+  #        run: |
+  #          export PATH=$PATH:$HOME/d/protoc/bin
+  #          cargo test --lib --tests --bins --features avro,json,backtrace
+
+  # Commenting out intel mac build as so few users would ever use it
+  # Details: https://github.com/apache/datafusion/issues/13846
+  #  macos:
+  #    name: cargo test (macos)
+  #    runs-on: macos-latest
+  #    steps:
+  #      - uses: actions/checkout@v4
+  #        with:
+  #          submodules: true
+  #          fetch-depth: 1
+  #      - name: Setup Rust toolchain
+  #        uses: ./.github/actions/setup-macos-builder
+  #      - name: Run tests (excluding doctests)
+  #        shell: bash
+  #        run: cargo test run --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace
 
   macos-aarch64:
     name: cargo test (macos-aarch64)
@@ -416,10 +384,7 @@ jobs:
         uses: ./.github/actions/setup-macos-aarch64-builder
       - name: Run tests (excluding doctests)
         shell: bash
-        run: |
-          cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests
-          cd datafusion-cli
-          cargo test --profile ci --lib --tests --bins --all-features
+        run: cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests
 
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)
@@ -615,19 +580,19 @@ jobs:
           # (Min Supported Rust Version) than the one specified in the
           # `rust-version` key of `Cargo.toml`.
           #
-          # To reproduce: 
-          # 1. Install the version of Rust that is failing. Example: 
+          # To reproduce:
+          # 1. Install the version of Rust that is failing. Example:
           #    rustup install 1.80.1
           # 2. Run the command that failed with that version. Example:
           #    cargo +1.80.1 check -p datafusion
-          # 
+          #
           # To resolve, either:
-          # 1. Change your code to use older Rust features, 
+          # 1. Change your code to use older Rust features,
           # 2. Revert dependency update
           # 3. Update the MSRV version in `Cargo.toml`
           #
           # Please see the DataFusion Rust Version Compatibility Policy before
-          # updating Cargo.toml. You may have to update the code instead. 
+          # updating Cargo.toml. You may have to update the code instead.
           # https://github.com/apache/datafusion/blob/main/README.md#rust-version-compatibility-policy
           cargo msrv --output-format json --log-target stdout verify
       - name: Check datafusion-substrait
@@ -636,6 +601,3 @@ jobs:
       - name: Check datafusion-proto
         working-directory: datafusion/proto
         run: cargo msrv --output-format json --log-target stdout verify
-      - name: Check datafusion-cli
-        working-directory: datafusion-cli
-        run: cargo msrv --output-format json --log-target stdout verify
diff --git a/.gitignore b/.gitignore
@@ -42,8 +42,6 @@ venv/*
 
 # Rust
 target
-Cargo.lock
-!datafusion-cli/Cargo.lock
 
 rusty-tags.vi
 .history