Merge remote-tracking branch 'upstream/main' into issue_14953

.github/workflows/docs.yaml

@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 on:
   push:
     branches:

.github/workflows/docs_pr.yaml

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Tests for Docs that runs on PRs
 name: Docs
 
 concurrency:
@@ -48,7 +49,34 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      - name: Run doctests
+      - name: Run doctests (embedded rust examples)
         run: cargo test --doc --features avro,json
       - name: Verify Working Directory Clean
         run: git diff --exit-code
+
+  # Test doc build
+  linux-test-doc-build:
+    name: Test doc build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install doc dependencies
+        run: |
+          set -x
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r docs/requirements.txt
+      - name: Build docs html and check for warnings
+        run: |
+          set -x
+          source venv/bin/activate
+          cd docs
+          ./build.sh # fails on errors
+

.github/workflows/extended.yml

@@ -36,6 +36,7 @@ jobs:
   linux-build-lib:
     name: linux build test
     runs-on: ubuntu-latest
+    # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: actions/checkout@v4
         with:
@@ -45,7 +46,7 @@ jobs:
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
           source $HOME/.cargo/env
-          rustup default stable
+          rustup toolchain install
       - name: Install Protobuf Compiler
         run: sudo apt-get install -y protobuf-compiler
       - name: Prepare cargo build
@@ -58,6 +59,7 @@ jobs:
     name: cargo test 'extended_tests' (amd64)
     needs: linux-build-lib
     runs-on: ubuntu-latest
+    # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: actions/checkout@v4
         with:
@@ -69,7 +71,7 @@ jobs:
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
           source $HOME/.cargo/env
-          rustup default stable
+          rustup toolchain install
       - name: Install Protobuf Compiler
         run: sudo apt-get install -y protobuf-compiler
       # For debugging, test binaries can be large.

Cargo.toml

@@ -16,12 +16,16 @@
 # under the License.
 
 [workspace]
-exclude = ["dev/depcheck"]
 members = [
     "datafusion/common",
     "datafusion/common-runtime",
     "datafusion/catalog",
     "datafusion/catalog-listing",
+    "datafusion/datasource",
+    "datafusion/datasource-avro",
+    "datafusion/datasource-csv",
+    "datafusion/datasource-json",
+    "datafusion/datasource-parquet",
     "datafusion/core",
     "datafusion/expr",
     "datafusion/expr-common",
@@ -57,6 +61,7 @@ members = [
     "datafusion/macros",
     "datafusion/doc",
 ]
+exclude = ["dev/depcheck"]
 resolver = "2"
 
 [workspace.package]
@@ -80,6 +85,7 @@ version = "46.0.0"
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
+apache-avro = { version = "0.17", default-features = false }
 arrow = { version = "54.2.1", features = [
     "prettyprint",
     "chrono-tz",
@@ -106,6 +112,10 @@ datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "4
 datafusion-common = { path = "datafusion/common", version = "46.0.0", default-features = false }
 datafusion-common-runtime = { path = "datafusion/common-runtime", version = "46.0.0" }
 datafusion-datasource = { path = "datafusion/datasource", version = "46.0.0", default-features = false }
+datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "46.0.0", default-features = false }
+datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "46.0.0", default-features = false }
+datafusion-datasource-json = { path = "datafusion/datasource-json", version = "46.0.0", default-features = false }
+datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "46.0.0", default-features = false }
 datafusion-doc = { path = "datafusion/doc", version = "46.0.0" }
 datafusion-execution = { path = "datafusion/execution", version = "46.0.0" }
 datafusion-expr = { path = "datafusion/expr", version = "46.0.0" }

datafusion-cli/Cargo.toml

@@ -30,6 +30,10 @@ rust-version = { workspace = true }
 [package.metadata.docs.rs]
 all-features = true
 
+[features]
+default = []
+backtrace = ["datafusion/backtrace"]
+
 [dependencies]
 arrow = { workspace = true }
 async-trait = { workspace = true }

datafusion-examples/Cargo.toml

@@ -61,7 +61,7 @@ async-trait = { workspace = true }
 bytes = { workspace = true }
 dashmap = { workspace = true }
 # note only use main datafusion crate for examples
-datafusion = { workspace = true, default-features = true, features = ["avro"] }
+datafusion = { workspace = true, default-features = true }
 datafusion-proto = { workspace = true }
 env_logger = { workspace = true }
 futures = { workspace = true }

datafusion-examples/examples/advanced_parquet_index.rs

@@ -23,16 +23,14 @@ use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
-use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
-use arrow::datatypes::SchemaRef;
 use datafusion::catalog::Session;
 use datafusion::common::{
     internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,
 };
 use datafusion::datasource::listing::PartitionedFile;
 use datafusion::datasource::physical_plan::parquet::ParquetAccessPlan;
 use datafusion::datasource::physical_plan::{
-    parquet::ParquetFileReaderFactory, FileMeta, FileScanConfig, ParquetSource,
+    FileMeta, FileScanConfig, ParquetFileReaderFactory, ParquetSource,
 };
 use datafusion::datasource::TableProvider;
 use datafusion::execution::object_store::ObjectStoreUrl;
@@ -53,6 +51,8 @@ use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::*;
 
+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
+use arrow::datatypes::SchemaRef;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures::future::BoxFuture;

datafusion-examples/examples/csv_json_opener.rs

@@ -18,15 +18,15 @@
 use std::sync::Arc;
 
 use arrow::datatypes::{DataType, Field, Schema};
-use datafusion::datasource::physical_plan::JsonSource;
 use datafusion::{
     assert_batches_eq,
-    datasource::physical_plan::FileSource,
     datasource::{
         file_format::file_compression_type::FileCompressionType,
         listing::PartitionedFile,
         object_store::ObjectStoreUrl,
-        physical_plan::{CsvSource, FileScanConfig, FileStream, JsonOpener},
+        physical_plan::{
+            CsvSource, FileScanConfig, FileSource, FileStream, JsonOpener, JsonSource,
+        },
     },
     error::Result,
     physical_plan::metrics::ExecutionPlanMetricsSet,