Split out avro, parquet, json and csv into individual crates

Cargo.toml

@@ -16,12 +16,16 @@
 # under the License.
 
 [workspace]
-exclude = ["dev/depcheck"]
 members = [
     "datafusion/common",
     "datafusion/common-runtime",
     "datafusion/catalog",
     "datafusion/catalog-listing",
+    "datafusion/datasource",
+    "datafusion/datasource-avro",
+    "datafusion/datasource-csv",
+    "datafusion/datasource-json",
+    "datafusion/datasource-parquet",
     "datafusion/core",
     "datafusion/expr",
     "datafusion/expr-common",
@@ -57,6 +61,7 @@ members = [
     "datafusion/macros",
     "datafusion/doc",
 ]
+exclude = ["dev/depcheck"]
 resolver = "2"
 
 [workspace.package]
@@ -80,6 +85,7 @@ version = "46.0.0"
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
+apache-avro = { version = "0.17", default-features = false }
 arrow = { version = "54.2.1", features = [
     "prettyprint",
     "chrono-tz",
@@ -106,6 +112,10 @@ datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "4
 datafusion-common = { path = "datafusion/common", version = "46.0.0", default-features = false }
 datafusion-common-runtime = { path = "datafusion/common-runtime", version = "46.0.0" }
 datafusion-datasource = { path = "datafusion/datasource", version = "46.0.0", default-features = false }
+datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "46.0.0", default-features = false }
+datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "46.0.0", default-features = false }
+datafusion-datasource-json = { path = "datafusion/datasource-json", version = "46.0.0", default-features = false }
+datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "46.0.0", default-features = false }
 datafusion-doc = { path = "datafusion/doc", version = "46.0.0" }
 datafusion-execution = { path = "datafusion/execution", version = "46.0.0" }
 datafusion-expr = { path = "datafusion/expr", version = "46.0.0" }

datafusion-examples/Cargo.toml

@@ -61,7 +61,7 @@ async-trait = { workspace = true }
 bytes = { workspace = true }
 dashmap = { workspace = true }
 # note only use main datafusion crate for examples
-datafusion = { workspace = true, default-features = true, features = ["avro"] }
+datafusion = { workspace = true, default-features = true }
 datafusion-proto = { workspace = true }
 env_logger = { workspace = true }
 futures = { workspace = true }

datafusion-examples/examples/advanced_parquet_index.rs

@@ -23,16 +23,14 @@ use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
-use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
-use arrow::datatypes::SchemaRef;
 use datafusion::catalog::Session;
 use datafusion::common::{
     internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,
 };
 use datafusion::datasource::listing::PartitionedFile;
 use datafusion::datasource::physical_plan::parquet::ParquetAccessPlan;
 use datafusion::datasource::physical_plan::{
-    parquet::ParquetFileReaderFactory, FileMeta, FileScanConfig, ParquetSource,
+    FileMeta, FileScanConfig, ParquetFileReaderFactory, ParquetSource,
 };
 use datafusion::datasource::TableProvider;
 use datafusion::execution::object_store::ObjectStoreUrl;
@@ -53,6 +51,8 @@ use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::*;
 
+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
+use arrow::datatypes::SchemaRef;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures::future::BoxFuture;

datafusion-examples/examples/csv_json_opener.rs

@@ -18,15 +18,15 @@
 use std::sync::Arc;
 
 use arrow::datatypes::{DataType, Field, Schema};
-use datafusion::datasource::physical_plan::JsonSource;
 use datafusion::{
     assert_batches_eq,
-    datasource::physical_plan::FileSource,
     datasource::{
         file_format::file_compression_type::FileCompressionType,
         listing::PartitionedFile,
         object_store::ObjectStoreUrl,
-        physical_plan::{CsvSource, FileScanConfig, FileStream, JsonOpener},
+        physical_plan::{
+            CsvSource, FileScanConfig, FileSource, FileStream, JsonOpener, JsonSource,
+        },
     },
     error::Result,
     physical_plan::metrics::ExecutionPlanMetricsSet,

datafusion/catalog-listing/src/helpers.rs

@@ -532,6 +532,7 @@ pub fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {
 #[cfg(test)]
 mod tests {
     use async_trait::async_trait;
+    use datafusion_common::config::TableOptions;
     use datafusion_execution::config::SessionConfig;
     use datafusion_execution::runtime_env::RuntimeEnv;
     use futures::FutureExt;
@@ -1068,5 +1069,13 @@ mod tests {
         fn as_any(&self) -> &dyn Any {
             unimplemented!()
         }
+
+        fn table_options(&self) -> &TableOptions {
+            unimplemented!()
+        }
+
+        fn table_options_mut(&mut self) -> &mut TableOptions {
+            unimplemented!()
+        }
     }
 }

datafusion/catalog/src/session.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use async_trait::async_trait;
-use datafusion_common::config::ConfigOptions;
+use datafusion_common::config::{ConfigOptions, TableOptions};
 use datafusion_common::{DFSchema, Result};
 use datafusion_execution::config::SessionConfig;
 use datafusion_execution::runtime_env::RuntimeEnv;
@@ -120,6 +120,18 @@ pub trait Session: Send + Sync {
     fn execution_props(&self) -> &ExecutionProps;
 
     fn as_any(&self) -> &dyn Any;
+
+    /// Return the table options
+    fn table_options(&self) -> &TableOptions;
+
+    /// return the TableOptions options with its extensions
+    fn default_table_options(&self) -> TableOptions {
+        self.table_options()
+            .combine_with_session_config(self.config_options())
+    }
+
+    /// Returns a mutable reference to [`TableOptions`]
+    fn table_options_mut(&mut self) -> &mut TableOptions;
 }
 
 /// Create a new task context instance from Session

datafusion/core/Cargo.toml

@@ -40,9 +40,15 @@ nested_expressions = ["datafusion-functions-nested"]
 # This feature is deprecated. Use the `nested_expressions` feature instead.
 array_expressions = ["nested_expressions"]
 # Used to enable the avro format
-avro = ["apache-avro", "num-traits", "datafusion-common/avro", "datafusion-datasource/avro"]
+avro = ["datafusion-common/avro", "datafusion-datasource-avro"]
 backtrace = ["datafusion-common/backtrace"]
-compression = ["xz2", "bzip2", "flate2", "zstd", "datafusion-datasource/compression"]
+compression = [
+    "xz2",
+    "bzip2",
+    "flate2",
+    "zstd",
+    "datafusion-datasource/compression",
+]
 crypto_expressions = ["datafusion-functions/crypto_expressions"]
 datetime_expressions = ["datafusion-functions/datetime_expressions"]
 default = [
@@ -61,7 +67,7 @@ encoding_expressions = ["datafusion-functions/encoding_expressions"]
 # Used for testing ONLY: causes all values to hash to the same value (test for collisions)
 force_hash_collisions = ["datafusion-physical-plan/force_hash_collisions", "datafusion-common/force_hash_collisions"]
 math_expressions = ["datafusion-functions/math_expressions"]
-parquet = ["datafusion-common/parquet", "dep:parquet"]
+parquet = ["datafusion-common/parquet", "dep:parquet", "datafusion-datasource-parquet"]
 pyarrow = ["datafusion-common/pyarrow", "parquet"]
 regex_expressions = [
     "datafusion-functions/regex_expressions",
@@ -82,7 +88,6 @@ unicode_expressions = [
 extended_tests = []
 
 [dependencies]
-apache-avro = { version = "0.17", optional = true }
 arrow = { workspace = true }
 arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
@@ -95,6 +100,10 @@ datafusion-catalog-listing = { workspace = true }
 datafusion-common = { workspace = true, features = ["object_store"] }
 datafusion-common-runtime = { workspace = true }
 datafusion-datasource = { workspace = true }
+datafusion-datasource-avro = { workspace = true, optional = true }
+datafusion-datasource-csv = { workspace = true }
+datafusion-datasource-json = { workspace = true }
+datafusion-datasource-parquet = { workspace = true, optional = true }
 datafusion-execution = { workspace = true }
 datafusion-expr = { workspace = true }
 datafusion-expr-common = { workspace = true }
@@ -114,7 +123,6 @@ flate2 = { version = "1.1.0", optional = true }
 futures = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
-num-traits = { version = "0.2", optional = true }
 object_store = { workspace = true }
 parking_lot = { workspace = true }
 parquet = { workspace = true, optional = true, default-features = true }