diff --git a/datafusion/core/src/physical_plan/file_format/file_stream.rs b/datafusion/core/src/physical_plan/file_format/file_stream.rs index 1a54d685228c..609e3b3a9b20 100644 --- a/datafusion/core/src/physical_plan/file_format/file_stream.rs +++ b/datafusion/core/src/physical_plan/file_format/file_stream.rs @@ -51,7 +51,11 @@ use crate::physical_plan::RecordBatchStream; pub type FileOpenFuture = BoxFuture<'static, Result>>>; +/// Generic API for opening a file using an [`ObjectStore`] and resolving to a +/// stream of [`RecordBatch`] pub trait FileOpener: Unpin { + /// Asynchronously open the specified file and return a stream + /// of [`RecordBatch`] fn open( &self, store: Arc, @@ -167,6 +171,7 @@ impl FileStreamMetrics { } impl FileStream { + /// Create a new `FileStream` using the give `FileOpener` to scan underlying files pub fn new( config: &FileScanConfig, partition: usize, diff --git a/datafusion/core/src/physical_plan/file_format/mod.rs b/datafusion/core/src/physical_plan/file_format/mod.rs index b2fb5b0275df..5a7406dbb634 100644 --- a/datafusion/core/src/physical_plan/file_format/mod.rs +++ b/datafusion/core/src/physical_plan/file_format/mod.rs @@ -41,6 +41,7 @@ use arrow::{ record_batch::RecordBatch, }; pub use avro::AvroExec; +pub use file_stream::{FileOpenFuture, FileOpener, FileStream}; pub(crate) use json::plan_to_json; pub use json::NdJsonExec;