[POC] feat: Add datafusion-storage

Cargo.toml

@@ -45,6 +45,7 @@ members = [
     "datafusion/proto-common/gen",
     "datafusion/sql",
     "datafusion/sqllogictest",
+    "datafusion/storage",
     "datafusion/substrait",
     "datafusion/wasmtest",
     "datafusion-cli",
@@ -127,6 +128,7 @@ datafusion-physical-plan = { path = "datafusion/physical-plan", version = "45.0.
 datafusion-proto = { path = "datafusion/proto", version = "45.0.0" }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "45.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "45.0.0" }
+datafusion-storage = { path = "datafusion/storage", version = "45.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
@@ -144,6 +146,7 @@ parquet = { version = "54.2.0", default-features = false, features = [
 ] }
 pbjson = { version = "0.7.0" }
 pbjson-types = "0.7"
+percent-encoding = "2.1.0"
 # Should match arrow-flight's version of prost.
 prost = "0.13.1"
 rand = "0.8.5"

datafusion/storage/Cargo.toml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+authors = { workspace = true }
+description = "DataFusion Storage"
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+name = "datafusion-storage"
+readme = "README.md"
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[lints]
+workspace = true
+
+[lib]
+name = "datafusion_storage"
+
+[dependencies]
+datafusion-common = { workspace = true, default-features = true }
+
+async-trait = { workspace = true }
+futures =  { workspace = true }
+url =  { workspace = true }
+chrono =  { workspace = true }
+bytes =  { workspace = true }
+dashmap =  { workspace = true }

datafusion/storage/LICENSE.txt

@@ -0,0 +1 @@
+../../LICENSE.txt

datafusion/storage/NOTICE.txt

@@ -0,0 +1 @@
+../../NOTICE.txt

datafusion/storage/README.md

@@ -0,0 +1,24 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# DataFusion Storage
+
+This crate provides a general-purpose storage solution that can interact with various storage services, such as S3 and the local fs. Although it is used by the [DataFusion][df] query engine, it is designed to be easily integrated into any project requiring a storage interface.
+
+[df]: https://crates.io/crates/datafusion

datafusion/storage/src/api.rs

@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::context::StorageContext;
+use crate::file_metadata::StorageFileMetadata;
+use crate::list::{StorageFileLister, StorageListOptions};
+use crate::read::StorageFileReader;
+use crate::stat::StorageStatOptions;
+use crate::StorageFileWriter;
+use crate::{StorageReadOptions, StorageWriteOptions, StorageWriteResult};
+use async_trait::async_trait;
+use bytes::Bytes;
+use datafusion_common::Result;
+use futures::Stream;
+use std::fmt::Debug;
+use std::ops::Range;
+
+#[async_trait]
+pub trait Storage: std::fmt::Display + Send + Sync + Debug + 'static {
+    async fn write_opts(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        options: StorageWriteOptions,
+    ) -> Result<StorageFileWriter>;
+
+    async fn read_opts(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        options: StorageReadOptions,
+    ) -> Result<StorageFileReader>;
+
+    async fn stat_opts(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        options: StorageStatOptions,
+    ) -> Result<StorageFileMetadata>;
+
+    async fn list_opts(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        options: StorageListOptions,
+    ) -> Result<StorageFileLister>;
+}
+
+#[async_trait]
+pub trait StorageExt: Storage {
+    async fn write(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        payload: Bytes,
+    ) -> Result<StorageWriteResult> {
+        todo!()
+    }
+
+    async fn write_iter(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        payload: impl Iterator<Item = Result<Bytes>> + Send,
+    ) -> Result<StorageWriteResult> {
+        todo!()
+    }
+
+    async fn write_stream(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        stream: impl Stream<Item = Result<Bytes>> + Send,
+    ) -> Result<StorageWriteResult> {
+        todo!()
+    }
+
+    async fn read(&self, ctx: &StorageContext, location: &str) -> Result<Bytes> {
+        todo!()
+    }
+
+    async fn read_range(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        range: Range<usize>,
+    ) -> Result<Bytes> {
+        todo!()
+    }
+
+    async fn read_ranges(
+        &self,
+        ctx: &StorageContext,
+        location: &str,
+        ranges: &[Range<usize>],
+    ) -> Result<Vec<Bytes>> {
+        todo!()
+    }
+}

datafusion/storage/src/context.rs

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Context for storage operations
+pub struct StorageContext {
+    // TODO: add more datafusion context here
+
+    // metrics: metrics for the storage operations
+    // tracing_span: tracing span for the storage operations
+    // http_client: http_client for all the storage operations
+    // runtime: async runtime for the storage operations
+}

datafusion/storage/src/error.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub enum StorageError {
+    Generic {
+        /// The store this error originated from
+        store: &'static str,
+        /// The wrapped error
+        source: Box<dyn std::error::Error + Send + Sync + 'static>,
+    },
+}

datafusion/storage/src/file_metadata.rs

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use chrono::{DateTime, Utc};
+
+/// The metadata that describes a file inside a storage system.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct StorageFileMetadata {
+    /// The full path to the object
+    pub location: String,
+    /// The last modified time
+    pub last_modified: DateTime<Utc>,
+    /// The size in bytes of the object
+    pub size: u64,
+    /// The unique identifier for the object
+    ///
+    /// <https://datatracker.ietf.org/doc/html/rfc9110#name-etag>
+    pub e_tag: Option<String>,
+    /// A version indicator for this object
+    pub version: Option<String>,
+}