diff --git a/Cargo.lock b/Cargo.lock index 71a3c073c552a..575c18189a14b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1462,7 +1462,7 @@ dependencies = [ "log", "paste", "pin-project", - "quick-xml", + "quick-xml 0.31.0", "rand 0.8.5", "reqwest 0.11.26", "rustc_version 0.4.1", @@ -1549,13 +1549,12 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113" dependencies = [ - "fastrand 1.9.0", - "futures-core", - "pin-project", + "fastrand 2.3.0", + "gloo-timers", "tokio", ] @@ -3624,12 +3623,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" -[[package]] -name = "flagset" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779" - [[package]] name = "flate2" version = "1.0.35" @@ -3913,6 +3906,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "gloo-utils" version = "0.2.0" @@ -4564,7 +4569,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.8", "tokio", "tower-service", "tracing 0.1.41", @@ -6624,26 +6629,25 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.45.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c17c077f23fa2d2c25d9d22af98baa43b8bbe2ef0de80cf66339aa70401467" +checksum = "a55c840b5a6ad96106d6c0612fabb8f35a5ace826e0474fc55ebda33042b8d33" dependencies = [ "anyhow", "async-trait", "backon", - "base64 0.21.7", + "base64 0.22.1", "bytes 1.10.0", "chrono", - "flagset", "futures 0.3.31", "getrandom 0.2.15", - "http 0.2.9", + "http 1.1.0", "log", "md-5", "once_cell", "percent-encoding", - "quick-xml", - "reqwest 0.11.26", + "quick-xml 0.36.2", + "reqwest 0.12.4", "serde", "serde_json", "tokio", @@ -7580,7 +7584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.93", "quote 1.0.38", "syn 2.0.98", @@ -7748,6 +7752,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quickcheck" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 0963bb6c7c711..7d165215c068a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -251,7 +251,7 @@ azure_storage = { version = "0.17", default-features = false, optional = true } azure_storage_blobs = { version = "0.17", default-features = false, optional = true } # OpenDAL -opendal = { version = "0.45", default-features = false, features = ["native-tls", "services-webhdfs"], optional = true } +opendal = { version = "0.52", default-features = false, features = ["services-webhdfs"], optional = true } # Tower tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 52b29cc6d6d92..f4fb5f0c82926 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -82,7 +82,7 @@ azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage_blobs,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. backoff,https://github.com/ihrwein/backoff,MIT OR Apache-2.0,Tibor Benke -backon,https://github.com/Xuanwo/backon,Apache-2.0,Xuanwo +backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors backtrace,https://github.com/rust-lang/backtrace-rs,MIT OR Apache-2.0,The Rust Project Developers base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni base16ct,https://github.com/RustCrypto/formats/tree/master/base16ct,Apache-2.0 OR MIT,RustCrypto Developers @@ -251,6 +251,7 @@ generic-array,https://github.com/fizyk20/generic-array,MIT,"Bartłomiej Kamińsk getrandom,https://github.com/rust-random/getrandom,MIT OR Apache-2.0,The Rand Project Developers gimli,https://github.com/gimli-rs/gimli,MIT OR Apache-2.0,The gimli Authors glob,https://github.com/rust-lang/glob,MIT OR Apache-2.0,The Rust Project Developers +gloo-timers,https://github.com/rustwasm/gloo/tree/master/crates/timers,MIT OR Apache-2.0,Rust and WebAssembly Working Group goauth,https://github.com/durch/rust-goauth,MIT,Drazen Urch governor,https://github.com/boinkor-net/governor,MIT,Andreas Fuchs graphql-introspection-query,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé diff --git a/src/sinks/webhdfs/config.rs b/src/sinks/webhdfs/config.rs index e74ffd3da1523..749ba310d97cd 100644 --- a/src/sinks/webhdfs/config.rs +++ b/src/sinks/webhdfs/config.rs @@ -123,8 +123,8 @@ impl WebHdfsConfig { // Build OpenDal Operator let mut builder = Webhdfs::default(); // Prefix logic will be handled by key_partitioner. - builder.root(&self.root); - builder.endpoint(&self.endpoint); + builder = builder.root(&self.root); + builder = builder.endpoint(&self.endpoint); let op = Operator::new(builder)? .layer(LoggingLayer::default()) diff --git a/src/sinks/webhdfs/integration_tests.rs b/src/sinks/webhdfs/integration_tests.rs index 7a453e21cf4dc..ad0bb93855c24 100644 --- a/src/sinks/webhdfs/integration_tests.rs +++ b/src/sinks/webhdfs/integration_tests.rs @@ -4,7 +4,7 @@ use std::{ }; use futures::stream; -use opendal::{Entry, Metakey}; +use opendal::Entry; use similar_asserts::assert_eq; use vector_lib::codecs::{encoding::FramingConfig, TextSerializerConfig}; use vector_lib::event::{Event, LogEvent}; @@ -77,7 +77,6 @@ async fn hdfs_rotate_files_after_the_buffer_size_is_reached() { let mut objects: Vec = op .list_with("/") .recursive(true) - .metakey(Metakey::Mode) .await .unwrap(); @@ -87,7 +86,7 @@ async fn hdfs_rotate_files_after_the_buffer_size_is_reached() { let mut response_lines: Vec> = Vec::new(); for o in objects { - let bs = op.read(o.path()).await.unwrap(); + let bs = op.read(o.path()).await.unwrap().to_vec(); let buf_read = BufReader::new(Cursor::new(bs)); response_lines.push(buf_read.lines().map(|l| l.unwrap()).collect());