From e1b1e8a5ef9c91d8a0fb9c3b21952f822ef7714a Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 6 Mar 2025 17:39:46 +0800 Subject: [PATCH 1/2] refactor: use fasyslog for creating syslog payload Signed-off-by: tison --- Cargo.lock | 143 +++++++++++++++++++++++++++++++--------- Cargo.toml | 4 +- src/sinks/papertrail.rs | 47 +++++++------ 3 files changed, 137 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71a3c073c552a..ed8cef28d0e92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3413,15 +3413,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "version_check", -] - [[package]] name = "error-code" version = "3.0.0" @@ -3556,6 +3547,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fasyslog" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1347a22d141c0c2ac2ed1acef6809f1bb2bd42de43bd228c891199c3375d64e" +dependencies = [ + "cfg-if", + "jiff", + "nix 0.29.0", + "windows-targets 0.53.0", +] + [[package]] name = "ff" version = "0.13.0" @@ -4564,7 +4567,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.8", "tokio", "tower-service", "tracing 0.1.41", @@ -5181,6 +5184,35 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3590fea8e9e22d449600c9bbd481a8163bef223e4ff938e5f55899f8cf1adb93" +dependencies = [ + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", + "windows-sys 0.59.0", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2cec2f5d266af45a071ece48b1fb89f3b00b2421ac3a5fe10285a6caaa60d3" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a63c62e404e7b92979d2792352d885a7f8f83fd1d0d31eea582d77b2ceca697e" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -7199,15 +7231,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "portable-atomic-util" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdd8420072e66d54a407b3316991fe946ce3ab1083a7f575b2463866624704d" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" dependencies = [ "portable-atomic", ] @@ -7580,7 +7612,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.93", "quote 1.0.38", "syn 2.0.98", @@ -9580,19 +9612,6 @@ dependencies = [ "windows 0.57.0", ] -[[package]] -name = "syslog" -version = "6.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc7e95b5b795122fafe6519e27629b5ab4232c73ebb2428f568e82b1a457ad3" -dependencies = [ - "error-chain", - "hostname 0.3.1", - "libc", - "log", - "time", -] - [[package]] name = "syslog_loose" version = "0.21.0" @@ -10933,6 +10952,7 @@ dependencies = [ "evmap-derive", "exitcode", "fakedata", + "fasyslog", "flate2", "futures 0.3.31", "futures-util", @@ -11030,7 +11050,6 @@ dependencies = [ "stream-cancel", "strip-ansi-escapes", "sysinfo", - "syslog", "tempfile", "test-generator", "thread_local", @@ -12009,13 +12028,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -12034,6 +12069,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -12052,6 +12093,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -12070,12 +12117,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -12094,6 +12153,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -12112,6 +12177,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -12130,6 +12201,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -12148,6 +12225,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.5.18" diff --git a/Cargo.toml b/Cargo.toml index 0963bb6c7c711..eeb2c7632d154 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -372,7 +372,7 @@ snap = { version = "1.1.1", default-features = false } socket2 = { version = "0.5.8", default-features = false } stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } -syslog = { version = "6.1.1", default-features = false, optional = true } +fasyslog = { version = "1.0.0", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } tokio-postgres = { version = "0.7.13", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect"], optional = true } @@ -807,7 +807,7 @@ sinks-nats = ["dep:async-nats", "dep:nkeys"] sinks-new_relic_logs = ["sinks-http"] sinks-new_relic = [] sinks-opentelemetry = ["sinks-http"] -sinks-papertrail = ["dep:syslog"] +sinks-papertrail = ["dep:fasyslog"] sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"] sinks-pulsar = ["dep:apache-avro", "dep:pulsar", "dep:lru"] sinks-redis = ["dep:redis"] diff --git a/src/sinks/papertrail.rs b/src/sinks/papertrail.rs index f152260a37be8..a0d4ee15a4cad 100644 --- a/src/sinks/papertrail.rs +++ b/src/sinks/papertrail.rs @@ -1,5 +1,6 @@ use bytes::{BufMut, BytesMut}; -use syslog::{Facility, Formatter3164, LogFormat, Severity}; +use fasyslog::format::SyslogContext; +use fasyslog::{Facility, Severity}; use vector_lib::configurable::configurable_component; use vrl::value::Kind; @@ -138,30 +139,27 @@ impl tokio_util::codec::Encoder for PapertrailEncoder { mut event: Event, buffer: &mut bytes::BytesMut, ) -> Result<(), Self::Error> { - let host = event + let mut context = SyslogContext::const_new(); + if let Some(host) = event .as_mut_log() .get_host() - .map(|host| host.to_string_lossy().into_owned()); - - let process = self - .process - .render_string(&event) - .map_err(|error| { - emit!(TemplateRenderingError { - error, - field: Some("process"), - drop_event: false, - }) + .map(|host| host.to_string_lossy().into_owned()) + { + context.hostname(host); + } + if let Ok(process) = self.process.render_string(&event).map_err(|error| { + emit!(TemplateRenderingError { + error, + field: Some("process"), + drop_event: false, }) - .ok() - .unwrap_or_else(|| String::from("vector")); - - let formatter = Formatter3164 { - facility: Facility::LOG_USER, - hostname: host, - process, - pid: self.pid, - }; + }) { + context.appname(process); + } else { + context.appname("vector"); + } + context.procid(self.pid.to_string()); + context.facility(Facility::USER); self.transformer.transform(&mut event); @@ -169,9 +167,8 @@ impl tokio_util::codec::Encoder for PapertrailEncoder { self.encoder.encode(event, &mut bytes)?; let message = String::from_utf8_lossy(&bytes); - - formatter - .format(&mut buffer.writer(), Severity::LOG_INFO, message) + let format = context.format_rfc3164(Severity::INFORMATIONAL, Some(message)); + write!(buffer, "{format}") .map_err(|error| Self::Error::SerializingError(format!("{}", error).into()))?; buffer.put_u8(b'\n'); From 7d9d3ebcaf896eb38e4a7897b420b8e0ca5aacb0 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 6 Mar 2025 18:11:57 +0800 Subject: [PATCH 2/2] tidy Signed-off-by: tison --- LICENSE-3rdparty.csv | 5 +++-- src/sinks/papertrail.rs | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 52b29cc6d6d92..c9dc064bbc9dc 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -208,7 +208,6 @@ env_logger,https://github.com/env-logger-rs/env_logger,MIT OR Apache-2.0,The Rus equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors erased-serde,https://github.com/dtolnay/erased-serde,MIT OR Apache-2.0,David Tolnay errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,Chris Wong -error-chain,https://github.com/rust-lang-nursery/error-chain,MIT OR Apache-2.0,"Brian Anderson , Paul Colomiets , Colin Kiegel , Yamakaky , Andrew Gauger " error-code,https://github.com/DoumanAsh/error-code,BSL-1.0,Douman event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjepan Glavina event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " @@ -220,6 +219,7 @@ fakedata_generator,https://github.com/kevingimbel/fakedata_generator,MIT,Kevin G fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina +fasyslog,https://github.com/fast/fasyslog,Apache-2.0,The fasyslog Authors ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors filetime,https://github.com/alexcrichton/filetime,MIT OR Apache-2.0,Alex Crichton @@ -328,6 +328,7 @@ is-terminal,https://github.com/sunfishcode/is-terminal,MIT,"softprops itertools,https://github.com/rust-itertools/itertools,MIT OR Apache-2.0,bluss itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay +jiff,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase jni-sys,https://github.com/sfackler/rust-jni-sys,MIT OR Apache-2.0,Steven Fackler js-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers @@ -456,6 +457,7 @@ polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,Stjepan Glavina , John Nunley " poly1305,https://github.com/RustCrypto/universal-hashes,Apache-2.0 OR MIT,RustCrypto Developers portable-atomic,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic Authors +portable-atomic-util,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic-util Authors postgres-openssl,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler postgres-protocol,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler postgres-types,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler @@ -606,7 +608,6 @@ syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay synstructure,https://github.com/mystor/synstructure,MIT,Nika Layzell sysinfo,https://github.com/GuillaumeGomez/sysinfo,MIT,Guillaume Gomez -syslog,https://github.com/Geal/rust-syslog,MIT,contact@geoffroycouprie.com syslog_loose,https://github.com/FungusHumungus/syslog-loose,MIT,Stephen Wakely system-configuration,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN tagptr,https://github.com/oliver-giersch/tagptr,MIT OR Apache-2.0,Oliver Giersch diff --git a/src/sinks/papertrail.rs b/src/sinks/papertrail.rs index a0d4ee15a4cad..753ee95c66ff4 100644 --- a/src/sinks/papertrail.rs +++ b/src/sinks/papertrail.rs @@ -1,6 +1,7 @@ use bytes::{BufMut, BytesMut}; use fasyslog::format::SyslogContext; use fasyslog::{Facility, Severity}; +use std::fmt::Write; use vector_lib::configurable::configurable_component; use vrl::value::Kind;