From 7642b9a50e9746c16a3fb7fa788f3ffb57ddbfae Mon Sep 17 00:00:00 2001 From: mikemiles-dev Date: Sat, 22 Feb 2025 21:01:25 -0600 Subject: [PATCH] fix(Can now parse enterprise fields in non options templates for IPFIX.) --- Cargo.toml | 2 +- README.md | 10 + RELEASES.md | 3 + SECURITY.md | 11 +- src/lib.rs | 11 ++ ...it_doesnt_parse_0_length_fields_ipfix.snap | 1 - ...nterprise_bit_in_non_options_template.snap | 28 +++ ...tests__it_parses_ipfix_scappy_example.snap | 176 ++++++++++++++++++ ...x_scappy_example_options_template.snap.new | 31 +++ ..._with_no_template_fields_raises_error.snap | 1 - ...__base_tests__it_parses_v5_incomplete.snap | 1 - src/tests.rs | 31 +++ src/variable_versions/ipfix.rs | 15 +- src/variable_versions/v9_lookup.rs | 2 + 14 files changed, 301 insertions(+), 22 deletions(-) create mode 100644 src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap create mode 100644 src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap create mode 100644 src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap.new diff --git a/Cargo.toml b/Cargo.toml index b6d605a..68de73b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "netflow_parser" description = "Parser for Netflow Cisco V5, V7, V9, IPFIX" -version = "0.5.1" +version = "0.5.2" edition = "2021" authors = ["michael.mileusnich@gmail.com"] license = "MIT OR Apache-2.0" diff --git a/README.md b/README.md index 982a466..b40c85f 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,8 @@ if let NetflowPacket::V5(v5) = NetflowParser::default() Parse the data ('&[u8]' as any other versions. The parser (NetflowParser) holds onto already parsed templates, so you can just send a header/data flowset combo and it will use the cached templates.) To see cached templates simply use the parser for the correct version (v9_parser for v9, ipfix_parser for IPFix.) +**IPFIx Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it. + ```rust use netflow_parser::NetflowParser; let parser = NetflowParser::default(); @@ -178,3 +180,11 @@ or or ```cargo run --example netflow_udp_listener_tokio``` + +## Support My Work + +If you find my work helpful, consider supporting me! + +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/michaelmileusnich) + +[![GitHub Sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/mikemiles-dev) diff --git a/RELEASES.md b/RELEASES.md index ad80da9..db919d8 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,6 @@ +# 0.5.2 +* Can now parse enterprise fields in non options templates for IPFIX. + # 0.5.1 * Reworked NetflowParseError. Added a Partial Type. * Added ability to parse only `allowed_versions`. diff --git a/SECURITY.md b/SECURITY.md index 02ec611..bae7c96 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,14 +4,5 @@ | Version | Supported | |---------| ------------------ | -| 0.5.1 | :white_check_mark: | -| 0.5.0 | :white_check_mark: | -| 0.4.9 | :white_check_mark: | -| 0.4.8 | :white_check_mark: | -| 0.4.7 | :white_check_mark: | -| 0.4.6 | :white_check_mark: | -| 0.4.5 | :white_check_mark: | -| 0.4.4 | :white_check_mark: | -| 0.4.3 | :white_check_mark: | -| 0.4.2 | :white_check_mark: | +| >0.4.1 | :white_check_mark: | | <0.4.1 | Not Supported | diff --git a/src/lib.rs b/src/lib.rs index 9fd6380..c91bedc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,6 +150,9 @@ //! ## V9/IPFix notes: //! //! Parse the data (`&[u8]` as any other versions. The parser (NetflowParser) holds onto already parsed templates, so you can just send a header/data flowset combo, and it will use the cached templates.) To see cached templates simply use the parser for the correct version (v9_parser for v9, ipfix_parser for IPFix.) +//! +//! **IPFIx Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it. +//! //! ```rust //! use netflow_parser::NetflowParser; //! let parser = NetflowParser::default(); @@ -176,6 +179,14 @@ //! or //! //! ```cargo run --example netflow_udp_listener_tokio``` +//! +//! ## Support My Work +//! +//! If you find my work helpful, consider supporting me! +//! +//! [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/michaelmileusnich) +//! +//! [![GitHub Sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/mikemiles-dev) pub mod netflow_common; pub mod protocol; diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap index 5fe2f8a..0dc21e4 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap @@ -103,4 +103,3 @@ expression: "NetflowParser::default().parse_bytes(&packet)" - 2 - 3 - 4 - diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap new file mode 100644 index 0000000..b0f146c --- /dev/null +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap @@ -0,0 +1,28 @@ +--- +source: src/tests.rs +expression: "NetflowParser::default().parse_bytes(&packet)" +--- +- IPFix: + header: + version: 10 + length: 42 + export_time: 1670052913 + sequence_number: 0 + observation_domain_id: 0 + flowsets: + - header: + header_id: 2 + length: 26 + body: + templates: + template_id: 260 + field_count: 2 + fields: + - field_type_number: 32871 + field_type: Unknown + field_length: 65535 + enterprise_number: 407732327 + - field_type_number: 65535 + field_type: Unknown + field_length: 0 + enterprise_number: 407732544 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap new file mode 100644 index 0000000..63422d5 --- /dev/null +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap @@ -0,0 +1,176 @@ +--- +source: src/tests.rs +expression: result +--- +- IPFix: + header: + version: 10 + length: 116 + export_time: 1480450135 + sequence_number: 3791 + observation_domain_id: 0 + flowsets: + - header: + header_id: 2 + length: 100 + body: + templates: + template_id: 307 + field_count: 23 + fields: + - field_type_number: 8 + field_type: SourceIpv4address + field_length: 4 + - field_type_number: 12 + field_type: DestinationIpv4address + field_length: 4 + - field_type_number: 5 + field_type: IpClassOfService + field_length: 1 + - field_type_number: 4 + field_type: ProtocolIdentifier + field_length: 1 + - field_type_number: 7 + field_type: SourceTransportPort + field_length: 2 + - field_type_number: 11 + field_type: DestinationTransportPort + field_length: 2 + - field_type_number: 32 + field_type: IcmpTypeCodeIpv4 + field_length: 2 + - field_type_number: 10 + field_type: IngressInterface + field_length: 4 + - field_type_number: 16 + field_type: BgpSourceAsNumber + field_length: 4 + - field_type_number: 17 + field_type: BgpDestinationAsNumber + field_length: 4 + - field_type_number: 18 + field_type: BgpNextHopIpv4address + field_length: 4 + - field_type_number: 14 + field_type: EgressInterface + field_length: 4 + - field_type_number: 1 + field_type: OctetDeltaCount + field_length: 4 + - field_type_number: 2 + field_type: PacketDeltaCount + field_length: 4 + - field_type_number: 22 + field_type: FlowStartSysUpTime + field_length: 4 + - field_type_number: 21 + field_type: FlowEndSysUpTime + field_length: 4 + - field_type_number: 15 + field_type: IpNextHopIpv4address + field_length: 4 + - field_type_number: 9 + field_type: SourceIpv4prefixLength + field_length: 1 + - field_type_number: 13 + field_type: DestinationIpv4prefixLength + field_length: 1 + - field_type_number: 6 + field_type: TcpControlBits + field_length: 1 + - field_type_number: 60 + field_type: IpVersion + field_length: 1 + - field_type_number: 152 + field_type: FlowStartMilliseconds + field_length: 8 + - field_type_number: 153 + field_type: FlowEndMilliseconds + field_length: 8 +- IPFix: + header: + version: 10 + length: 96 + export_time: 1480450137 + sequence_number: 3812 + observation_domain_id: 0 + flowsets: + - header: + header_id: 307 + length: 80 + body: + data: + data_fields: + - 0: + - SourceIpv4address + - Ip4Addr: 70.1.115.1 + 1: + - DestinationIpv4address + - Ip4Addr: 50.0.71.1 + 2: + - IpClassOfService + - DataNumber: 0 + 3: + - ProtocolIdentifier + - DataNumber: 61 + 4: + - SourceTransportPort + - DataNumber: 0 + 5: + - DestinationTransportPort + - DataNumber: 0 + 6: + - IcmpTypeCodeIpv4 + - DataNumber: 0 + 7: + - IngressInterface + - DataNumber: 827 + 8: + - BgpSourceAsNumber + - DataNumber: 2 + 9: + - BgpDestinationAsNumber + - DataNumber: 3 + 10: + - BgpNextHopIpv4address + - Ip4Addr: 204.42.110.101 + 11: + - EgressInterface + - DataNumber: 854 + 12: + - OctetDeltaCount + - DataNumber: 1312 + 13: + - PacketDeltaCount + - DataNumber: 9 + 14: + - FlowStartSysUpTime + - DataNumber: 3019441902 + 15: + - FlowEndSysUpTime + - DataNumber: 3019616060 + 16: + - IpNextHopIpv4address + - Ip4Addr: 204.42.110.189 + 17: + - SourceIpv4prefixLength + - DataNumber: 24 + 18: + - DestinationIpv4prefixLength + - DataNumber: 24 + 19: + - TcpControlBits + - DataNumber: 0 + 20: + - IpVersion + - DataNumber: 4 + 21: + - FlowStartMilliseconds + - Duration: + secs: 1480449931 + nanos: 519000000 + 22: + - FlowEndMilliseconds + - Duration: + secs: 1480450105 + nanos: 677000000 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap.new b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap.new new file mode 100644 index 0000000..db66b8e --- /dev/null +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap.new @@ -0,0 +1,31 @@ +--- +source: src/tests.rs +assertion_line: 437 +expression: result +--- +- IPFix: + header: + version: 10 + length: 40 + export_time: 1480450135 + sequence_number: 3791 + observation_domain_id: 0 + flowsets: + - header: + header_id: 3 + length: 24 + body: + options_templates: + template_id: 308 + field_count: 3 + scope_field_count: 1 + fields: + - field_type_number: 32773 + field_type: IpClassOfService + field_length: 2 + - field_type_number: 32804 + field_type: FlowActiveTimeout + field_length: 2 + - field_type_number: 32805 + field_type: FlowIdleTimeout + field_length: 2 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap index b1557e2..97c826c 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap @@ -59,4 +59,3 @@ expression: parser.parse_bytes(&packet) - 0 - 1 - 1 - diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_v5_incomplete.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_v5_incomplete.snap index ffdda02..9960f8a 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_v5_incomplete.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_v5_incomplete.snap @@ -23,4 +23,3 @@ expression: "NetflowParser::default().parse_bytes(&packet)" - 1 - 1 - 1 - diff --git a/src/tests.rs b/src/tests.rs index 2308277..f0614ff 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -405,4 +405,35 @@ mod base_tests { ]; assert_yaml_snapshot!(NetflowParser::default().parse_bytes(&packet)); } + + #[test] + fn it_parses_ipfix_enterprise_bit_in_non_options_template() { + let packet = [ + 0, 10, 0, 42, 99, 138, 252, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 1, 4, 0, 2, + 128, 103, 255, 255, 24, 77, 128, 103, 255, 255, 0, 0, 24, 77, 129, 64, 0, 4, 0, 0, + 24, 77, + ]; + assert_yaml_snapshot!(NetflowParser::default().parse_bytes(&packet)); + } + + #[test] + fn it_parses_ipfix_scappy_example() { + let hex_template = r#"000a0074583de05700000ecf00000000000200640133001700080004000c0004000500010004000100070002000b000200200002000a0004001000040011000400120004000e000400010004000200040016000400150004000f000400090001000d000100060001003c00010098000800990008"#; + let packet = hex::decode(hex_template).unwrap(); + let mut parser = NetflowParser::default(); + let mut result = parser.parse_bytes(&packet); + let hex_data = r#"000a0060583de05900000ee400000000013300504601730132004701003d0000000000000000033b0000000200000003cc2a6e65000003560000052000000009b3f906eeb3fbaf3ccc2a6ebd1818000400000158b1b138ff00000158b1b3e14d"#; + let packet = hex::decode(hex_data).unwrap(); + result.append(&mut parser.parse_bytes(&packet)); + assert_yaml_snapshot!(result); + } + + #[test] + fn it_parses_ipfix_scappy_example_options_template() { + let hex_template = r#"000a0028583de05700000ecf00000000000300180134000300010005000200240002002500020000"#; + let packet = hex::decode(hex_template).unwrap(); + let mut parser = NetflowParser::default(); + let result = parser.parse_bytes(&packet); + assert_yaml_snapshot!(result); + } } diff --git a/src/variable_versions/ipfix.rs b/src/variable_versions/ipfix.rs index 818ed7c..f185d50 100644 --- a/src/variable_versions/ipfix.rs +++ b/src/variable_versions/ipfix.rs @@ -21,7 +21,6 @@ use Nom; use std::collections::BTreeMap; -const TEMPLATE_ID: u16 = 2; const OPTIONS_TEMPLATE_ID: u16 = 3; const SET_MIN_RANGE: u16 = 255; @@ -115,7 +114,7 @@ pub struct FlowSetHeader { #[nom(ExtraArgs(parser: &mut IPFixParser, id: u16, length: u16))] pub struct FlowSetBody { #[nom( - Cond = "id == TEMPLATE_ID", + Cond = "id < SET_MIN_RANGE && id != OPTIONS_TEMPLATE_ID", // Save our templates PostExec = "if let Some(templates) = templates.clone() { parser.templates.insert(templates.template_id, templates); }" )] @@ -171,8 +170,8 @@ pub struct OptionsTemplate { pub field_count: u16, pub scope_field_count: u16, #[nom( - PreExec = "let combined_count = scope_field_count as usize + - field_count.checked_sub(scope_field_count).unwrap_or(field_count) as usize;", + PreExec = "let combined_count = scope_field_count.saturating_add( + field_count.checked_sub(scope_field_count).unwrap_or(field_count)) as usize;", Parse = "count(|i| TemplateField::parse(i, true), combined_count)", PostExec = "let options_remaining = set_length.checked_sub(field_count * 4).unwrap_or(set_length) > 0;" )] @@ -212,7 +211,7 @@ pub struct TemplateField { pub field_type: IPFixField, pub field_length: u16, #[nom( - Cond = "options_template && field_type_number > 32767", + Cond = "field_type_number > 32767", PostExec = "let field_type_number = if options_template { field_type_number.overflowing_sub(32768).0 } else { field_type_number };", @@ -328,7 +327,7 @@ fn parse_field<'a>( if has_enterprise_number { // Simplified parsing when `enterprise_number` is present - parse_enterprise_field(i) + parse_enterprise_field(i, template_field.field_length) } else { // Parse field based on its type and length DataNumber::from_field_type( @@ -339,8 +338,8 @@ fn parse_field<'a>( } } -fn parse_enterprise_field(i: &[u8]) -> IResult<&[u8], FieldValue> { - let (remaining, data_number) = DataNumber::parse(i, 4, false)?; +fn parse_enterprise_field(i: &[u8], length: u16) -> IResult<&[u8], FieldValue> { + let (remaining, data_number) = DataNumber::parse(i, length, false)?; Ok((remaining, FieldValue::DataNumber(data_number))) } diff --git a/src/variable_versions/v9_lookup.rs b/src/variable_versions/v9_lookup.rs index e8057c0..d45d850 100644 --- a/src/variable_versions/v9_lookup.rs +++ b/src/variable_versions/v9_lookup.rs @@ -1,3 +1,5 @@ +//! See: + use super::data_number::*; use nom_derive::*;