From 8130360d30b9a8e8bce7e0a77f6c3afb15b7a590 Mon Sep 17 00:00:00 2001 From: mikemiles-dev Date: Sat, 1 Mar 2025 23:34:42 -0600 Subject: [PATCH] Fix: Fixed bug when calcualting the enteperise field. --- Cargo.toml | 4 +- RELEASES.md | 6 + benches/netflow_common_bench.rs | 2 +- benches/netflow_ipfix_bench.rs | 2 +- benches/netflow_parser_bench.rs | 2 +- benches/netflow_v5_bench.rs | 2 +- benches/netflow_v7_bench.rs | 2 +- benches/netflow_v9_bench.rs | 2 +- .../netflow_udp_listener_multi_threaded.rs | 10 +- src/lib.rs | 2 +- src/netflow_common.rs | 6 +- ...it_doesnt_parse_0_length_fields_ipfix.snap | 105 ----- ...ests__it_parses_0_length_fields_ipfix.snap | 44 +++ ...r__tests__base_tests__it_parses_ipfix.snap | 10 +- ..._it_parses_ipfix_data_cached_template.snap | 2 +- ...nterprise_bit_in_non_options_template.snap | 10 +- ...pfix_enterprise_bit_template_and_data.snap | 362 ++++++++++++++++++ ...sts__it_parses_ipfix_options_template.snap | 6 +- ...rses_ipfix_options_template_with_data.snap | 26 +- ...tests__it_parses_ipfix_scappy_example.snap | 46 +-- ...ipfix_scappy_example_options_template.snap | 8 +- ...ase_tests__it_parses_multiple_packets.snap | 10 +- src/static_versions/v5.rs | 2 +- src/static_versions/v7.rs | 2 +- src/tests.rs | 46 ++- src/variable_versions/data_number.rs | 103 +++-- src/variable_versions/ipfix.rs | 155 ++++---- src/variable_versions/v9.rs | 8 +- 28 files changed, 680 insertions(+), 305 deletions(-) delete mode 100644 src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap create mode 100644 src/snapshots/netflow_parser__tests__base_tests__it_parses_0_length_fields_ipfix.snap create mode 100644 src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_template_and_data.snap diff --git a/Cargo.toml b/Cargo.toml index 68de73b..6e073c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "netflow_parser" description = "Parser for Netflow Cisco V5, V7, V9, IPFIX" -version = "0.5.2" -edition = "2021" +version = "0.5.3" +edition = "2024" authors = ["michael.mileusnich@gmail.com"] license = "MIT OR Apache-2.0" categories = ["encoding", "network-programming", "parser-implementations",] diff --git a/RELEASES.md b/RELEASES.md index db919d8..a558ef0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,9 @@ +# 0.5.3 +* Fixed bug when calcualting the enteperise field. +* Now properly parses variable length fields. +* Cleanup ipfix code. +* Rust 2024 Edition. + # 0.5.2 * Can now parse enterprise fields in non options templates for IPFIX. diff --git a/benches/netflow_common_bench.rs b/benches/netflow_common_bench.rs index 8c19ee4..6a527ee 100644 --- a/benches/netflow_common_bench.rs +++ b/benches/netflow_common_bench.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use netflow_parser::NetflowParser; use std::hint::black_box; diff --git a/benches/netflow_ipfix_bench.rs b/benches/netflow_ipfix_bench.rs index 296ede5..7f45ebb 100644 --- a/benches/netflow_ipfix_bench.rs +++ b/benches/netflow_ipfix_bench.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use netflow_parser::NetflowParser; use std::hint::black_box; diff --git a/benches/netflow_parser_bench.rs b/benches/netflow_parser_bench.rs index b4505c7..c765f9a 100644 --- a/benches/netflow_parser_bench.rs +++ b/benches/netflow_parser_bench.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use netflow_parser::NetflowParser; use std::hint::black_box; diff --git a/benches/netflow_v5_bench.rs b/benches/netflow_v5_bench.rs index 82cffc8..3521c42 100644 --- a/benches/netflow_v5_bench.rs +++ b/benches/netflow_v5_bench.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use netflow_parser::NetflowParser; use std::hint::black_box; diff --git a/benches/netflow_v7_bench.rs b/benches/netflow_v7_bench.rs index 4cd824f..0664983 100644 --- a/benches/netflow_v7_bench.rs +++ b/benches/netflow_v7_bench.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use netflow_parser::NetflowParser; use std::hint::black_box; diff --git a/benches/netflow_v9_bench.rs b/benches/netflow_v9_bench.rs index 36a98eb..bfd1c86 100644 --- a/benches/netflow_v9_bench.rs +++ b/benches/netflow_v9_bench.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use netflow_parser::NetflowParser; use std::hint::black_box; diff --git a/examples/netflow_udp_listener_multi_threaded.rs b/examples/netflow_udp_listener_multi_threaded.rs index b135a04..13feeb2 100644 --- a/examples/netflow_udp_listener_multi_threaded.rs +++ b/examples/netflow_udp_listener_multi_threaded.rs @@ -10,10 +10,12 @@ use netflow_parser::NetflowParser; fn create_thread() -> Sender> { let (tx, rx): (Sender>, Receiver>) = mpsc::channel(); let mut parser = NetflowParser::default(); - thread::spawn(move || loop { - if let Ok(data) = rx.recv() { - let result = parser.parse_bytes(data.as_slice()); - println!("{:?}", result); + thread::spawn(move || { + loop { + if let Ok(data) = rx.recv() { + let result = parser.parse_bytes(data.as_slice()); + println!("{:?}", result); + } } }); tx diff --git a/src/lib.rs b/src/lib.rs index c91bedc..f03819e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -198,7 +198,7 @@ use crate::netflow_common::{NetflowCommon, NetflowCommonError, NetflowCommonFlow use static_versions::{v5::V5, v7::V7}; use variable_versions::ipfix::{IPFix, IPFixParser}; -use variable_versions::v9::{V9Parser, V9}; +use variable_versions::v9::{V9, V9Parser}; use crate::static_versions::v5; use crate::static_versions::v7; diff --git a/src/netflow_common.rs b/src/netflow_common.rs index 9592017..ecd16cf 100644 --- a/src/netflow_common.rs +++ b/src/netflow_common.rs @@ -1,13 +1,13 @@ use std::collections::BTreeMap; use std::net::IpAddr; +use crate::NetflowPacket; use crate::protocol::ProtocolTypes; use crate::static_versions::{v5::V5, v7::V7}; use crate::variable_versions::data_number::FieldValue; use crate::variable_versions::ipfix_lookup::IPFixField; use crate::variable_versions::v9_lookup::V9Field; use crate::variable_versions::{ipfix::IPFix, v9::V9}; -use crate::NetflowPacket; #[derive(Debug)] pub enum NetflowCommonError { @@ -513,8 +513,8 @@ mod common_tests { length: 0, }, body: IPFixFlowSetBody { - templates: None, - options_templates: None, + template: None, + options_template: None, options_data: None, data: Some(IPFixData { data_fields: vec![BTreeMap::from([ diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap deleted file mode 100644 index 0dc21e4..0000000 --- a/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap +++ /dev/null @@ -1,105 +0,0 @@ ---- -source: src/tests.rs -expression: "NetflowParser::default().parse_bytes(&packet)" ---- -- Error: - error: - Partial: - version: 10 - remaining: - - 0 - - 48 - - 1 - - 2 - - 3 - - 4 - - 0 - - 0 - - 0 - - 0 - - 1 - - 2 - - 3 - - 4 - - 0 - - 2 - - 0 - - 20 - - 1 - - 0 - - 0 - - 3 - - 0 - - 8 - - 0 - - 4 - - 0 - - 12 - - 0 - - 4 - - 0 - - 65 - - 0 - - 0 - - 1 - - 0 - - 0 - - 12 - - 1 - - 2 - - 3 - - 4 - - 1 - - 2 - - 3 - - 4 - error: "Parsing Error: Error { input: [], code: Fail }" - remaining: - - 0 - - 10 - - 0 - - 48 - - 1 - - 2 - - 3 - - 4 - - 0 - - 0 - - 0 - - 0 - - 1 - - 2 - - 3 - - 4 - - 0 - - 2 - - 0 - - 20 - - 1 - - 0 - - 0 - - 3 - - 0 - - 8 - - 0 - - 4 - - 0 - - 12 - - 0 - - 4 - - 0 - - 65 - - 0 - - 0 - - 1 - - 0 - - 0 - - 12 - - 1 - - 2 - - 3 - - 4 - - 1 - - 2 - - 3 - - 4 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_0_length_fields_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_0_length_fields_ipfix.snap new file mode 100644 index 0000000..b9e5500 --- /dev/null +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_0_length_fields_ipfix.snap @@ -0,0 +1,44 @@ +--- +source: src/tests.rs +expression: "NetflowParser::default().parse_bytes(&packet)" +--- +- IPFix: + header: + version: 10 + length: 48 + export_time: 16909060 + sequence_number: 0 + observation_domain_id: 16909060 + flowsets: + - header: + header_id: 2 + length: 20 + body: + template: + template_id: 256 + field_count: 3 + fields: + - field_type_number: 8 + field_type: SourceIpv4address + field_length: 4 + - field_type_number: 12 + field_type: DestinationIpv4address + field_length: 4 + - field_type_number: 65 + field_type: AssignedforNetFlowv9compatibility + field_length: 0 + - header: + header_id: 256 + length: 12 + body: + data: + data_fields: + - 0: + - SourceIpv4address + - Ip4Addr: 1.2.3.4 + - 1: + - DestinationIpv4address + - Ip4Addr: 1.2.3.4 + - 2: + - AssignedforNetFlowv9compatibility + - Vec: [] diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap index f3aa77d..5f6d30b 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap @@ -14,7 +14,7 @@ expression: "NetflowParser::default().parse_bytes(&packet)" header_id: 2 length: 20 body: - templates: + template: template_id: 256 field_count: 3 fields: @@ -36,18 +36,18 @@ expression: "NetflowParser::default().parse_bytes(&packet)" - 0: - SourceIpv4address - Ip4Addr: 1.2.3.4 - 1: + - 1: - DestinationIpv4address - Ip4Addr: 1.2.3.3 - 2: + - 2: - PacketDeltaCount - DataNumber: 16909058 - 0: - SourceIpv4address - Ip4Addr: 0.2.0.2 - 1: + - 1: - DestinationIpv4address - Ip4Addr: 0.1.2.3 - 2: + - 2: - PacketDeltaCount - DataNumber: 67438087 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap index 3e8b9ba..ba647e1 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap @@ -19,6 +19,6 @@ expression: parser.parse_bytes(&packet) - 0: - PacketDeltaCount - DataNumber: 8 - 1: + - 1: - SourceIpv4address - Ip4Addr: 0.0.1.1 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap index b0f146c..af17661 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap @@ -14,15 +14,15 @@ expression: "NetflowParser::default().parse_bytes(&packet)" header_id: 2 length: 26 body: - templates: + template: template_id: 260 field_count: 2 fields: - - field_type_number: 32871 - field_type: Unknown + - field_type_number: 103 + field_type: Enterprise field_length: 65535 enterprise_number: 407732327 - - field_type_number: 65535 - field_type: Unknown + - field_type_number: 32767 + field_type: Enterprise field_length: 0 enterprise_number: 407732544 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_template_and_data.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_template_and_data.snap new file mode 100644 index 0000000..60767f4 --- /dev/null +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_template_and_data.snap @@ -0,0 +1,362 @@ +--- +source: src/tests.rs +expression: result +--- +- IPFix: + header: + version: 10 + length: 244 + export_time: 1530614917 + sequence_number: 34503 + observation_domain_id: 768 + flowsets: + - header: + header_id: 2 + length: 228 + body: + template: + template_id: 268 + field_count: 34 + fields: + - field_type_number: 12236 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 12237 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 12240 + field_type: Enterprise + field_length: 2 + enterprise_number: 9 + - field_type_number: 12241 + field_type: Enterprise + field_length: 2 + enterprise_number: 9 + - field_type_number: 234 + field_type: IngressVrfid + field_length: 4 + - field_type_number: 10 + field_type: IngressInterface + field_length: 4 + - field_type_number: 239 + field_type: BiflowDirection + field_length: 1 + - field_type_number: 12242 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 95 + field_type: ApplicationId + field_length: 4 + - field_type_number: 14 + field_type: EgressInterface + field_length: 4 + - field_type_number: 48 + field_type: SamplerId + field_length: 1 + - field_type_number: 9252 + field_type: Enterprise + field_length: 1 + enterprise_number: 9 + - field_type_number: 9253 + field_type: Enterprise + field_length: 1 + enterprise_number: 9 + - field_type_number: 9357 + field_type: Enterprise + field_length: 65535 + enterprise_number: 9 + - field_type_number: 12235 + field_type: Enterprise + field_length: 65535 + enterprise_number: 9 + - field_type_number: 22 + field_type: FlowStartSysUpTime + field_length: 4 + - field_type_number: 21 + field_type: FlowEndSysUpTime + field_length: 4 + - field_type_number: 278 + field_type: NewConnectionDeltaCount + field_length: 4 + - field_type_number: 232 + field_type: ResponderOctets + field_length: 8 + - field_type_number: 299 + field_type: ResponderPackets + field_length: 8 + - field_type_number: 231 + field_type: InitiatorOctets + field_length: 8 + - field_type_number: 298 + field_type: InitiatorPackets + field_length: 8 + - field_type_number: 9303 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9292 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9300 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9319 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9316 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9268 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9313 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9306 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9307 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9309 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9273 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 + - field_type_number: 9272 + field_type: Enterprise + field_length: 4 + enterprise_number: 9 +- IPFix: + header: + version: 10 + length: 188 + export_time: 1530614908 + sequence_number: 34502 + observation_domain_id: 768 + flowsets: + - header: + header_id: 268 + length: 172 + body: + data: + data_fields: + - 0: + - Enterprise + - Vec: + - 10 + - 209 + - 101 + - 66 + - 1: + - Enterprise + - Vec: + - 10 + - 0 + - 0 + - 241 + - 2: + - Enterprise + - Vec: + - 192 + - 224 + - 3: + - Enterprise + - Vec: + - 0 + - 80 + - 4: + - IngressVrfid + - DataNumber: 0 + - 5: + - IngressInterface + - DataNumber: 13 + - 6: + - BiflowDirection + - DataNumber: 1 + - 7: + - Enterprise + - Vec: + - 141 + - 195 + - 63 + - 224 + - 8: + - ApplicationId + - String: "\u0003\u0000\u0000P" + - 9: + - EgressInterface + - DataNumber: 10 + - 10: + - SamplerId + - DataNumber: 9 + - 11: + - Enterprise + - Vec: + - 16 + - 12: + - Enterprise + - Vec: + - 0 + - 13: + - Enterprise + - Vec: + - 47 + - 68 + - 111 + - 99 + - 76 + - 105 + - 98 + - 49 + - 0 + - 0 + - 1 + - 14: + - Enterprise + - Vec: + - 3 + - 0 + - 0 + - 80 + - 52 + - 2 + - 112 + - 111 + - 114 + - 116 + - 97 + - 108 + - 46 + - 109 + - 101 + - 100 + - 115 + - 105 + - 103 + - 114 + - 111 + - 117 + - 112 + - 46 + - 114 + - 117 + - 15: + - FlowStartSysUpTime + - DataNumber: 564191867 + - 16: + - FlowEndSysUpTime + - DataNumber: 564303806 + - 17: + - NewConnectionDeltaCount + - DataNumber: 1 + - 18: + - ResponderOctets + - DataNumber: 0 + - 19: + - ResponderPackets + - DataNumber: 0 + - 20: + - InitiatorOctets + - DataNumber: 6208 + - 21: + - InitiatorPackets + - DataNumber: 15 + - 22: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 23: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 24: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 25: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 26: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 27: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 28: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 29: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 30: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 31: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 32: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 + - 33: + - Enterprise + - Vec: + - 0 + - 0 + - 0 + - 0 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template.snap index ca3b59e..03a0427 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template.snap @@ -14,7 +14,7 @@ expression: "NetflowParser::default().parse_bytes(&packet)" header_id: 3 length: 28 body: - options_templates: + options_template: template_id: 260 field_count: 3 scope_field_count: 1 @@ -23,9 +23,9 @@ expression: "NetflowParser::default().parse_bytes(&packet)" field_type: Enterprise field_length: 4 enterprise_number: 2 - - field_type_number: 32809 + - field_type_number: 41 field_type: ExportedMessageTotalCount field_length: 2 - - field_type_number: 32810 + - field_type_number: 42 field_type: ExportedFlowRecordTotalCount field_length: 2 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template_with_data.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template_with_data.snap index 49bc742..49af19d 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template_with_data.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template_with_data.snap @@ -14,7 +14,7 @@ expression: "NetflowParser::default().parse_bytes(&packet)" header_id: 3 length: 28 body: - options_templates: + options_template: template_id: 260 field_count: 3 scope_field_count: 1 @@ -23,10 +23,10 @@ expression: "NetflowParser::default().parse_bytes(&packet)" field_type: Enterprise field_length: 4 enterprise_number: 2 - - field_type_number: 32809 + - field_type_number: 41 field_type: ExportedMessageTotalCount field_length: 2 - - field_type_number: 32810 + - field_type_number: 42 field_type: ExportedFlowRecordTotalCount field_length: 2 - header: @@ -37,19 +37,27 @@ expression: "NetflowParser::default().parse_bytes(&packet)" data_fields: - 0: - Enterprise - - DataNumber: 1 - 1: + - Vec: + - 0 + - 0 + - 0 + - 1 + - 1: - ExportedMessageTotalCount - DataNumber: 276 - 2: + - 2: - ExportedFlowRecordTotalCount - DataNumber: 5140 - 0: - Enterprise - - DataNumber: 2 - 1: + - Vec: + - 0 + - 0 + - 0 + - 2 + - 1: - ExportedMessageTotalCount - DataNumber: 5140 - 2: + - 2: - ExportedFlowRecordTotalCount - DataNumber: 7710 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap index 63422d5..04e86c2 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap @@ -14,7 +14,7 @@ expression: result header_id: 2 length: 100 body: - templates: + template: template_id: 307 field_count: 23 fields: @@ -104,72 +104,72 @@ expression: result - 0: - SourceIpv4address - Ip4Addr: 70.1.115.1 - 1: + - 1: - DestinationIpv4address - Ip4Addr: 50.0.71.1 - 2: + - 2: - IpClassOfService - DataNumber: 0 - 3: + - 3: - ProtocolIdentifier - DataNumber: 61 - 4: + - 4: - SourceTransportPort - DataNumber: 0 - 5: + - 5: - DestinationTransportPort - DataNumber: 0 - 6: + - 6: - IcmpTypeCodeIpv4 - DataNumber: 0 - 7: + - 7: - IngressInterface - DataNumber: 827 - 8: + - 8: - BgpSourceAsNumber - DataNumber: 2 - 9: + - 9: - BgpDestinationAsNumber - DataNumber: 3 - 10: + - 10: - BgpNextHopIpv4address - Ip4Addr: 204.42.110.101 - 11: + - 11: - EgressInterface - DataNumber: 854 - 12: + - 12: - OctetDeltaCount - DataNumber: 1312 - 13: + - 13: - PacketDeltaCount - DataNumber: 9 - 14: + - 14: - FlowStartSysUpTime - DataNumber: 3019441902 - 15: + - 15: - FlowEndSysUpTime - DataNumber: 3019616060 - 16: + - 16: - IpNextHopIpv4address - Ip4Addr: 204.42.110.189 - 17: + - 17: - SourceIpv4prefixLength - DataNumber: 24 - 18: + - 18: - DestinationIpv4prefixLength - DataNumber: 24 - 19: + - 19: - TcpControlBits - DataNumber: 0 - 20: + - 20: - IpVersion - DataNumber: 4 - 21: + - 21: - FlowStartMilliseconds - Duration: secs: 1480449931 nanos: 519000000 - 22: + - 22: - FlowEndMilliseconds - Duration: secs: 1480450105 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap index 23d0f8d..b560319 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example_options_template.snap @@ -14,17 +14,17 @@ expression: result header_id: 3 length: 24 body: - options_templates: + options_template: template_id: 308 field_count: 3 scope_field_count: 1 fields: - - field_type_number: 32773 + - field_type_number: 5 field_type: IpClassOfService field_length: 2 - - field_type_number: 32804 + - field_type_number: 36 field_type: FlowActiveTimeout field_length: 2 - - field_type_number: 32805 + - field_type_number: 37 field_type: FlowIdleTimeout field_length: 2 diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_multiple_packets.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_multiple_packets.snap index 0f7e6f0..a2490eb 100644 --- a/src/snapshots/netflow_parser__tests__base_tests__it_parses_multiple_packets.snap +++ b/src/snapshots/netflow_parser__tests__base_tests__it_parses_multiple_packets.snap @@ -149,7 +149,7 @@ expression: "NetflowParser::default().parse_bytes(&all)" header_id: 2 length: 20 body: - templates: + template: template_id: 256 field_count: 3 fields: @@ -171,18 +171,18 @@ expression: "NetflowParser::default().parse_bytes(&all)" - 0: - SourceIpv4address - Ip4Addr: 1.2.3.4 - 1: + - 1: - DestinationIpv4address - Ip4Addr: 1.2.3.3 - 2: + - 2: - PacketDeltaCount - DataNumber: 16909058 - 0: - SourceIpv4address - Ip4Addr: 0.2.0.2 - 1: + - 1: - DestinationIpv4address - Ip4Addr: 0.1.2.3 - 2: + - 2: - PacketDeltaCount - DataNumber: 67438087 diff --git a/src/static_versions/v5.rs b/src/static_versions/v5.rs index f1ede06..1684a47 100644 --- a/src/static_versions/v5.rs +++ b/src/static_versions/v5.rs @@ -6,10 +6,10 @@ use crate::protocol::ProtocolTypes; use crate::{NetflowPacket, NetflowParseError, ParsedNetflow, PartialParse}; +use Nom; use nom::number::complete::be_u32; use nom_derive::*; use serde::Serialize; -use Nom; use std::net::Ipv4Addr; diff --git a/src/static_versions/v7.rs b/src/static_versions/v7.rs index 5a4eb46..f922962 100644 --- a/src/static_versions/v7.rs +++ b/src/static_versions/v7.rs @@ -6,10 +6,10 @@ use crate::protocol::ProtocolTypes; use crate::{NetflowPacket, NetflowParseError, ParsedNetflow, PartialParse}; +use Nom; use nom::number::complete::be_u32; use nom_derive::*; use serde::Serialize; -use Nom; use std::net::Ipv4Addr; diff --git a/src/tests.rs b/src/tests.rs index f0614ff..b118503 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -309,7 +309,7 @@ mod base_tests { } #[test] - fn it_doesnt_parse_0_length_fields_ipfix() { + fn it_parses_0_length_fields_ipfix() { let packet = [ 0, 10, 0, 48, 1, 2, 3, 4, 0, 0, 0, 0, 1, 2, 3, 4, 0, 2, 0, 20, 1, 0, 0, 3, 0, 8, 0, 4, 0, 12, 0, 4, 0, 65, 0, 0, 1, 0, 0, 12, 1, 2, 3, 4, 1, 2, 3, 4, @@ -416,6 +416,50 @@ mod base_tests { assert_yaml_snapshot!(NetflowParser::default().parse_bytes(&packet)); } + #[test] + fn it_parses_ipfix_enterprise_bit_template_and_data() { + let template_packet = [ + 0x00, 0x0a, 0x00, 0xf4, 0x5b, 0x3b, 0x54, 0x85, 0x00, 0x00, 0x86, 0xc7, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x02, 0x00, 0xe4, 0x01, 0x0c, 0x00, 0x22, 0xaf, 0xcc, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x09, 0xaf, 0xcd, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xaf, 0xd0, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x09, 0xaf, 0xd1, 0x00, 0x02, 0x00, 0x00, 0x00, 0x09, + 0x00, 0xea, 0x00, 0x04, 0x00, 0x0a, 0x00, 0x04, 0x00, 0xef, 0x00, 0x01, 0xaf, 0xd2, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0x00, 0x5f, 0x00, 0x04, 0x00, 0x0e, 0x00, 0x04, + 0x00, 0x30, 0x00, 0x01, 0xa4, 0x24, 0x00, 0x01, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x25, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x8d, 0xff, 0xff, 0x00, 0x00, 0x00, 0x09, + 0xaf, 0xcb, 0xff, 0xff, 0x00, 0x00, 0x00, 0x09, 0x00, 0x16, 0x00, 0x04, 0x00, 0x15, + 0x00, 0x04, 0x01, 0x16, 0x00, 0x04, 0x00, 0xe8, 0x00, 0x08, 0x01, 0x2b, 0x00, 0x08, + 0x00, 0xe7, 0x00, 0x08, 0x01, 0x2a, 0x00, 0x08, 0xa4, 0x57, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x09, 0xa4, 0x4c, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x54, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x09, 0xa4, 0x67, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x64, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x34, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, + 0xa4, 0x61, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x5a, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x09, 0xa4, 0x5b, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x5d, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x09, 0xa4, 0x39, 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, 0xa4, 0x38, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x09, + ]; + let data_packet = [ + 0x00, 0x0a, 0x00, 0xbc, 0x5b, 0x3b, 0x54, 0x7c, 0x00, 0x00, 0x86, 0xc6, 0x00, 0x00, + 0x03, 0x00, 0x01, 0x0c, 0x00, 0xac, 0x0a, 0xd1, 0x65, 0x42, 0x0a, 0x00, 0x00, 0xf1, + 0xc0, 0xe0, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x01, 0x8d, + 0xc3, 0x3f, 0xe0, 0x03, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x0a, 0x09, 0x10, 0x00, + 0x0b, 0x2f, 0x44, 0x6f, 0x63, 0x4c, 0x69, 0x62, 0x31, 0x00, 0x00, 0x01, 0x1a, 0x03, + 0x00, 0x00, 0x50, 0x34, 0x02, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x2e, 0x6d, 0x65, + 0x64, 0x73, 0x69, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x72, 0x75, 0x21, 0xa0, 0xe2, + 0x7b, 0x21, 0xa2, 0x97, 0xbe, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mut parser = NetflowParser::default(); + let mut result = parser.parse_bytes(&template_packet); + result.append(&mut parser.parse_bytes(&data_packet)); + assert_yaml_snapshot!(result); + } + #[test] fn it_parses_ipfix_scappy_example() { let hex_template = r#"000a0074583de05700000ecf00000000000200640133001700080004000c0004000500010004000100070002000b000200200002000a0004001000040011000400120004000e000400010004000200040016000400150004000f000400090001000d000100060001003c00010098000800990008"#; diff --git a/src/variable_versions/data_number.rs b/src/variable_versions/data_number.rs index f11d96f..e1019fd 100644 --- a/src/variable_versions/data_number.rs +++ b/src/variable_versions/data_number.rs @@ -1,11 +1,11 @@ use crate::protocol::ProtocolTypes; use byteorder::{BigEndian, WriteBytesExt}; -use nom::bytes::complete::take; -use nom::error::{Error as NomError, ErrorKind}; -use nom::number::complete::{be_i24, be_u128, be_u24, be_u32}; use nom::Err as NomErr; use nom::IResult; +use nom::bytes::complete::take; +use nom::error::{Error as NomError, ErrorKind}; +use nom::number::complete::{be_i24, be_u24, be_u32, be_u128}; use nom_derive::*; use serde::Serialize; @@ -150,7 +150,55 @@ impl DataNumber { DataNumber::I32(n) => n.to_be_bytes().to_vec(), } } +} + +/// Convert into usize, mainly for serialization purposes +impl From for usize { + fn from(val: DataNumber) -> Self { + match val { + DataNumber::U8(i) => i as usize, + DataNumber::I24(i) => i as usize, + DataNumber::U24(i) => i as usize, + DataNumber::U32(i) => i as usize, + DataNumber::I32(i) => i as usize, + DataNumber::U16(i) => i as usize, + DataNumber::U64(i) => i as usize, + DataNumber::U128(i) => i as usize, + } + } +} +/// Holds the post parsed field with its relevant datatype +#[derive(Debug, PartialEq, PartialOrd, Clone, Serialize)] +pub enum FieldValue { + String(String), + DataNumber(DataNumber), + Float64(f64), + Duration(Duration), + Ip4Addr(Ipv4Addr), + Ip6Addr(Ipv6Addr), + MacAddr(String), + Vec(Vec), + ProtocolType(ProtocolTypes), + Unknown, +} + +#[derive(Debug)] +pub enum FieldValueError { + InvalidDataType, +} + +impl FieldValue { + pub fn to_be_bytes(&self) -> Vec { + match self { + FieldValue::String(s) => s.as_bytes().to_vec(), + FieldValue::DataNumber(d) => d.to_be_bytes(), + FieldValue::Float64(f) => f.to_be_bytes().to_vec(), + FieldValue::Duration(d) => (d.as_secs() as u32).to_be_bytes().to_vec(), + FieldValue::Ip4Addr(ip) => ip.octets().to_vec(), + _ => vec![], + } + } pub fn from_field_type( remaining: &[u8], field_type: FieldDataType, @@ -245,55 +293,6 @@ impl DataNumber { } } -/// Convert into usize, mainly for serialization purposes -impl From for usize { - fn from(val: DataNumber) -> Self { - match val { - DataNumber::U8(i) => i as usize, - DataNumber::I24(i) => i as usize, - DataNumber::U24(i) => i as usize, - DataNumber::U32(i) => i as usize, - DataNumber::I32(i) => i as usize, - DataNumber::U16(i) => i as usize, - DataNumber::U64(i) => i as usize, - DataNumber::U128(i) => i as usize, - } - } -} - -/// Holds the post parsed field with its relevant datatype -#[derive(Debug, PartialEq, PartialOrd, Clone, Serialize)] -pub enum FieldValue { - String(String), - DataNumber(DataNumber), - Float64(f64), - Duration(Duration), - Ip4Addr(Ipv4Addr), - Ip6Addr(Ipv6Addr), - MacAddr(String), - Vec(Vec), - ProtocolType(ProtocolTypes), - Unknown, -} - -#[derive(Debug)] -pub enum FieldValueError { - InvalidDataType, -} - -impl FieldValue { - pub fn to_be_bytes(&self) -> Vec { - match self { - FieldValue::String(s) => s.as_bytes().to_vec(), - FieldValue::DataNumber(d) => d.to_be_bytes(), - FieldValue::Float64(f) => f.to_be_bytes().to_vec(), - FieldValue::Duration(d) => (d.as_secs() as u32).to_be_bytes().to_vec(), - FieldValue::Ip4Addr(ip) => ip.octets().to_vec(), - _ => vec![], - } - } -} - /// Helps the parser indent the data type to parse the field as #[derive(Debug, PartialEq, Eq, Clone, Serialize)] pub enum FieldDataType { diff --git a/src/variable_versions/ipfix.rs b/src/variable_versions/ipfix.rs index f185d50..9e11bee 100644 --- a/src/variable_versions/ipfix.rs +++ b/src/variable_versions/ipfix.rs @@ -10,14 +10,15 @@ use super::data_number::*; use crate::variable_versions::ipfix_lookup::*; use crate::{NetflowPacket, NetflowParseError, ParsedNetflow, PartialParse}; +use Nom; +use nom::Err as NomErr; +use nom::IResult; use nom::bytes::complete::take; use nom::error::{Error as NomError, ErrorKind}; use nom::multi::count; -use nom::Err as NomErr; -use nom::IResult; +use nom::number::complete::{be_u8, be_u16}; use nom_derive::*; use serde::Serialize; -use Nom; use std::collections::BTreeMap; @@ -115,22 +116,27 @@ pub struct FlowSetHeader { pub struct FlowSetBody { #[nom( Cond = "id < SET_MIN_RANGE && id != OPTIONS_TEMPLATE_ID", + Verify = "usize::from(template.field_count) == template.fields.len()", + Verify = "template.get_fields().iter().any(|f| f.field_length > 0)", // Save our templates - PostExec = "if let Some(templates) = templates.clone() { parser.templates.insert(templates.template_id, templates); }" + PostExec = "if let Some(template) = template.clone() { parser.templates.insert(template.template_id, template); }" + , )] #[serde(skip_serializing_if = "Option::is_none")] - pub templates: Option