fix(Can now parse enterprise fields in non options templates for IPFIX.)

mikemiles-dev · Feb 24, 2025 · 876ba47 · 876ba47
1 parent 02285a4
commit 876ba47
Show file tree

Hide file tree

Showing 13 changed files with 270 additions and 22 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "netflow_parser"
 description = "Parser for Netflow Cisco V5, V7, V9, IPFIX"
-version = "0.5.1"
+version = "0.5.2"
 edition = "2021"
 authors = ["michael.mileusnich@gmail.com"]
 license = "MIT OR Apache-2.0"

diff --git a/README.md b/README.md
@@ -150,6 +150,8 @@ if let NetflowPacket::V5(v5) = NetflowParser::default()
 
 Parse the data ('&[u8]' as any other versions.  The parser (NetflowParser) holds onto already parsed templates, so you can just send a header/data flowset combo and it will use the cached templates.)   To see cached templates simply use the parser for the correct version (v9_parser for v9, ipfix_parser for IPFix.)
 
+**IPFIx Note:**  We only parse sequence number and domain id, it is up to you if you wish to validate it.
+
 ```rust
 use netflow_parser::NetflowParser;
 let parser = NetflowParser::default();
@@ -178,3 +180,11 @@ or
 or
 
 ```cargo run --example netflow_udp_listener_tokio```
+
+## Support My Work
+
+If you find my work helpful, consider supporting me!
+
+[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/michaelmileusnich)
+
+[![GitHub Sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/mikemiles-dev)
diff --git a/RELEASES.md b/RELEASES.md
@@ -1,3 +1,6 @@
+# 0.5.2
+* Can now parse enterprise fields in non options templates for IPFIX.
+
 # 0.5.1
 * Reworked NetflowParseError.  Added a Partial Type.
 * Added ability to parse only `allowed_versions`.

diff --git a/SECURITY.md b/SECURITY.md
@@ -4,14 +4,5 @@
 
 | Version | Supported          |
 |---------| ------------------ |
-| 0.5.1   | :white_check_mark: |
-| 0.5.0   | :white_check_mark: |
-| 0.4.9   | :white_check_mark: |
-| 0.4.8   | :white_check_mark: |
-| 0.4.7   | :white_check_mark: |
-| 0.4.6   | :white_check_mark: |
-| 0.4.5   | :white_check_mark: |
-| 0.4.4   | :white_check_mark: |
-| 0.4.3   | :white_check_mark: |
-| 0.4.2   | :white_check_mark: |
+| >0.4.1  | :white_check_mark: |
 | <0.4.1  |   Not Supported    | 
diff --git a/src/lib.rs b/src/lib.rs
@@ -150,6 +150,9 @@
 //! ## V9/IPFix notes:
 //!
 //! Parse the data (`&[u8]` as any other versions.  The parser (NetflowParser) holds onto already parsed templates, so you can just send a header/data flowset combo, and it will use the cached templates.)   To see cached templates simply use the parser for the correct version (v9_parser for v9, ipfix_parser for IPFix.)
+//!
+//! **IPFIx Note:**  We only parse sequence number and domain id, it is up to you if you wish to validate it.
+//!
 //! ```rust
 //! use netflow_parser::NetflowParser;
 //! let parser = NetflowParser::default();
@@ -176,6 +179,14 @@
 //! or
 //!
 //! ```cargo run --example netflow_udp_listener_tokio```
+//!
+//! ## Support My Work
+//!
+//! If you find my work helpful, consider supporting me!
+//!
+//! [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/michaelmileusnich)
+//!
+//! [![GitHub Sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/mikemiles-dev)
 
 pub mod netflow_common;
 pub mod protocol;

diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap
@@ -103,4 +103,3 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
       - 2
       - 3
       - 4
-
diff --git a/...ow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap b/...ow_parser__tests__base_tests__it_parses_ipfix_enterprise_bit_in_non_options_template.snap
@@ -0,0 +1,28 @@
+---
+source: src/tests.rs
+expression: "NetflowParser::default().parse_bytes(&packet)"
+---
+- IPFix:
+    header:
+      version: 10
+      length: 42
+      export_time: 1670052913
+      sequence_number: 0
+      observation_domain_id: 0
+    flowsets:
+      - header:
+          header_id: 2
+          length: 26
+        body:
+          templates:
+            template_id: 260
+            field_count: 2
+            fields:
+              - field_type_number: 32871
+                field_type: Unknown
+                field_length: 65535
+                enterprise_number: 407732327
+              - field_type_number: 65535
+                field_type: Unknown
+                field_length: 0
+                enterprise_number: 407732544
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_scappy_example.snap
@@ -0,0 +1,176 @@
+---
+source: src/tests.rs
+expression: result
+---
+- IPFix:
+    header:
+      version: 10
+      length: 116
+      export_time: 1480450135
+      sequence_number: 3791
+      observation_domain_id: 0
+    flowsets:
+      - header:
+          header_id: 2
+          length: 100
+        body:
+          templates:
+            template_id: 307
+            field_count: 23
+            fields:
+              - field_type_number: 8
+                field_type: SourceIpv4address
+                field_length: 4
+              - field_type_number: 12
+                field_type: DestinationIpv4address
+                field_length: 4
+              - field_type_number: 5
+                field_type: IpClassOfService
+                field_length: 1
+              - field_type_number: 4
+                field_type: ProtocolIdentifier
+                field_length: 1
+              - field_type_number: 7
+                field_type: SourceTransportPort
+                field_length: 2
+              - field_type_number: 11
+                field_type: DestinationTransportPort
+                field_length: 2
+              - field_type_number: 32
+                field_type: IcmpTypeCodeIpv4
+                field_length: 2
+              - field_type_number: 10
+                field_type: IngressInterface
+                field_length: 4
+              - field_type_number: 16
+                field_type: BgpSourceAsNumber
+                field_length: 4
+              - field_type_number: 17
+                field_type: BgpDestinationAsNumber
+                field_length: 4
+              - field_type_number: 18
+                field_type: BgpNextHopIpv4address
+                field_length: 4
+              - field_type_number: 14
+                field_type: EgressInterface
+                field_length: 4
+              - field_type_number: 1
+                field_type: OctetDeltaCount
+                field_length: 4
+              - field_type_number: 2
+                field_type: PacketDeltaCount
+                field_length: 4
+              - field_type_number: 22
+                field_type: FlowStartSysUpTime
+                field_length: 4
+              - field_type_number: 21
+                field_type: FlowEndSysUpTime
+                field_length: 4
+              - field_type_number: 15
+                field_type: IpNextHopIpv4address
+                field_length: 4
+              - field_type_number: 9
+                field_type: SourceIpv4prefixLength
+                field_length: 1
+              - field_type_number: 13
+                field_type: DestinationIpv4prefixLength
+                field_length: 1
+              - field_type_number: 6
+                field_type: TcpControlBits
+                field_length: 1
+              - field_type_number: 60
+                field_type: IpVersion
+                field_length: 1
+              - field_type_number: 152
+                field_type: FlowStartMilliseconds
+                field_length: 8
+              - field_type_number: 153
+                field_type: FlowEndMilliseconds
+                field_length: 8
+- IPFix:
+    header:
+      version: 10
+      length: 96
+      export_time: 1480450137
+      sequence_number: 3812
+      observation_domain_id: 0
+    flowsets:
+      - header:
+          header_id: 307
+          length: 80
+        body:
+          data:
+            data_fields:
+              - 0:
+                  - SourceIpv4address
+                  - Ip4Addr: 70.1.115.1
+                1:
+                  - DestinationIpv4address
+                  - Ip4Addr: 50.0.71.1
+                2:
+                  - IpClassOfService
+                  - DataNumber: 0
+                3:
+                  - ProtocolIdentifier
+                  - DataNumber: 61
+                4:
+                  - SourceTransportPort
+                  - DataNumber: 0
+                5:
+                  - DestinationTransportPort
+                  - DataNumber: 0
+                6:
+                  - IcmpTypeCodeIpv4
+                  - DataNumber: 0
+                7:
+                  - IngressInterface
+                  - DataNumber: 827
+                8:
+                  - BgpSourceAsNumber
+                  - DataNumber: 2
+                9:
+                  - BgpDestinationAsNumber
+                  - DataNumber: 3
+                10:
+                  - BgpNextHopIpv4address
+                  - Ip4Addr: 204.42.110.101
+                11:
+                  - EgressInterface
+                  - DataNumber: 854
+                12:
+                  - OctetDeltaCount
+                  - DataNumber: 1312
+                13:
+                  - PacketDeltaCount
+                  - DataNumber: 9
+                14:
+                  - FlowStartSysUpTime
+                  - DataNumber: 3019441902
+                15:
+                  - FlowEndSysUpTime
+                  - DataNumber: 3019616060
+                16:
+                  - IpNextHopIpv4address
+                  - Ip4Addr: 204.42.110.189
+                17:
+                  - SourceIpv4prefixLength
+                  - DataNumber: 24
+                18:
+                  - DestinationIpv4prefixLength
+                  - DataNumber: 24
+                19:
+                  - TcpControlBits
+                  - DataNumber: 0
+                20:
+                  - IpVersion
+                  - DataNumber: 4
+                21:
+                  - FlowStartMilliseconds
+                  - Duration:
+                      secs: 1480449931
+                      nanos: 519000000
+                22:
+                  - FlowEndMilliseconds
+                  - Duration:
+                      secs: 1480450105
+                      nanos: 677000000
diff --git a/...flow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap b/...flow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap
@@ -59,4 +59,3 @@ expression: parser.parse_bytes(&packet)
       - 0
       - 1
       - 1
-
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_v5_incomplete.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_v5_incomplete.snap
@@ -23,4 +23,3 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
       - 1
       - 1
       - 1
-
diff --git a/src/tests.rs b/src/tests.rs
@@ -405,4 +405,35 @@ mod base_tests {
         ];
         assert_yaml_snapshot!(NetflowParser::default().parse_bytes(&packet));
     }
+
+    #[test]
+    fn it_parses_ipfix_enterprise_bit_in_non_options_template() {
+        let packet = [
+            0, 10, 0, 42, 99, 138, 252, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 1, 4, 0, 2,
+            128, 103, 255, 255, 24, 77, 128, 103, 255, 255, 0, 0, 24, 77, 129, 64, 0, 4, 0, 0,
+            24, 77,
+        ];
+        assert_yaml_snapshot!(NetflowParser::default().parse_bytes(&packet));
+    }
+
+    #[test]
+    fn it_parses_ipfix_scappy_example() {
+        let hex_template = r#"000a0074583de05700000ecf00000000000200640133001700080004000c0004000500010004000100070002000b000200200002000a0004001000040011000400120004000e000400010004000200040016000400150004000f000400090001000d000100060001003c00010098000800990008"#;
+        let packet = hex::decode(hex_template).unwrap();
+        let mut parser = NetflowParser::default();
+        let mut result = parser.parse_bytes(&packet);
+        let hex_data = r#"000a0060583de05900000ee400000000013300504601730132004701003d0000000000000000033b0000000200000003cc2a6e65000003560000052000000009b3f906eeb3fbaf3ccc2a6ebd1818000400000158b1b138ff00000158b1b3e14d"#;
+        let packet = hex::decode(hex_data).unwrap();
+        result.append(&mut parser.parse_bytes(&packet));
+        assert_yaml_snapshot!(result);
+    }
+
+    #[test]
+    fn it_parses_ipfix_scappy_example_options_template() {
+        let hex_template = r#"000a0028583de05700000ecf00000000000300180134000300010005000200240002002500020000"#;
+        let packet = hex::decode(hex_template).unwrap();
+        let mut parser = NetflowParser::default();
+        let result = parser.parse_bytes(&packet);
+        assert_yaml_snapshot!(result);
+    }
 }
diff --git a/src/variable_versions/ipfix.rs b/src/variable_versions/ipfix.rs
@@ -21,7 +21,6 @@ use Nom;
 
 use std::collections::BTreeMap;
 
-const TEMPLATE_ID: u16 = 2;
 const OPTIONS_TEMPLATE_ID: u16 = 3;
 const SET_MIN_RANGE: u16 = 255;
 
@@ -115,7 +114,7 @@ pub struct FlowSetHeader {
 #[nom(ExtraArgs(parser: &mut IPFixParser, id: u16, length: u16))]
 pub struct FlowSetBody {
     #[nom(
-        Cond = "id == TEMPLATE_ID",
+        Cond = "id < SET_MIN_RANGE && id != OPTIONS_TEMPLATE_ID",
         // Save our templates
         PostExec = "if let Some(templates) = templates.clone() { parser.templates.insert(templates.template_id, templates); }"
     )]
@@ -171,8 +170,8 @@ pub struct OptionsTemplate {
     pub field_count: u16,
     pub scope_field_count: u16,
     #[nom(
-        PreExec = "let combined_count = scope_field_count as usize + 
-                       field_count.checked_sub(scope_field_count).unwrap_or(field_count) as usize;",
+        PreExec = "let combined_count = scope_field_count.saturating_add(
+                       field_count.checked_sub(scope_field_count).unwrap_or(field_count)) as usize;",
         Parse = "count(|i| TemplateField::parse(i, true), combined_count)",
         PostExec = "let options_remaining = set_length.checked_sub(field_count * 4).unwrap_or(set_length) > 0;"
     )]
@@ -212,7 +211,7 @@ pub struct TemplateField {
     pub field_type: IPFixField,
     pub field_length: u16,
     #[nom(
-        Cond = "options_template && field_type_number > 32767",
+        Cond = "field_type_number > 32767",
         PostExec = "let field_type_number = if options_template {
                       field_type_number.overflowing_sub(32768).0
                     } else { field_type_number };",
@@ -328,7 +327,7 @@ fn parse_field<'a>(
 
     if has_enterprise_number {
         // Simplified parsing when `enterprise_number` is present
-        parse_enterprise_field(i)
+        parse_enterprise_field(i, template_field.field_length)
     } else {
         // Parse field based on its type and length
         DataNumber::from_field_type(
@@ -339,8 +338,8 @@ fn parse_field<'a>(
     }
 }
 
-fn parse_enterprise_field(i: &[u8]) -> IResult<&[u8], FieldValue> {
-    let (remaining, data_number) = DataNumber::parse(i, 4, false)?;
+fn parse_enterprise_field(i: &[u8], length: u16) -> IResult<&[u8], FieldValue> {
+    let (remaining, data_number) = DataNumber::parse(i, length, false)?;
     Ok((remaining, FieldValue::DataNumber(data_number)))
 }
 

diff --git a/src/variable_versions/v9_lookup.rs b/src/variable_versions/v9_lookup.rs
@@ -1,3 +1,5 @@
+//! See: <https://www.ibm.com/docs/en/npi/1.3.0?topic=versions-v9-field-type-definitions>
+
 use super::data_number::*;
 
 use nom_derive::*;
Original file line number	Diff line number	Diff line change
Expand Up		@@ -103,4 +103,3 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
		- 2
		- 3
		- 4
Original file line number	Diff line number	Diff line change
Expand Up		@@ -59,4 +59,3 @@ expression: parser.parse_bytes(&packet)
		- 0
		- 1
		- 1
Original file line number	Diff line number	Diff line change
Expand Up		@@ -23,4 +23,3 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
		- 1
		- 1
		- 1