Merge captured PCAPNG into a single file

Fluxzy.Benchmarks/Fluxzy.Benchmarks.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+    <ItemGroup>
+      <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
+    </ItemGroup>
+    <ItemGroup>
+      <ProjectReference Include="..\test\Fluxzy.Tests\Fluxzy.Tests.csproj" />
+    </ItemGroup>
+</Project>

Fluxzy.Benchmarks/Pcap/BlockMergeBenchmark.cs

@@ -0,0 +1,60 @@
+// Copyright 2021 - Haga Rakotoharivelo - https://github.com/haga-rak
+
+using System.Text;
+using BenchmarkDotNet.Attributes;
+using Fluxzy.Core.Pcap.Pcapng;
+using Fluxzy.Tests.UnitTests.Pcap.Merge;
+
+namespace Fluxzy.Benchmarks.Pcap
+{
+    [MemoryDiagnoser(true)]
+    public class BlockMergeBenchmark
+    {
+        private static readonly string Format = "0000";
+        private readonly int _formatLength = Format.Length;
+        private readonly int _concurrentCount = 50;
+        private BlockMerger<byte[]> _merger = null!;
+        private DoNothingWritter _writer = null!;
+        private byte[][] _allLines = null!;
+        private StreamLimiter _streamLimiter = null!;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            var rawInput = MergeTestContentProvider
+                .GetTestData(2000, format: Format);
+
+            _allLines = rawInput.Split(new[] { "\r\n", "\n" },
+                                       StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+                                   .Select(s => Encoding.UTF8.GetBytes(s.Replace(",", string.Empty)))
+                                   .ToArray();
+
+            _merger = new BlockMerger<byte[]>();
+            _writer = new DoNothingWritter();
+        }
+
+        [IterationSetup]
+        public void CreateLimiter()
+        {
+            _streamLimiter = new StreamLimiter(_concurrentCount);
+        }
+
+        [Benchmark]
+        public void MergeBlock()
+        {
+            _merger.Merge(_writer, BlockFactory, _allLines);
+        }
+
+        private IBlockReader BlockFactory(byte[] s)
+        {
+            return new SleepyDummyBlockReader(_streamLimiter, s, _formatLength);
+        }
+    }
+
+    internal class DoNothingWritter : IBlockWriter
+    {
+        public void Write(ref DataBlock content)
+        {
+        }
+    }
+}

Fluxzy.Benchmarks/Program.cs

@@ -0,0 +1,13 @@
+using BenchmarkDotNet.Running;
+using Fluxzy.Benchmarks.Pcap;
+
+namespace Fluxzy.Benchmarks;
+
+internal static class Program
+{
+    public static void Main(string[] args)
+    {
+        // var summary = BenchmarkRunner.Run<Md5VsSha256>();
+        var summary = BenchmarkRunner.Run<BlockMergeBenchmark>();
+    }
+}

codecov.yml

@@ -21,14 +21,16 @@ component_management:
       name: Fluxzy.Core
       paths:
         - src/Fluxzy.Core/**
+    - component_id: Fluxzy.Core.Pcap
+      name: Fluxzy.Core.Pcap
+      paths:
+        - src/Fluxzy.Core.Pcap/**
     - component_id: Fluxzy
       name: Fluxzy
       paths:
         - src/Fluxzy/**
 ignore:
-  - "src/Fluxzy.Core.Pcap/**"
   - "src/Fluxzy.Core.Pcap.Cli/**"
-  - "src/Fluxzy.Core.Pcap/**"
   - "src/Fluxzy.Extensions/**"
   - "src/Fluxzy.Tools.DocGen/**"
 

fluxzy.core.sln

@@ -68,6 +68,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fluxzy.Core.Pcap", "src\Flu
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fluxzy.Core.Pcap.Cli", "src\Fluxzy.Core.Pcap.Cli\Fluxzy.Core.Pcap.Cli.csproj", "{680F1967-AC85-47B7-8420-08290D9F2D03}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fluxzy.Benchmarks", "Fluxzy.Benchmarks\Fluxzy.Benchmarks.csproj", "{40E33F06-755D-4055-9443-2D193A7C8C92}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -130,6 +132,10 @@ Global
 		{680F1967-AC85-47B7-8420-08290D9F2D03}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{680F1967-AC85-47B7-8420-08290D9F2D03}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{680F1967-AC85-47B7-8420-08290D9F2D03}.Release|Any CPU.Build.0 = Release|Any CPU
+		{40E33F06-755D-4055-9443-2D193A7C8C92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{40E33F06-755D-4055-9443-2D193A7C8C92}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{40E33F06-755D-4055-9443-2D193A7C8C92}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{40E33F06-755D-4055-9443-2D193A7C8C92}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -146,6 +152,7 @@ Global
 		{B591162B-5B2B-4E69-9CB6-367C7BB1FC46} = {3A06E788-DEFD-4463-8E7B-48404C45D3B0}
 		{B650DAE5-17D9-459E-840D-A1DE637D27FF} = {AA4E8234-B9D8-4327-8E98-471CE95CEB2C}
 		{680F1967-AC85-47B7-8420-08290D9F2D03} = {AA4E8234-B9D8-4327-8E98-471CE95CEB2C}
+		{40E33F06-755D-4055-9443-2D193A7C8C92} = {CAD5F2E1-227B-43AD-AFF9-1311097038D6}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {82E271CB-7073-4A6C-8FF6-6DA616720D3A}

src/Fluxzy.Core.Pcap.Cli/Fluxzy.Core.Pcap.Cli.csproj

@@ -5,17 +5,18 @@
 		<ImplicitUsings>enable</ImplicitUsings>
 		<AssemblyName>fluxzynetcap</AssemblyName>
 	</PropertyGroup>
-
 	<ItemGroup>
-		<ProjectReference Include="..\Fluxzy.Core.Pcap\Fluxzy.Core.Pcap.csproj" />
+	  <Compile Remove="Logging\**" />
+	  <EmbeddedResource Remove="Logging\**" />
+	  <None Remove="Logging\**" />
 	</ItemGroup>
 
 	<ItemGroup>
-		<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+		<ProjectReference Include="..\Fluxzy.Core.Pcap\Fluxzy.Core.Pcap.csproj" />
 	</ItemGroup>
 
 	<ItemGroup>
-	  <Folder Include="Logging\" />
+		<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
 	</ItemGroup>
 
 </Project>

src/Fluxzy.Core.Pcap/AssemblyInfo.cs

@@ -0,0 +1,5 @@
+using System.ComponentModel;
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("Fluxzy.Tests")]
+[assembly: InternalsVisibleTo("Fluxzy.Benchmarks")]

src/Fluxzy.Core.Pcap/DirectCaptureContext.cs

@@ -217,7 +217,7 @@ private void OnCaptureDeviceOnPacketArrival(object sender, PacketCapture capture
                 return;
 
             try {
-                writer.Write(capture);
+                writer.Write(ref capture);
             }
             catch {
                 // We ignore any write error here to not break the capture thread

src/Fluxzy.Core.Pcap/Messages/SubscribeMessage.cs

@@ -55,6 +55,5 @@ public override int GetHashCode()
         {
             return HashCode.Combine(RemoteAddress, RemotePort, LocalPort, OutFileName);
         }
-
     }
 }

src/Fluxzy.Core.Pcap/PacketKeyBuilder.cs

@@ -1,8 +1,6 @@
 // Copyright © 2022 Haga Rakotoharivelo
 
 using System.Net;
-using System.Net.NetworkInformation;
-using PacketDotNet;
 
 namespace Fluxzy.Core.Pcap
 {
@@ -19,19 +17,6 @@ public static long GetConnectionKey(int localPort, int remotePort, IPAddress rem
             return ((long) portCombination << 32) | (uint) addressHash.GetHashCode();
         }
 
-        public static long GetConnectionKey(TcpPacket tcpPacket, PhysicalAddress physicalAddress)
-        {
-            var ipPacket = (IPPacket) tcpPacket.ParentPacket;
-            var ethernetPacket = (EthernetPacket) ipPacket.ParentPacket;
-
-            var sendPacket = ethernetPacket.SourceHardwareAddress.Equals(physicalAddress);
-            var localPort = sendPacket ? tcpPacket.SourcePort : tcpPacket.DestinationPort;
-            var remotePort = !sendPacket ? tcpPacket.SourcePort : tcpPacket.DestinationPort;
-            var remoteAddress = sendPacket ? ipPacket.DestinationAddress : ipPacket.SourceAddress;
-
-            return GetConnectionKey(localPort, remotePort, remoteAddress);
-        }
-
         public static long GetAuthorityKey(IPAddress address, int port)
         {
             var addressHash = address.GetHashCode(); // TODO : Upgrade to 64 bits hash to cover IPv6

src/Fluxzy.Core.Pcap/Pcapng/ArrayUtilities.cs

@@ -0,0 +1,21 @@
+namespace Fluxzy.Core.Pcap.Pcapng
+{
+    internal static class ArrayUtilities
+    {
+        public static void Reposition<T>(T[] sortedArrayAtIndex1, T firstElement,
+            IComparer<T> comparer)
+        {
+            for (int i = 1; i < sortedArrayAtIndex1.Length; i++)
+            {
+                var current = sortedArrayAtIndex1[i];
+
+                if (comparer.Compare(firstElement, current) <= 0) {
+                    return;
+                }
+
+                sortedArrayAtIndex1[i - 1] = current;
+                sortedArrayAtIndex1[i] = firstElement;
+            }
+        }
+    }
+}

src/Fluxzy.Core.Pcap/Pcapng/Merge/BlockComparer.cs

@@ -0,0 +1,31 @@
+namespace Fluxzy.Core.Pcap.Pcapng.Merge
+{
+    internal class BlockComparer : IComparer<IBlockReader>
+    {
+        public static readonly BlockComparer Instance = new();
+
+        private BlockComparer()
+        {
+
+        }
+
+        public int Compare(IBlockReader? x, IBlockReader? y)
+        {
+            var xTimeStamp = x!.NextTimeStamp;
+
+            if (xTimeStamp == long.MaxValue)
+            {
+                return 1;
+            }
+
+            var yTimeStamp = y!.NextTimeStamp;
+
+            if (yTimeStamp == long.MaxValue)
+            {
+                return -1;
+            }
+
+            return xTimeStamp.CompareTo(yTimeStamp);
+        }
+    }
+}

src/Fluxzy.Core.Pcap/Pcapng/Merge/BlockMerger.cs

@@ -0,0 +1,39 @@
+// Copyright 2021 - Haga Rakotoharivelo - https://github.com/haga-rak
+
+namespace Fluxzy.Core.Pcap.Pcapng.Merge
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <typeparam name="TArgs"></typeparam>
+    internal class BlockMerger<TArgs> where TArgs : notnull
+    {
+        public void Merge(IBlockWriter writer,
+            Func<TArgs, IBlockReader> blockFactory,
+            params TArgs[] items)
+        {
+            if (!items.Any())
+                return;
+
+            var array = items.Select(blockFactory).ToArray();
+
+            Array.Sort(array, BlockComparer.Instance);
+
+            while (true)
+            {
+                if (!array[0].Dequeue(out var block))
+                    break; // No more block to read
+
+                writer.Write(ref block);
+
+                ArrayUtilities.Reposition(array, array[0], BlockComparer.Instance);
+            }
+
+            foreach (var resource in array)
+            {
+                resource.Dispose();
+            }
+        }
+    }
+}

src/Fluxzy.Core.Pcap/Pcapng/Merge/DataBlock.cs

@@ -0,0 +1,17 @@
+// Copyright 2021 - Haga Rakotoharivelo - https://github.com/haga-rak
+
+namespace Fluxzy.Core.Pcap.Pcapng.Merge
+{
+    internal struct DataBlock
+    {
+        public DataBlock(long timeStamp, ReadOnlyMemory<byte> data)
+        {
+            TimeStamp = timeStamp;
+            Data = data;
+        }
+
+        public long TimeStamp { get; }
+
+        public ReadOnlyMemory<byte> Data { get; }
+    }
+}