From bb3af489ccef92902a7c2168e9afdf5b9cc647d4 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 3 Mar 2020 23:57:56 +0100 Subject: [PATCH 1/2] implemented support for ja3 server hashes --- .gitignore | 1 + README.md | 201 +++++++++++++++++---- cmd/main.go | 9 +- csv.go | 24 ++- go.mod | 5 +- go.sum | 16 +- gopacket.go | 69 +++++-- ja3.go | 8 +- ja3_output_testpcap.json | 378 +++++++++++++++++++++++++++++++++++++++ ja3_test.go | 30 ++-- ja3s.go | 168 +++++++++++++++++ ja3s_test.go | 188 +++++++++++++++++++ ja3s_test.log | 71 ++++++++ json.go | 66 ++++--- live.go | 10 +- utils.go | 4 +- 16 files changed, 1145 insertions(+), 103 deletions(-) create mode 100644 .gitignore create mode 100644 ja3s.go create mode 100644 ja3s_test.go create mode 100644 ja3s_test.log diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..62c8935 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index e9828a2..840d117 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ![Windows](https://img.shields.io/badge/Supports-Windows-green.svg) [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/dreadl0ck/ja3) -JA3 is a technique developed by Salesforce, to fingerprint the TLS client hello. +JA3 is a technique developed by Salesforce, to fingerprint the TLS client and server hellos. The official python implementation can be found here: https://github.com/salesforce/ja3 @@ -17,6 +17,8 @@ More details can be found in their blog post: https://engineering.salesforce.com This package provides a pure golang implementation, that was created to operate on **ClientHello** instances from the **bradleyfalzon/tlsx** package. For this to work, the package needed to be slightly modified to store the values for extensions in the order they are encountered. It was forked to **github.com/dreadl0ck/tlsx**. +By now, the tlsx fork has been improved to support extracting the server hello message, +besides unit tests, benchmarks and a faster unmarshaling interface for use with Ja3 fingerprinting. This implementation uses gopacket for packet decoding. Assembly of the ja3 bare was previously implemented with the golang 1.10 **strings.Builder** type, @@ -28,7 +30,7 @@ Thanks to @guigzzz for his pull request on further reducing allocations! This package exports the following API: -Live Capture: +Live Capture from interface, outputs CSV: ```go func ReadInterfaceCSV(iface string, out io.Writer, separator string) @@ -43,16 +45,32 @@ func ReadFileJSON(file string, out io.Writer) func ReadFileCSV(file string, out io.Writer, separator string) ``` +Read file and only print Ja3s values, mimics the python implementation from salesforce: + +```go +func ReadFileJa3s(file string, out io.Writer) +``` + Using gopacket.Packet: ```go func BarePacket(p gopacket.Packet) []byte ``` ```go +func BarePacketJa3s(p gopacket.Packet) []byte +``` +```go func DigestPacket(p gopacket.Packet) [md5.Size]byte ``` ```go -func DigestHexPacket(p gopacket.Packet) string +func DigestPacketJa3s(p gopacket.Packet) [md5.Size]byte +``` + +```go +func DigestHexPacket(p gopacket.Packet) string +``` +```go +func DigestHexPacketJa3s(p gopacket.Packet) string ``` Using tlsx.ClientHello: @@ -61,11 +79,20 @@ Using tlsx.ClientHello: func Bare(hello *tlsx.ClientHello) []byte ``` ```go +func BareJa3s(hello *tlsx.ServerHello) []byte +``` +```go func Digest(hello *tlsx.ClientHello) [md5.Size]byte ``` ```go +func DigestJa3s(hello *tlsx.ServerHello) [md5.Size]byte +``` +```go func DigestHex(hello *tlsx.ClientHello) string ``` +```go +func DigestHexJa3s(hello *tlsx.ServerHello) string +``` ## Commandline Tool @@ -76,33 +103,46 @@ Printing output as CSV, tab separated values or with a custom separator is also $ goja3 -h Usage of goja3: - -csv - print as CSV - -json - print as JSON array (default true) - -read string - read PCAP file - -separator string - set a custom separator (default ",") - -tsv - print as TAB separated values + -csv + print as CSV + -debug + toggle debug mode + -iface string + specify network interface to read packets from + -ja3s + dump ja3s only + -json + print as JSON array (default true) + -read string + read PCAP file + -separator string + set a custom separator (default ",") + -tsv + print as TAB separated values Benchmark of the python reference implementation VS this one, -on a 109 MB PCAP dumpfile (DEF CON 23 ICS Village.pcap): +on a 109 MB PCAP dumpfile (DEF CON 23 ICS Village.pcap). +This dump file is interesting for comparison because it contains handshakes without extensions set, +as well as TLS traffic over non standard ports. - Python implementation: +Python implementation: - $ time ja3 pcaps/DEF\ CON\ 23\ ICS\ Village.pcap - real 0m19.456s - user 0m18.424s - sys 0m0.345s + # https://github.com/salesforce/ja3 + $ time ja3 -a DEF\ CON\ 23\ ICS\ Village.pcap &> /dev/null + + real 0m29.031s + user 0m28.803s + sys 0m0.141s - Go implementation: +Go implementation: - $ time goja3 -read pcaps/DEF\ CON\ 23\ ICS\ Village.pcap + # github.com/dreadl0ck/ja3 + # disable server fingerprint generation to mimic the behavior of the reference implementation + $ time goja3 -ja3s=false -json -read DEF\ CON\ 23\ ICS\ Village.pcap &> /dev/null + real 0m0.996s - user 0m1.800s - sys 0m0.119s + user 0m2.245s + sys 0m0.166s Output: @@ -113,6 +153,8 @@ Output: "destination_port": 443, "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "ja3s": "", + "ja3s_digest": "", "source_ip": "192.168.0.112", "source_port": 59935, "timestamp": "1438978270.855224" @@ -122,6 +164,8 @@ Output: "destination_port": 443, "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "ja3s": "", + "ja3s_digest": "", "source_ip": "192.168.0.112", "source_port": 42455, "timestamp": "1438978270.855228" @@ -158,6 +202,29 @@ JA3 is also an excellent detection mechanism in locked-down environments where o For more details on what you can see and do with JA3 and JA3S, please see this Shmoocon 2018 talk: https://youtu.be/oprPu7UIEuk?t=6m44s +## JA3S Details + +JA3S is JA3 for the Server side of the SSL/TLS communication and fingerprints how servers respond to particular clients. + +JA3S uses the following field order: +``` +SSLVersion,Cipher,SSLExtension +``` +With JA3S it is possible to fingerprint the entire cryptographic negotiation between client and it's server by combining JA3 + JA3S. That is because servers will respond to different clients differently but will always respond to the same client the same. + +For the Trickbot example: +``` +JA3 = 6734f37431670b3ab4292b8f60f29984 ( Fingerprint of Trickbot ) +JA3S = 623de93db17d313345d7ea481e7443cf ( Fingerprint of Command and Control Server Response ) +``` +For the Emotet example: +``` +JA3 = 4d7a28d6f2263ed61de88ca66eb011e3 ( Fingerprint of Emotet ) +JA3S = 80b3a14bccc8598a1f3bbe83e71f735f ( Fingerprint of Command and Control Server Response ) +``` + +In these malware examples, the command and control server always responds to the malware client in exactly the same way, it does not deviate. So even though the traffic is encrypted and one may not know the command and control server's IPs or domains as they are constantly changing, we can still identify, with reasonable confidence, the malicious communication by fingerprinting the TLS negotiation between client and server. Again, please be aware that these are examples, not indicative of all versions ever, and are intended to illustrate what is possible. + ## Benchmarks & Tests Run the benchmarks and the correctness test with: @@ -166,15 +233,87 @@ Run the benchmarks and the correctness test with: === RUN TestDigestHexCorrect --- PASS: TestDigestHexCorrect (0.00s) === RUN TestDigestHexComparePcap - --- PASS: TestDigestHexComparePcap (0.13s) + --- PASS: TestDigestHexComparePcap (0.10s) + === RUN TestDigestHexJa3sCorrect + --- PASS: TestDigestHexJa3sCorrect (0.00s) + === RUN TestDigestHexJa3sComparePcap + --- PASS: TestDigestHexJa3sComparePcap (0.10s) goos: darwin goarch: amd64 pkg: github.com/dreadl0ck/ja3 - BenchmarkDigestHexPacket-12 1000000 1814 ns/op 464 B/op 13 allocs/op - BenchmarkDigestPacket-12 1000000 1768 ns/op 400 B/op 11 allocs/op - BenchmarkBarePacket-12 1000000 1548 ns/op 400 B/op 11 allocs/op - BenchmarkDigestHex-12 1000000 1430 ns/op 256 B/op 3 allocs/op - BenchmarkDigest-12 1000000 1368 ns/op 192 B/op 1 allocs/op - BenchmarkBare-12 1000000 1128 ns/op 192 B/op 1 allocs/op + BenchmarkDigestHexPacket + BenchmarkDigestHexPacket-12 781578 1499 ns/op 352 B/op 9 allocs/op + BenchmarkDigestPacket + BenchmarkDigestPacket-12 827286 1406 ns/op 288 B/op 7 allocs/op + BenchmarkBarePacket + BenchmarkBarePacket-12 1000000 1212 ns/op 288 B/op 7 allocs/op + BenchmarkDigestHex + BenchmarkDigestHex-12 895094 1278 ns/op 256 B/op 3 allocs/op + BenchmarkDigest + BenchmarkDigest-12 996800 1318 ns/op 192 B/op 1 allocs/op + BenchmarkBare + BenchmarkBare-12 1101476 1188 ns/op 192 B/op 1 allocs/op + BenchmarkDigestHexPacketJa3s + BenchmarkDigestHexPacketJa3s-12 1654360 619 ns/op 120 B/op 4 allocs/op + BenchmarkDigestPacketJa3s + BenchmarkDigestPacketJa3s-12 2328241 513 ns/op 56 B/op 2 allocs/op + BenchmarkBarePacketJa3s + BenchmarkBarePacketJa3s-12 3294260 387 ns/op 56 B/op 2 allocs/op + BenchmarkDigestHexJa3s + BenchmarkDigestHexJa3s-12 2476090 464 ns/op 112 B/op 3 allocs/op + BenchmarkDigestJa3s + BenchmarkDigestJa3s-12 3321294 357 ns/op 48 B/op 1 allocs/op + BenchmarkBareJa3s + BenchmarkBareJa3s-12 4964508 238 ns/op 48 B/op 1 allocs/op PASS - ok github.com/dreadl0ck/ja3 9.314s \ No newline at end of file + ok github.com/dreadl0ck/ja3 18.414s + +Performance comparison with other implementations: + + # https://github.com/open-ch/ja3 + $ time ja3exporter -pcap DEF\ CON\ 23\ ICS\ Village.pcap &> /dev/null + + real 0m0.912s + user 0m1.979s + sys 0m0.142s + + # github.com/dreadl0ck/ja3 + # disable server fingerprint generation to mimic the behavior of the other implementations + $ time goja3 -ja3s=false -json -read DEF\ CON\ 23\ ICS\ Village.pcap &> /dev/null + + real 0m0.996s + user 0m2.245s + sys 0m0.166s + + # https://github.com/salesforce/ja3 + $ time ja3 -a DEF\ CON\ 23\ ICS\ Village.pcap &> /dev/null + + real 0m29.031s + user 0m28.803s + sys 0m0.141s + +Number of records detected: + + # https://github.com/open-ch/ja3 + $ time ja3exporter -pcap DEF\ CON\ 23\ ICS\ Village.pcap | wc -l + 138 + + # github.com/dreadl0ck/ja3 + $ goja3 -ja3s=false -json -read DEF\ CON\ 23\ ICS\ Village.pcap | jq '. | length' + 138 + + # https://github.com/salesforce/ja3 + $ time ja3 -a DEF\ CON\ 23\ ICS\ Village.pcap | jq '. | length' + 138 + + # https://github.com/salesforce/ja3 + # without -a flag, only tls traffic over port 443 will be extracted + $ time ja3 DEF\ CON\ 23\ ICS\ Village.pcap | jq '. | length' + 96 + +## Notes + +The ja3_output_testpcap.json file for output comparison in the unit tests +was generated with the python reference implementation using: + + ja3 -a test.pcap > ja3_output_testpcap.json \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index 73f0cdc..192c3ff 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -30,6 +30,8 @@ var ( flagInput = flag.String("read", "", "read PCAP file") flagDebug = flag.Bool("debug", false, "toggle debug mode") flagInterface = flag.String("iface", "", "specify network interface to read packets from") + flagJa3S = flag.Bool("ja3s", true, "include ja3 server hashes (ja3s)") + flagOnlyJa3S = flag.Bool("ja3s-only", false, "dump ja3s only") ) func main() { @@ -48,6 +50,11 @@ func main() { os.Exit(1) } + if *flagOnlyJa3S { + ja3.ReadFileJa3s(*flagInput, os.Stdout) + return + } + if *flagTSV { ja3.ReadFileCSV(*flagInput, os.Stdout, "\t") return @@ -59,6 +66,6 @@ func main() { } if *flagJSON { - ja3.ReadFileJSON(*flagInput, os.Stdout) + ja3.ReadFileJSON(*flagInput, os.Stdout, *flagJa3S) } } diff --git a/csv.go b/csv.go index 6078b4f..f0436cb 100644 --- a/csv.go +++ b/csv.go @@ -19,8 +19,8 @@ import ( "io" "strings" - "github.com/dreadl0ck/gopacket" - "github.com/dreadl0ck/gopacket/layers" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" ) // ReadFileCSV reads the PCAP file at the given path @@ -33,7 +33,7 @@ func ReadFileCSV(file string, out io.Writer, separator string) { } defer f.Close() - columns := []string{"timestamp", "source_ip", "source_port", "destination_ip", "destination_port", "ja3_digest"} + columns := []string{"timestamp", "source_ip", "source_port", "destination_ip", "destination_port", "ja3_digest", "ja3s_digest"} _, err = out.Write([]byte(strings.Join(columns, separator) + "\n")) if err != nil { panic(err) @@ -45,7 +45,7 @@ func ReadFileCSV(file string, out io.Writer, separator string) { data, ci, err := r.ReadPacketData() if err == io.EOF { if Debug { - fmt.Println(count, "fingeprints.") + fmt.Println(count, "fingerprints.") } return } else if err != nil { @@ -57,8 +57,14 @@ func ReadFileCSV(file string, out io.Writer, separator string) { p = gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Lazy) // get JA3 if possible digest = DigestHexPacket(p) + isServer bool ) + if digest == "" { + digest = DigestHexPacketJa3s(p) + isServer = true + } + // check if we got a result if digest != "" { @@ -88,7 +94,15 @@ func ReadFileCSV(file string, out io.Writer, separator string) { b.WriteString(separator) b.WriteString(tl.TransportFlow().Dst().String()) b.WriteString(separator) - b.WriteString(digest) + if isServer { + b.WriteString("") + b.WriteString(separator) + b.WriteString(digest) + } else { // client + b.WriteString(digest) + b.WriteString(separator) + b.WriteString("") + } b.WriteString("\n") _, err := out.Write([]byte(b.String())) diff --git a/go.mod b/go.mod index 4c6f5fe..5ace0a4 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module github.com/dreadl0ck/ja3 require ( - github.com/dreadl0ck/gopacket v1.1.16-0.20200112124841-257d0538ed04 - github.com/dreadl0ck/tlsx v0.0.0-20181116232845-5334a78b2ee2 + github.com/dreadl0ck/tlsx v0.0.0-20200303221230-8cb859306e07 + github.com/google/gopacket v1.1.17 + golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 7a55510..ac09430 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,19 @@ -github.com/dreadl0ck/gopacket v1.1.16-0.20200112124841-257d0538ed04 h1:sZx5SOCbrOfLXXvaUXscwc8ZBYGyNb3B3x99KKJWrxw= -github.com/dreadl0ck/gopacket v1.1.16-0.20200112124841-257d0538ed04/go.mod h1:KBFGDQYW2zq081ooQxvs0vpaCqRHeeAF517Olq7oAI8= -github.com/dreadl0ck/tlsx v0.0.0-20181116232845-5334a78b2ee2 h1:7CANO0lLxbFsiuw+/CC0sVC99NtH1mqJ09Qtw/QWxO4= -github.com/dreadl0ck/tlsx v0.0.0-20181116232845-5334a78b2ee2/go.mod h1:CWMT29xEORO82gFDJTlKbFgp16ol79EKBR4gedRZKlE= +github.com/dreadl0ck/tlsx v0.0.0-20200303221230-8cb859306e07 h1:ZPbZ1CELNcwKr0D/MHZK0oPRrb4hH28AKldTWknV6lg= +github.com/dreadl0ck/tlsx v0.0.0-20200303221230-8cb859306e07/go.mod h1:amAb73WEEgPHWniMfwro6UpN6St3e5ypgq2tXM89IOo= +github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= +github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 h1:1Fzlr8kkDLQwqMP8GxrhptBLqZG/EDpiATneiZHY998= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/gopacket.go b/gopacket.go index 5658118..2dfbc02 100644 --- a/gopacket.go +++ b/gopacket.go @@ -20,8 +20,8 @@ import ( "fmt" "github.com/dreadl0ck/tlsx" - "github.com/dreadl0ck/gopacket" - "github.com/dreadl0ck/gopacket/layers" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" ) // DigestPacket returns the Ja3 digest @@ -31,6 +31,13 @@ func DigestPacket(p gopacket.Packet) [md5.Size]byte { return md5.Sum(BarePacket(p)) } +// DigestPacket returns the Ja3s digest +// for a packet carrying a TLS Server Hello +// or an empty byte slice +func DigestPacketJa3s(p gopacket.Packet) [md5.Size]byte { + return md5.Sum(BarePacketJa3s(p)) +} + // DigestHexPacket returns the hex string for the packet // for a packet carrying a TLS Client Hello func DigestHexPacket(p gopacket.Packet) string { @@ -44,6 +51,19 @@ func DigestHexPacket(p gopacket.Packet) string { return hex.EncodeToString(sum[:]) } +// DigestHexPacket returns the hex string for the packet +// for a packet carrying a TLS Client Hello +func DigestHexPacketJa3s(p gopacket.Packet) string { + + bare := BarePacketJa3s(p) + if len(bare) == 0 { + return "" + } + + sum := md5.Sum(bare) + return hex.EncodeToString(sum[:]) +} + // BarePacket returns the Ja3 digest if the supplied packet contains a TLS client hello // otherwise returns an empty string func BarePacket(p gopacket.Packet) []byte { @@ -58,27 +78,56 @@ func BarePacket(p gopacket.Packet) []byte { } else if tcp.RST { // Unexpected packet } else { - - // invalid length, this is not handled by the tsx package - // bail out otherwise there will be a panic - if len(tcp.LayerPayload()) < 6 { + // data packet + var ( + hello = tlsx.ClientHelloBasic{} + err = hello.Unmarshal(tcp.LayerPayload()) + ) + if err != nil { + if Debug { + fmt.Println(err) + //fmt.Println(p.Dump()) + } return []byte{} } + // return JA3 bare + return Bare(&hello) + } + } + } + return []byte{} +} + +// BarePacket returns the Ja3 digest if the supplied packet contains a TLS client hello +// otherwise returns an empty string +func BarePacketJa3s(p gopacket.Packet) []byte { + if tl := p.TransportLayer(); tl != nil { + if tcp, ok := tl.(*layers.TCP); ok { + if tcp.SYN { + // Connection setup + } else if tcp.FIN { + // Connection teardown + } else if tcp.ACK && len(tcp.LayerPayload()) == 0 { + // Acknowledgement packet + } else if tcp.RST { + // Unexpected packet + } else { // data packet var ( - hello = tlsx.ClientHello{} - err = hello.Unmarshall(tcp.LayerPayload()) + hello = tlsx.ServerHelloBasic{} + err = hello.Unmarshal(tcp.LayerPayload()) ) if err != nil { if Debug { - fmt.Println(err, p.Dump()) + fmt.Println(err, p.NetworkLayer().NetworkFlow(), p.TransportLayer().TransportFlow()) + //fmt.Println(p.Dump()) } return []byte{} } // return JA3 bare - return Bare(&hello) + return BareJa3s(&hello) } } } diff --git a/ja3.go b/ja3.go index 3abcd76..aa5f203 100644 --- a/ja3.go +++ b/ja3.go @@ -49,16 +49,16 @@ func BareToDigestHex(bare []byte) string { } // Digest returns only the digest md5. -func Digest(hello *tlsx.ClientHello) [md5.Size]byte { +func Digest(hello *tlsx.ClientHelloBasic) [md5.Size]byte { return md5.Sum(Bare(hello)) } // DigestHex produce md5 hash from bare string. -func DigestHex(hello *tlsx.ClientHello) string { +func DigestHex(hello *tlsx.ClientHelloBasic) string { return BareToDigestHex(Bare(hello)) } -// Bare returns the JA3 bare string for a given tlsx.ClientHello instance +// Bare returns the JA3 bare string for a given tlsx.ClientHelloBasic instance // JA3 is a technique developed by Salesforce, to fingerprint TLS Client Hellos. // the official python implementation can be found here: https://github.com/salesforce/ja3 // JA3 gathers the decimal values of the bytes for the following fields; SSL Version, Accepted Ciphers, List of Extensions, Elliptic Curves, and Elliptic Curve Formats. @@ -72,7 +72,7 @@ func DigestHex(hello *tlsx.ClientHello) string { // 769,4–5–10–9–100–98–3–6–19–18–99,,, // These strings are then MD5 hashed to produce an easily consumable and shareable 32 character fingerprint. // This is the JA3 SSL Client Fingerprint returned by this function. -func Bare(hello *tlsx.ClientHello) []byte { +func Bare(hello *tlsx.ClientHelloBasic) []byte { // TODO: refactor into struct with inbuilt buffer to reduce allocations to ~ zero // i.e. only realloc if previously allocated buffer is too small for current packet diff --git a/ja3_output_testpcap.json b/ja3_output_testpcap.json index 9e2fb9a..5652c73 100644 --- a/ja3_output_testpcap.json +++ b/ja3_output_testpcap.json @@ -89,6 +89,42 @@ "source_port": 42577, "timestamp": 1438975412.19272 }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40052, + "timestamp": 1438975415.681516 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40053, + "timestamp": 1438975415.695585 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40054, + "timestamp": 1438975415.784149 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40055, + "timestamp": 1438975415.784384 + }, { "destination_ip": "192.168.0.30", "destination_port": 443, @@ -134,6 +170,33 @@ "source_port": 42942, "timestamp": 1438975418.252502 }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40062, + "timestamp": 1438975418.728398 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40063, + "timestamp": 1438975418.735892 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40066, + "timestamp": 1438975418.912281 + }, { "destination_ip": "192.168.0.30", "destination_port": 443, @@ -143,6 +206,321 @@ "source_port": 42949, "timestamp": 1438975418.953542 }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40069, + "timestamp": 1438975418.970933 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40071, + "timestamp": 1438975419.160157 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40072, + "timestamp": 1438975419.167724 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40074, + "timestamp": 1438975419.363007 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40075, + "timestamp": 1438975419.370242 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40077, + "timestamp": 1438975419.563502 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40078, + "timestamp": 1438975419.569185 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40080, + "timestamp": 1438975419.763242 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40081, + "timestamp": 1438975419.769096 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40083, + "timestamp": 1438975419.965889 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40084, + "timestamp": 1438975419.972691 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40086, + "timestamp": 1438975420.167612 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40087, + "timestamp": 1438975420.174799 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40089, + "timestamp": 1438975420.370286 + }, + { + "destination_ip": "192.168.0.21", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 40090, + "timestamp": 1438975420.376324 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33325, + "timestamp": 1438975627.692959 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33329, + "timestamp": 1438975630.680471 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33330, + "timestamp": 1438975630.730156 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33331, + "timestamp": 1438975630.732841 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33333, + "timestamp": 1438975630.929733 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33334, + "timestamp": 1438975630.932921 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33336, + "timestamp": 1438975631.130647 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33337, + "timestamp": 1438975631.134219 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33339, + "timestamp": 1438975631.332171 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33340, + "timestamp": 1438975631.334887 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33342, + "timestamp": 1438975631.533235 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33343, + "timestamp": 1438975631.537026 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33345, + "timestamp": 1438975631.733574 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33346, + "timestamp": 1438975631.735561 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33348, + "timestamp": 1438975631.935475 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33349, + "timestamp": 1438975631.93968 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33351, + "timestamp": 1438975632.137021 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33352, + "timestamp": 1438975632.14068 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33354, + "timestamp": 1438975632.338641 + }, + { + "destination_ip": "192.168.0.32", + "destination_port": 80, + "ja3": "771,5-4-2-8-20-3-1-21-6-22-23-51-57-25-58-26-24-53-9-10-27-47-52-49168-49158-49173-49163-49153-59-49200-49196-49192-49188-49172-49162-163-159-107-106-56-136-135-49177-167-109-137-49202-49198-49194-49190-49167-49157-157-61-132-49170-49160-19-49175-49165-49155-49199-49195-49191-49187-49171-49161-162-158-103-64-50-154-153-69-68-49176-166-108-155-70-49201-49197-49193-49189-49166-49156-156-60-150-65-49169-49159-49174-49164-49154-18-17-255,11-10-35-13-15,14-13-25-11-12-24-9-10-22-23-8-6-7-20-21-4-5-18-19-1-2-3-15-16-17,0-1-2", + "ja3_digest": "e3bb8f1cd407701c585e7a84e96c24e1", + "source_ip": "192.168.0.109", + "source_port": 33355, + "timestamp": 1438975632.342897 + }, { "destination_ip": "192.168.0.40", "destination_port": 443, diff --git a/ja3_test.go b/ja3_test.go index 928b37f..caf0e57 100644 --- a/ja3_test.go +++ b/ja3_test.go @@ -15,13 +15,13 @@ package ja3 import ( - "testing" "bytes" - "io/ioutil" "encoding/json" "github.com/dreadl0ck/tlsx" - "github.com/dreadl0ck/gopacket" - "github.com/dreadl0ck/gopacket/layers" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "io/ioutil" + "testing" ) var tlsPacket = []byte{ @@ -43,7 +43,7 @@ var tlsPacket = []byte{ 0x03, 0x02, 0x02, } -func getHello(p gopacket.Packet, b *testing.B) (hello *tlsx.ClientHello) { +func getHello(p gopacket.Packet, b *testing.B) (hello *tlsx.ClientHelloBasic) { if tl := p.TransportLayer(); tl != nil { if tcp, ok := tl.(*layers.TCP); ok { if tcp.SYN { @@ -55,16 +55,8 @@ func getHello(p gopacket.Packet, b *testing.B) (hello *tlsx.ClientHello) { } else if tcp.RST { b.Fatal("Unexpected packet") } else { - - // invalid length, this is not handled by the tsx package - // bail out otherwise there will be a panic - if len(tcp.LayerPayload()) < 6 { - b.Fatal("invalid packet") - } - - hello = &tlsx.ClientHello{} - - err := hello.Unmarshall(tcp.LayerPayload()) + hello = &tlsx.ClientHelloBasic{} + err := hello.Unmarshal(tcp.LayerPayload()) if err != nil { b.Fatal("invalid packet", err) } @@ -112,9 +104,9 @@ func TestDigestHexComparePcap(t *testing.T) { ) // read test.pcap and generate fingerprints - ReadFileJSON("test.pcap", &b) + ReadFileJSON("test.pcap", &b, false) - err = json.Unmarshal(data, &records) + err = json.Unmarshal(b.Bytes(), &records) if err != nil { t.Fatal(err) } @@ -131,6 +123,10 @@ func TestDigestHexComparePcap(t *testing.T) { if r.JA3Digest != records[i].JA3Digest { t.Fatal("r.JA3Digest != records[i].JA3Digest: ", r.JA3Digest, " != ", records[i].JA3Digest) } + // compare reference digest to the one produced by goja3 + if r.JA3SDigest != records[i].JA3SDigest { + t.Fatal("r.JA3SDigest != records[i].JAS3Digest: ", r.JA3SDigest, " != ", records[i].JA3SDigest) + } } } diff --git a/ja3s.go b/ja3s.go new file mode 100644 index 0000000..976bc9c --- /dev/null +++ b/ja3s.go @@ -0,0 +1,168 @@ +/* + * JA3 - TLS Client Hello Hash + * Copyright (c) 2017, Salesforce.com, Inc. + * this code was created by Philipp Mieden + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package ja3 + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "io" + "strconv" + "strings" + + "github.com/dreadl0ck/tlsx" +) + +// ReadFileJa3s reads the PCAP file at the given path +// and prints out all packets containing JA3S digests to the supplied io.Writer +func ReadFileJa3s(file string, out io.Writer) { + + r, f, err := openPcap(file) + if err != nil { + panic(err) + } + defer f.Close() + + count := 0 + for { + // read packet data + data, _, err := r.ReadPacketData() + if err == io.EOF { + if Debug { + fmt.Println(count, "fingerprints.") + } + return + } else if err != nil { + panic(err) + } + + var ( + // create gopacket + p = gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Lazy) + // get JA3 if possible + digest = DigestHexPacketJa3s(p) + ) + + // check if we got a result + if digest != "" { + + count++ + + var ( + b strings.Builder + nl = p.NetworkLayer() + tl = p.TransportLayer() + ) + + // got an a digest but no transport or network layer + if tl == nil || nl == nil { + if Debug { + fmt.Println("got a nil layer: ", nl, tl, p.Dump(), digest) + } + continue + } + + b.WriteString("[") + b.WriteString(nl.NetworkFlow().Dst().String()) + b.WriteString(":") + b.WriteString(tl.TransportFlow().Dst().String()) + b.WriteString("] JA3S: ") + b.WriteString(string(BarePacketJa3s(p))) + b.WriteString(" --> ") + b.WriteString(digest) + b.WriteString("\n") + + _, err := out.Write([]byte(b.String())) + if err != nil { + panic(err) + } + } + } +} + +// BareToDigestHex converts a bare []byte to a hex string. +func BareToDigestHexJa3s(bare []byte) string { + sum := md5.Sum(bare) + return hex.EncodeToString(sum[:]) +} + +// Digest returns only the digest md5. +func DigestJa3s(hello *tlsx.ServerHelloBasic) [md5.Size]byte { + return md5.Sum(BareJa3s(hello)) +} + +// DigestHex produce md5 hash from bare string. +func DigestHexJa3s(hello *tlsx.ServerHelloBasic) string { + return BareToDigestHex(BareJa3s(hello)) +} + +// BareJa3s returns the JA3S bare string for a given tlsx.ServerHelloBasic instance +// JA3S is JA3 for the Server side of the SSL/TLS communication and fingerprints how servers respond to particular clients. +// JA3S uses the following field order: +// SSLVersion,Cipher,SSLExtension +func BareJa3s(hello *tlsx.ServerHelloBasic) []byte { + + // TODO: refactor into struct with inbuilt buffer to reduce allocations to ~ zero + // i.e. only realloc if previously allocated buffer is too small for current packet + + var ( + maxPossibleBufferLength = 5 + 1 + // Version = uint16 => maximum = 65536 = 5chars + 1 field sep + (5+1)*1 + // CipherSuite = uint16 => maximum = 65536 = 5chars + (5+1)*len(hello.Extensions) // uint16 = 2B => maximum = 65536 = 5chars + + buffer = make([]byte, 0, maxPossibleBufferLength) + ) + + buffer = strconv.AppendInt(buffer, int64(hello.Vers), 10) + buffer = append(buffer, sepFieldByte) + + /* + * Cipher Suite + */ + + buffer = strconv.AppendInt(buffer, int64(hello.CipherSuite), 10) + buffer = append(buffer, sepFieldByte) + + if len(hello.Extensions) > 0 { + + + /* + * Extensions + */ + + // collect extensions + lastElem := len(hello.Extensions) - 1 + if len(hello.Extensions) > 1 { + for _, e := range hello.Extensions[:lastElem] { + // filter GREASE values + if !greaseValues[uint16(e)] { + buffer = strconv.AppendInt(buffer, int64(e), 10) + buffer = append(buffer, sepValueByte) + } + } + } + // append last element if extensions are not empty + if lastElem != -1 { + // filter GREASE values + if !greaseValues[uint16(hello.Extensions[lastElem])] { + buffer = strconv.AppendInt(buffer, int64(hello.Extensions[lastElem]), 10) + } + } + } + + return buffer +} diff --git a/ja3s_test.go b/ja3s_test.go new file mode 100644 index 0000000..ec88d30 --- /dev/null +++ b/ja3s_test.go @@ -0,0 +1,188 @@ +/* + * JA3 - TLS Client Hello Hash + * Copyright (c) 2017, Salesforce.com, Inc. + * this code was created by Philipp Mieden + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package ja3 + +import ( + "bytes" + "encoding/hex" + "github.com/dreadl0ck/tlsx" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "io/ioutil" + "log" + "testing" +) + +var tlsServerHelloPacket, errDecodeServerHelloPacket = hex.DecodeString("f018982a38be48d343aac4b80800450005b4447b00007906ea20225f7893c0a8b20d01bbefca6011d3eb8013192d801000f087ae00000101080a79facb3a56d9798616030300640200006003035e5bea4420be035069306e15e36eca187d56a201e8e96f0a1afeeb529413fba020746572b9b8832a5b8f236ba51743281c20e923a880067e4c38a754f53f3a8743c02f00001800170000ff01000100000b0002010000100005000302683216030309980b0009940009910004fc308204f8308203e0a00302010202100a86b904765831e240cc6211101f5736300d06092a864886f70d01010b0500305e310b300906035504061302555331153013060355040a130c446967694365727420496e6331193017060355040b13107777772e64696769636572742e636f6d311d301b0603550403131447656f5472757374205253412043412032303138301e170d3138303130343030303030305a170d3230303730393132303030305a3068310b3009060355040613025553311330110603550408130a43616c69666f726e69613111300f060355040713085061736164656e61311b3019060355040a13124f70656e5820546563686e6f6c6f676965733114301206035504030c0b2a2e6f70656e782e6e657430820122300d06092a864886f70d01010105000382010f003082010a0282010100bf0efabfe55538e687d4db8f8d615ab1d5dce14702307ca21c0547cd2cc43e455627b4d8decd8015296c19127d29025fbac71f2fc1af1bfe525ed0c2778028a3e7dd643b60842801d9d196651924057827c4d3ce6ae62253b869f384d56a72060055e3537a67a9904e61a4628067f07c680a8bf30c14444d090910c182ea3c5e473b1ab1233838645b0bdfb24265ee32e3521b4bdbae6d79acdbdd7a00c0f00479537079fd352aade5e8394be2f2b397366cfd64bb94b1b96515b4e78d3a2fac04e9d5787792fcf411b6b66cb5dfbc4cb023d93fa99cee630d8ed3558b33e0f2a019ecc5eac16919c48a844f693b5e7ccb4eb84724ab963979c570eb7a3b58ad0203010001a38201a6308201a2301f0603551d230418301680149058ffb09c75a8515477b1edf2a34316389e6cc5301d0603551d0e04160414e03952aaaeb23c67e939ec6022188686bcb314c430210603551d11041a3018820b2a2e6f70656e782e6e657482096f70656e782e6e6574300e0603551d0f0101ff0404030205a0301d0603551d250416301406082b0601050507030106082b06010505070302303e0603551d1f043730353033a031a02f862d687474703a2f2f6364702e67656f74727573742e636f6d2f47656f54727573745253414341323031382e63726c304c0603551d2004453043303706096086480186fd6c0101302a302806082b06010505070201161c68747470733a2f2f7777772e64696769636572742e636f6d2f4350533008060667810c010202307506082b0601050507010104693067302606082b06010505073001861a687474703a2f2f7374617475732e67656f74727573742e636f6d303d06082b060105050730028631687474703a2f2f636163657274732e67656f74727573742e636f6d2f47656f54727573745253414341323031382e63727430090603551d1304023000300d06092a864886f70d01010b0500038201010045a0d33fa95d251c861518096f2ffe89bae2e062cfc4505859b42adeaa775fcf34c2eebde155cfb6449a806abffbaad2e88d77b1397e76ea08265dbc2aa98cb04e3d141c27ed629961d9cd90bb7f6db672408bb15126dd4df4cdf831465c1ee9f321505ace6e7c09d864683fbbaa8ee9fc73e74feefaa5a027067856000f56c9ae39bcb3cc586928429bd8abe5d2a3d8df5c63b4db45fa2cc3ffa6f75b946188019e162fb8dfacf2be3064bf52788378e61c8397ec13e72649fbe15e5c5b6d8bae43ca7b188eda5a41e512f508206fefe96a832ce4676a425ddd06e3cfe93046140818a7359723f2477315f38ab51f83c9dced627efaf6541ed3e9c0767376b200048f3082048b30820373a0") + +func getServerHello(p gopacket.Packet, b *testing.B) (hello *tlsx.ServerHelloBasic) { + + if errDecodeServerHelloPacket != nil { + log.Fatal("failed to decode test packet", errDecodeServerHelloPacket) + } + + if tl := p.TransportLayer(); tl != nil { + if tcp, ok := tl.(*layers.TCP); ok { + if tcp.SYN { + b.Fatal("Connection setup") + } else if tcp.FIN { + b.Fatal("Connection teardown") + } else if tcp.ACK && len(tcp.LayerPayload()) == 0 { + b.Fatal("Acknowledgement packet") + } else if tcp.RST { + b.Fatal("Unexpected packet") + } else { + hello = &tlsx.ServerHelloBasic{} + err := hello.Unmarshal(tcp.LayerPayload()) + if err != nil { + b.Fatal("invalid packet", err) + } + } + } + } + return +} + +/* + * Tests + */ + +func TestDigestHexJa3sCorrect(t *testing.T) { + + p := gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Lazy) + if p.ErrorLayer() != nil { + t.Fatal(p.ErrorLayer().Error()) + } + + hash := DigestHexPacketJa3s(p) + if hash != "5b94af9bf6efc9dea416841602004fbb" { + t.Fatal(hash, "!=", "5b94af9bf6efc9dea416841602004fbb") + } +} + +func TestDigestHexJa3sComparePcap(t *testing.T) { + + // Step 1: Read output of reference implementation + data, err := ioutil.ReadFile("ja3s_test.log") + if err != nil { + t.Fatal(err) + } + + // Step 2: generate output wth goja3 and read into buffer + var ( + b bytes.Buffer + ) + + // read test.pcap and generate fingerprints + ReadFileJa3s("test.pcap", &b) + + // Step 3: compare the number of fingerprints and then the digest for each record + if len(data) != len(b.Bytes()) { + t.Fatal("len(records) != len(referenceRecords): ", len(data), " != ", len(b.Bytes())) + } + + reference := bytes.Split(data, []byte{'\n'}) + + // range reference records + for i, r := range bytes.Split(b.Bytes(), []byte{'\n'}) { + + // compare reference digest to the one produced by goja3 + if string(r) != string(reference[i]) { + t.Fatal("message does not match reference implementation: ", string(r), " != ", string(reference[i])) + } + } +} + +/* + * Benchmarks + */ + +func BenchmarkDigestHexPacketJa3s(b *testing.B) { + + p := gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Default) + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + DigestHexPacketJa3s(p) + } +} + +func BenchmarkDigestPacketJa3s(b *testing.B) { + p := gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Default) + if p.ErrorLayer() != nil { + b.Fatal(p.ErrorLayer().Error()) + } + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + DigestPacketJa3s(p) + } +} + +func BenchmarkBarePacketJa3s(b *testing.B) { + p := gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Default) + if p.ErrorLayer() != nil { + b.Fatal(p.ErrorLayer().Error()) + } + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + BarePacketJa3s(p) + } +} + +func BenchmarkDigestHexJa3s(b *testing.B) { + var ( + p = gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Lazy) + hello = getServerHello(p, b) + ) + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + DigestHexJa3s(hello) + } +} + +func BenchmarkDigestJa3s(b *testing.B) { + + var ( + p = gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Lazy) + hello = getServerHello(p, b) + ) + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + DigestJa3s(hello) + } +} + +func BenchmarkBareJa3s(b *testing.B) { + + var ( + p = gopacket.NewPacket(tlsServerHelloPacket, layers.LinkTypeEthernet, gopacket.Lazy) + hello = getServerHello(p, b) + ) + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + BareJa3s(hello) + } +} diff --git a/ja3s_test.log b/ja3s_test.log new file mode 100644 index 0000000..6bbe9bc --- /dev/null +++ b/ja3s_test.log @@ -0,0 +1,71 @@ +[192.168.0.104:47851] JA3S: 768,10, --> beee243d749b8f43c283eeaf7517600d +[192.168.0.104:47852] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:47436] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:47437] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:47438] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:47439] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:47440] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:47441] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:47442] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42577] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42938] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42939] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42940] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42941] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42942] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:42949] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:60089] JA3S: 768,10, --> beee243d749b8f43c283eeaf7517600d +[192.168.0.109:60090] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60093] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60096] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60097] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60098] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60099] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60091] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60094] JA3S: 769,4, --> 53611273a714cb4789c8222932efd5a7 +[192.168.0.109:60092] JA3S: 769,4, --> 53611273a714cb4789c8222932efd5a7 +[192.168.0.109:60100] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60101] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60102] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60103] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60104] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60105] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60106] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60107] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60108] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:60109] JA3S: 769,5, --> 9aeeb84942a46257594025306635f0ff +[192.168.0.109:50499] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:50539] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:50540] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:50541] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:50542] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:50543] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.109:50544] JA3S: 769,47, --> 18e962e106761869a61045bed0e81c2c +[192.168.0.102:7298] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7301] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7303] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7304] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7305] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7306] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7307] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7308] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7309] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7310] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7311] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7312] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7323] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7324] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7326] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7327] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7328] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7329] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7330] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7331] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7332] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7333] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7334] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7335] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7341] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7342] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7343] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 +[192.168.0.102:7344] JA3S: 769,53, --> ab1e32eeaf70ee94c0af00f08d126891 diff --git a/json.go b/json.go index d5ccb61..317a9c4 100644 --- a/json.go +++ b/json.go @@ -21,24 +21,26 @@ import ( "strconv" "time" - "github.com/dreadl0ck/gopacket" - "github.com/dreadl0ck/gopacket/layers" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" ) // Record contains all information for a calculated JA3 type Record struct { - DestinationIP string `json:"destination_ip"` - DestinationPort int `json:"destination_port"` - JA3 string `json:"ja3"` - JA3Digest string `json:"ja3_digest"` - SourceIP string `json:"source_ip"` - SourcePort int `json:"source_port"` + DestinationIP string `json:"destination_ip"` + DestinationPort int `json:"destination_port"` + JA3 string `json:"ja3"` + JA3Digest string `json:"ja3_digest"` + JA3S string `json:"ja3s"` + JA3SDigest string `json:"ja3s_digest"` + SourceIP string `json:"source_ip"` + SourcePort int `json:"source_port"` Timestamp float64 `json:"timestamp"` } // ReadFileJSON reads the PCAP file at the given path -// and prints out all packets containing JA3 digests fromatted as JSON to the supplied io.Writer -func ReadFileJSON(file string, out io.Writer) { +// and prints out all packets containing JA3 digests formatted as JSON to the supplied io.Writer +func ReadFileJSON(file string, out io.Writer, doJA3s bool) { r, f, err := openPcap(file) if err != nil { @@ -62,9 +64,15 @@ func ReadFileJSON(file string, out io.Writer) { p = gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Lazy) // get JA3 if possible - bare = BarePacket(p) + bare = BarePacket(p) + isServer bool ) + if len(bare) == 0 && doJA3s { + bare = BarePacketJa3s(p) + isServer = true + } + // check if we got a result if len(bare) > 0 { @@ -83,16 +91,24 @@ func ReadFileJSON(file string, out io.Writer) { srcPort, _ := strconv.Atoi(tl.TransportFlow().Src().String()) dstPort, _ := strconv.Atoi(tl.TransportFlow().Dst().String()) - // append record and populate all fields - records = append(records, &Record{ + r := &Record{ DestinationIP: nl.NetworkFlow().Dst().String(), DestinationPort: dstPort, - JA3: string(bare), - JA3Digest: BareToDigestHex(bare), SourceIP: nl.NetworkFlow().Src().String(), SourcePort: srcPort, Timestamp: timeToFloat(ci.Timestamp), - }) + } + + if isServer { + r.JA3S = string(bare) + r.JA3SDigest = BareToDigestHex(bare) + } else { + r.JA3 = string(bare) + r.JA3Digest = BareToDigestHex(bare) + } + + // append record and populate all fields + records = append(records, r) } } @@ -102,14 +118,16 @@ func ReadFileJSON(file string, out io.Writer) { panic(err) } - // write to output io.Writer - _, err = out.Write(b) - if err != nil { - panic(err) + if string(b) != "null" { // no matches will result in "null" json + // write to output io.Writer + _, err = out.Write(b) + if err != nil { + panic(err) + } } if Debug { - fmt.Println(len(records), "fingeprints.") + fmt.Println(len(records), "fingerprints.") } } @@ -122,11 +140,11 @@ func timeToString(t time.Time) string { func timeStringToFloat64(t string) float64 { f, err := strconv.ParseFloat(t, 64) if err != nil { - fmt.Println("[ERROR] failed to convert",t, "to float64. error:", err) + fmt.Println("[ERROR] failed to convert", t, "to float64. error:", err) } return f -} +} func timeToFloat(t time.Time) float64 { return timeStringToFloat64(timeToString(t)) -} \ No newline at end of file +} diff --git a/live.go b/live.go index 9f64305..130dc4f 100644 --- a/live.go +++ b/live.go @@ -5,9 +5,9 @@ import ( "io" "strings" - "github.com/dreadl0ck/gopacket" - "github.com/dreadl0ck/gopacket/layers" - "github.com/dreadl0ck/gopacket/pcap" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" ) // ReadInterfaceCSV reads packets from the named interface @@ -47,6 +47,10 @@ func ReadInterfaceCSV(iface string, out io.Writer, separator string) { digest = DigestHexPacket(p) ) + if digest == "" { + digest = DigestHexPacketJa3s(p) + } + // check if we got a result if digest != "" { diff --git a/utils.go b/utils.go index 2a32e37..578567b 100644 --- a/utils.go +++ b/utils.go @@ -4,8 +4,8 @@ import ( "fmt" "os" - "github.com/dreadl0ck/gopacket" - "github.com/dreadl0ck/gopacket/pcapgo" + "github.com/google/gopacket" + "github.com/google/gopacket/pcapgo" ) // PacketSource means we can read Packets. From c791ba2442a4b6631bd501b36d37e8cd530d3131 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 4 Mar 2020 20:48:19 +0100 Subject: [PATCH 2/2] documentation and api cleanup --- README.md | 22 ++++++++++------------ cmd/main.go | 4 ++-- csv.go | 4 ++-- json.go | 2 +- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 840d117..1cbb417 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,19 @@ JA3 is a technique developed by Salesforce, to fingerprint the TLS client and server hellos. -The official python implementation can be found here: https://github.com/salesforce/ja3 - +The official python implementation can be found [here](https://github.com/salesforce/ja3). More details can be found in their blog post: https://engineering.salesforce.com/open-sourcing-ja3-92c9e53c3c41 -This package provides a pure golang implementation, that was created to operate on **ClientHello** instances from the **bradleyfalzon/tlsx** package. -For this to work, the package needed to be slightly modified to store the values for extensions in the order they are encountered. -It was forked to **github.com/dreadl0ck/tlsx**. -By now, the tlsx fork has been improved to support extracting the server hello message, -besides unit tests, benchmarks and a faster unmarshaling interface for use with Ja3 fingerprinting. +This package provides a pure golang implementation of both the client and server hash functions, +with unit tests to ensure correct behavior and performance. +This implementation uses [google/gopacket](https://github.com/google/gopacket) for packet decoding. + +The **[bradleyfalzon/tlsx](github.com/bradleyfalzon/tlsx)** fork used for parsing the TLS handshakes (**[dreadl0ck/tlsx](github.com/dreadl0ck/tlsx)**), +has been extended to support extracting the server hello message, unit tests, +benchmarks and a faster decoding interface for use with Ja3 fingerprinting. -This implementation uses gopacket for packet decoding. Assembly of the ja3 bare was previously implemented with the golang 1.10 **strings.Builder** type, however using byte slices for this turned out to be faster and causes less allocations. (Thanks for the tips @lukechampine) - Thanks to @guigzzz for his pull request on further reducing allocations! ## Package Usage @@ -39,10 +38,10 @@ func ReadInterfaceCSV(iface string, out io.Writer, separator string) Files: ```go -func ReadFileJSON(file string, out io.Writer) +func ReadFileJSON(file string, out io.Writer, doJA3s bool) ``` ```go -func ReadFileCSV(file string, out io.Writer, separator string) +func ReadFileCSV(file string, out io.Writer, separator string, doJA3s bool) ``` Read file and only print Ja3s values, mimics the python implementation from salesforce: @@ -94,7 +93,6 @@ func DigestHex(hello *tlsx.ClientHello) string func DigestHexJa3s(hello *tlsx.ServerHello) string ``` - ## Commandline Tool The commandline program mimics the python reference implementation by default. diff --git a/cmd/main.go b/cmd/main.go index 192c3ff..d4fa9ff 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -56,12 +56,12 @@ func main() { } if *flagTSV { - ja3.ReadFileCSV(*flagInput, os.Stdout, "\t") + ja3.ReadFileCSV(*flagInput, os.Stdout, "\t", *flagJa3S) return } if *flagCSV { - ja3.ReadFileCSV(*flagInput, os.Stdout, *flagSeparator) + ja3.ReadFileCSV(*flagInput, os.Stdout, *flagSeparator, *flagJa3S) return } diff --git a/csv.go b/csv.go index f0436cb..663d266 100644 --- a/csv.go +++ b/csv.go @@ -25,7 +25,7 @@ import ( // ReadFileCSV reads the PCAP file at the given path // and prints out all packets containing JA3 digests to the supplied io.Writer -func ReadFileCSV(file string, out io.Writer, separator string) { +func ReadFileCSV(file string, out io.Writer, separator string, doJA3s bool) { r, f, err := openPcap(file) if err != nil { @@ -60,7 +60,7 @@ func ReadFileCSV(file string, out io.Writer, separator string) { isServer bool ) - if digest == "" { + if doJA3s && digest == "" { digest = DigestHexPacketJa3s(p) isServer = true } diff --git a/json.go b/json.go index 317a9c4..393769f 100644 --- a/json.go +++ b/json.go @@ -68,7 +68,7 @@ func ReadFileJSON(file string, out io.Writer, doJA3s bool) { isServer bool ) - if len(bare) == 0 && doJA3s { + if doJA3s && len(bare) == 0 { bare = BarePacketJa3s(p) isServer = true }