From 5e6983720745256d17f1c077925663c1eb72f7a8 Mon Sep 17 00:00:00 2001
From: Michael Henriksen <mhenriksen@gitlab.com>
Date: Fri, 1 Jan 2021 12:48:03 +0100
Subject: [PATCH 1/2] Add filter strings functionality

Adds a new filterStrings argument to isListening
which will cause the function to read the first
512 bytes from responses and check for any of
the given strings and return false if present.
This is implemented in order to support a new
-filter-cf-errors cli flag which will filter out
error pages from Cloudflare as they are common
false-positives.
---
 main.go | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/main.go b/main.go
index 60e5004..7623410 100644
--- a/main.go
+++ b/main.go
@@ -48,6 +48,10 @@ func main() {
 	var preferHTTPS bool
 	flag.BoolVar(&preferHTTPS, "prefer-https", false, "only try plain HTTP if HTTPS fails")
 
+	// filter out cloudflare error pages
+	var filterCloudflareErrors bool
+	flag.BoolVar(&filterCloudflareErrors, "filter-cf-errors", false, "Filter out Cloudflare error pages")
+
 	// HTTP method to use
 	var method string
 	flag.StringVar(&method, "method", "GET", "HTTP method to use")
@@ -57,6 +61,13 @@ func main() {
 	// make an actual time.Duration out of the timeout
 	timeout := time.Duration(to * 1000000)
 
+	var filterStrings []string
+
+	// Add Cloudflare signatures to filterStrings if filterCloudflareErrors
+	if filterCloudflareErrors {
+		filterStrings = append(filterStrings, "<center>cloudflare</center>", "cf_styles-css")
+	}
+
 	var tr = &http.Transport{
 		MaxIdleConns:      30,
 		IdleConnTimeout:   time.Second,
@@ -96,7 +107,7 @@ func main() {
 
 				// always try HTTPS first
 				withProto := "https://" + url
-				if isListening(client, withProto, method) {
+				if isListening(client, withProto, method, filterStrings) {
 					output <- withProto
 
 					// skip trying HTTP if --prefer-https is set
@@ -120,7 +131,7 @@ func main() {
 		go func() {
 			for url := range httpURLs {
 				withProto := "http://" + url
-				if isListening(client, withProto, method) {
+				if isListening(client, withProto, method, filterStrings) {
 					output <- withProto
 					continue
 				}
@@ -210,7 +221,7 @@ func main() {
 	outputWG.Wait()
 }
 
-func isListening(client *http.Client, url, method string) bool {
+func isListening(client *http.Client, url, method string, filterStrings []string) bool {
 
 	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
@@ -222,8 +233,21 @@ func isListening(client *http.Client, url, method string) bool {
 
 	resp, err := client.Do(req)
 	if resp != nil {
-		io.Copy(ioutil.Discard, resp.Body)
-		resp.Body.Close()
+		defer resp.Body.Close()
+
+		if len(filterStrings) != 0 {
+			// Read the first 512 bytes of the response and check for presence of any filter strings
+			peek := make([]byte, 512)
+			resp.Body.Read(peek)
+			peekStr := string(peek)
+			for _, filterString := range filterStrings {
+				if strings.Contains(peekStr, filterString) {
+					return true
+				}
+			}
+		} else {
+			io.Copy(ioutil.Discard, resp.Body)
+		}
 	}
 
 	if err != nil {

From 221d08030a39636fa92028c1be8e13b869bc4bc3 Mon Sep 17 00:00:00 2001
From: Michael Henriksen <mhenriksen@gitlab.com>
Date: Fri, 1 Jan 2021 12:54:23 +0100
Subject: [PATCH 2/2] Skip unresolvable domains

Performs a DNS lookup on incoming domains so that
channels are not filled with jobs to check dead
domains.
---
 main.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/main.go b/main.go
index 7623410..f484670 100644
--- a/main.go
+++ b/main.go
@@ -168,6 +168,11 @@ func main() {
 	for sc.Scan() {
 		domain := strings.ToLower(sc.Text())
 
+		// Skip unresolvable domains
+		if _, err := net.LookupIP(domain); err != nil {
+			continue
+		}
+
 		// submit standard port checks
 		if !skipDefault {
 			httpsURLs <- domain