Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RF: standardise the way url param is handled #26

Merged
merged 2 commits into from
Jun 10, 2024
Merged

RF: standardise the way url param is handled #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8f649a2
RF: standardise the way url param is handled
kynrai Jun 10, 2024
ff31e06
Merge remote-tracking branch 'origin/main' into rf/standardise_url_param
kynrai Jun 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,4 @@ deps:
env:
@source .env
.PHONY: env

20 changes: 3 additions & 17 deletions handlers/block_lists.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"net"
"net/http"
"net/url"
"slices"
"sort"
"sync"
Expand Down Expand Up @@ -129,29 +128,16 @@ func checkDomainAgainstDNSServers(domain string) []Blocklist {
return results
}

func urlToDomain(rawURL string) (string, error) {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return "", err
}
return parsedURL.Hostname(), nil
}

func HandleBlockLists() http.Handler {
type Response struct {
BlockLists []Blocklist `json:"blocklists"`
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rawURL := r.URL.Query().Get("url")
if rawURL == "" {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}
domain, err := urlToDomain(rawURL)
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrInvalidURL, http.StatusBadRequest)
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}
json.NewEncoder(w).Encode(Response{BlockLists: checkDomainAgainstDNSServers(domain)})
json.NewEncoder(w).Encode(Response{BlockLists: checkDomainAgainstDNSServers(rawURL.Hostname())})
})
}
9 changes: 3 additions & 6 deletions handlers/carbon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"io"
"net/http"
"strconv"
"strings"
"time"
)

Expand All @@ -32,9 +31,6 @@ type CarbonData struct {

// Function to get the HTML size of the website
func getHtmlSize(ctx context.Context, url string) (int, error) {
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
url = "http://" + url
}
client := &http.Client{
Timeout: time.Second * 5,
}
Expand Down Expand Up @@ -94,12 +90,13 @@ func getCarbonData(ctx context.Context, sizeInBytes int) (*CarbonData, error) {

func HandleCarbon() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

url := rawURL.String()
sizeInBytes, err := getHtmlSize(r.Context(), url)
if err != nil {
JSONError(w, fmt.Errorf("error getting HTML size: %v", err), http.StatusInternalServerError)
Expand Down
10 changes: 3 additions & 7 deletions handlers/cookies.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net/http"
"strings"
"time"

"github.com/chromedp/cdproto/network"
Expand Down Expand Up @@ -74,19 +73,16 @@ func HandleCookies() http.Handler {
ClientCookies []map[string]any `json:"clientCookies"`
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}
// Ensure the URL includes a scheme
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
url = "http://" + url
}

var headerCookies []string
var clientCookies []map[string]interface{}

url := rawURL.String()
// Fetch headers using http.Get
resp, err := http.Get(url)
if err != nil {
Expand Down
17 changes: 3 additions & 14 deletions handlers/dns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"net"
"net/http"
"strings"
"time"
)

Expand Down Expand Up @@ -107,28 +106,18 @@ func resolveDNSRecords(ctx context.Context, hostname string) (*DNSResponse, erro

func HandleDNS() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rawURL := r.URL.Query().Get("url")
if rawURL == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

// Extract the hostname from the URL
hostname := rawURL
if strings.HasPrefix(hostname, "http://") || strings.HasPrefix(hostname, "https://") {
hostname = strings.ReplaceAll(hostname, "http://", "")
hostname = strings.ReplaceAll(hostname, "https://", "")
if parts := strings.Split(hostname, "/"); len(parts) > 0 {
hostname = parts[0]
}
}

// Create a context with timeout
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()

// Resolve DNS records
dnsResponse, err := resolveDNSRecords(ctx, hostname)
dnsResponse, err := resolveDNSRecords(ctx, rawURL.Hostname())
if err != nil {
JSONError(w, fmt.Errorf("error resolving DNS: %v", err), http.StatusInternalServerError)
return
Expand Down
15 changes: 5 additions & 10 deletions handlers/dns_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"net"
"net/http"
"strings"
"time"
)

Expand Down Expand Up @@ -60,28 +59,24 @@ func resolveDNSServer(ctx context.Context, domain string) ([]Result, error) {

func HandleDNSServer() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

// Extract the domain from the URL
domain := strings.ReplaceAll(url, "http://", "")
domain = strings.ReplaceAll(domain, "https://", "")
domain = strings.TrimSuffix(domain, "/")

ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()

results, err := resolveDNSServer(ctx, domain)
hostname := rawURL.Hostname()
results, err := resolveDNSServer(ctx, hostname)
if err != nil {
JSONError(w, fmt.Errorf("error resolving DNS: %v", err), http.StatusInternalServerError)
return
}

response := Response{
Domain: domain,
Domain: hostname,
DNS: results,
}

Expand Down
9 changes: 4 additions & 5 deletions handlers/dnssec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,14 @@ import (
"fmt"
"net/http"
"net/url"
"strings"
)

const dnsGoogleURL = "https://dns.google/resolve"

func resolveDNS(domain string) (map[string]interface{}, error) {
dnsTypes := []string{"DNSKEY", "DS", "RRSIG"}
records := make(map[string]interface{})
domain = strings.TrimSuffix(domain, "")

for _, typ := range dnsTypes {

url := fmt.Sprintf("%s?name=%s&type=%s", dnsGoogleURL, url.PathEscape(domain), typ)
Expand Down Expand Up @@ -57,13 +56,13 @@ func resolveDNS(domain string) (map[string]interface{}, error) {

func HandleDnsSec() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
domain := r.URL.Query().Get("url")
if domain == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

records, err := resolveDNS(domain)
records, err := resolveDNS(rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
16 changes: 5 additions & 11 deletions handlers/firewall.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,8 @@ type wafResponse struct {
}

func checkWAF(url string) (wafResponse, error) {
fullURL := ""
if !strings.HasPrefix(url, "http") {
fullURL = "http://" + url
} else {
fullURL = url
}

resp, err := http.Get(fullURL)
// TODO(Lissy93): does this test require we set scheme to http?
resp, err := http.Get(url)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It just needs some protocol, whether http or https.
But that's now already covered by urlutils

if err != nil {
return wafResponse{}, fmt.Errorf("error fetching URL: %s", err.Error())
}
Expand Down Expand Up @@ -108,13 +102,13 @@ func checkWAF(url string) (wafResponse, error) {

func HandleFirewall() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
domain := r.URL.Query().Get("url")
if domain == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

result, err := checkWAF(domain)
result, err := checkWAF(rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
8 changes: 3 additions & 5 deletions handlers/getIP.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package handlers
import (
"net"
"net/http"
"strings"
)

func lookupAsync(address string) (map[string]interface{}, error) {
Expand All @@ -26,14 +25,13 @@ func lookupAsync(address string) (map[string]interface{}, error) {

func HandleGetIP() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

address := strings.ReplaceAll(strings.ReplaceAll(url, "https://", ""), "http://", "")
result, err := lookupAsync(address)
result, err := lookupAsync(rawURL.Hostname())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
11 changes: 3 additions & 8 deletions handlers/headers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,13 @@ import (

func HandleGetHeaders() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

// Ensure the URL has a scheme
if !(len(url) >= 7 && (url[:7] == "http://" || url[:8] == "https://")) {
url = "http://" + url
}

resp, err := http.Get(url)
resp, err := http.Get(rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
10 changes: 3 additions & 7 deletions handlers/hsts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@ type HSTSResponse struct {
}

func checkHSTS(url string) (HSTSResponse, error) {
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
url = "http://" + url // Assuming HTTP by default
}

client := &http.Client{}

req, err := http.NewRequest("HEAD", url, nil)
Expand Down Expand Up @@ -54,13 +50,13 @@ func checkHSTS(url string) (HSTSResponse, error) {

func HandleHsts() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

result, err := checkHSTS(url)
result, err := checkHSTS(rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
12 changes: 6 additions & 6 deletions handlers/http_security.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ type HTTPSecurityResponse struct {
}

func checkHTTPSecurity(url string) (HTTPSecurityResponse, error) {
fullURL := "http://" + url

resp, err := http.Get(fullURL)
// fullURL := "http://" + url
// TODO(Lissy93): does this test require we set scheme to http?
resp, err := http.Get(url)
if err != nil {
return HTTPSecurityResponse{}, fmt.Errorf("error making request: %s", err.Error())
}
Expand All @@ -35,13 +35,13 @@ func checkHTTPSecurity(url string) (HTTPSecurityResponse, error) {

func HandleHttpSecurity() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

result, err := checkHTTPSecurity(url)
result, err := checkHTTPSecurity(rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
6 changes: 3 additions & 3 deletions handlers/legacy_rank.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,13 +148,13 @@ func unzip(src, dest string) error {

func HandleLegacyRank() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" {
rawURL, err := extractURL(r)
if err != nil {
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
return
}

result, err := checkLegacyRank(url)
result, err := checkLegacyRank(rawURL.String())
if err != nil {
JSONError(w, err, http.StatusInternalServerError)
return
Expand Down
Loading
Loading