Merge pull request #4 from projectdiscovery/dev

Merging to master

README.md

@@ -14,7 +14,6 @@ Additionally a replay utility allows to import the dumped traffic (request/respo
 
 # Resources
 
-- [Resources](#resources)
 - [Features](#features)
 - [Installation](#installation)
     - [From Binary](#from-binary)
@@ -27,7 +26,8 @@ Additionally a replay utility allows to import the dumped traffic (request/respo
     - [Match/Filter traffic with with DSL language.](#matchfilter-traffic-with-with-dsl-language)
     - [Match and Replace on the fly](#match-and-replace-on-the-fly)
     - [Replay all traffic into burp](#replay-all-traffic-into-burp)
- - [Applications of Proxify](#applications-of-proxify)
+- [Installing SSL Certificate](#installing-ssl-certificate)
+- [Applications of Proxify](#applications-of-proxify)
 
 # Features
 
@@ -79,24 +79,25 @@ The installation is easy. You can download the pre-built binaries for your platf
 
 This will display help for the tool. Here are all the switches it supports.
 
-| Flag                       | Description                | Example                                                      |
-| -------------------------- | -------------------------- | ------------------------------------------------------------ |
-| addr                       | Listen HTTP IP and Port    | proxify -addr 127.0.0.1:8080                                 |
-| dns-addr                   | Listen DNS IP and Port     | proxify -dns-addr '127.0.0.1:80'                             |
-| dns-mapping                | DNS A mapping              | proxify -dns-mapping test.com:80                             |
-| dns-resolver               | Listen DNS IP and Port     | proxify -dns-resolver '127.0.0.1:5353'                       |
-| http-proxy                 | Upstream HTTP Proxy        | proxify -http-proxy hxxp://127.0.0.1:8080                    |
-| no-color                   | No Color in output         | proxify -no-color                                            |
-| output                     | Output Folder              | proxify -output logs                                         |
-| request-dsl                | Request Filter DSL         | proxify -request-dsl "contains(request,'admim')"             |
-| request-match-replace-dsl  | Request Match-Replace DSL  | proxify -request-match-replace-dsl "replace(request,'false','true')" |
-| response-dsl               | Response Filter DSL        | proxify -response-dsl "contains(response, md5('test'))"      |
-| response-match-replace-dsl | Response Match-Replace DSL | proxify -response-match-replace-dsl "regex(response, '^authentication failed$', 'authentication ok')" |
-| silent                     | Silent output              | proxify -silent                                              |
-| socks5-proxy               | Upstream socks5 proxy      | proxify -socks5-proxy socks5://proxy-ip:port                 |
-| v                          | Verbose output             | proxify -v                                                   |
-| version                    | Current version            | proxify -version                                             |
-
+| Flag                       | Description                     | Example                                                      |
+| -------------------------- | ------------------------------- | ------------------------------------------------------------ |
+| addr                       | Listen HTTP IP and Port         | proxify -addr 127.0.0.1:8080                                 |
+| config                     | Config data path                | proxify -config certs                                        |
+| cert-cache-size            | Number of certificates to cache | proxify -cert-cache-size 1024                                |
+| dns-addr                   | Listen DNS IP and Port          | proxify -dns-addr '127.0.0.1:80'                             |
+| dns-mapping                | DNS A mapping                   | proxify -dns-mapping test.com:80                             |
+| dns-resolver               | Listen DNS IP and Port          | proxify -dns-resolver '127.0.0.1:5353'                       |
+| http-proxy                 | Upstream HTTP Proxy             | proxify -http-proxy hxxp://127.0.0.1:8080                    |
+| no-color                   | No Color in output              | proxify -no-color                                            |
+| output                     | Output Folder                   | proxify -output logs                                         |
+| request-dsl                | Request Filter DSL              | proxify -request-dsl "contains(request,'admim')"             |
+| request-match-replace-dsl  | Request Match-Replace DSL       | proxify -request-match-replace-dsl "replace(request,'false','true')" |
+| response-dsl               | Response Filter DSL             | proxify -response-dsl "contains(response, md5('test'))"      |
+| response-match-replace-dsl | Response Match-Replace DSL      | proxify -response-match-replace-dsl "regex(response, '^authentication failed$', 'authentication ok')" |
+| silent                     | Silent output                   | proxify -silent                                              |
+| socks5-proxy               | Upstream socks5 proxy           | proxify -socks5-proxy socks5://proxy-ip:port                 |
+| v                          | Verbose output                  | proxify -v                                                   |
+| version                    | Current version                 | proxify -version                                             |
 
 
 ### Use Upstream proxy
@@ -160,6 +161,11 @@ Replay all the dumped requests/responses into the destination URL (http://127.0.
 ▶ replay -output "logs/"
 ```
 
+### Installing SSL Certificate
+
+A certificate authority is generated for proxify which is stored in the folder `~/.config/proxify/` as default, manually can be specified by `-config` flag. The generated certificate can be imported by visiting [http://proxify/cacert.crt](http://proxify/cacert.crt) in a browser connected to proxify. 
+
+Installation steps for the Root Certificate is similar to other proxy tools which includes adding the cert to system trusted root store.
 
 ### Applications of Proxify
 
@@ -203,7 +209,6 @@ Start Chrome browser in Mac OS,
 
 </details>
 
-
 <details>
 <summary> Store all the response of while you fuzz as per you config at run time. </summary>
 
@@ -222,4 +227,5 @@ ffuf -x http://127.0.0.1:9999 FFUF_CMD_HERE
 
 </details>
 
+
 Proxify is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/proxify/blob/master/THANKS.md)** file for more details.

cmd/replay/replay.go

@@ -23,14 +23,6 @@ var (
 	responses  map[string]*http.Response
 )
 
-func OnRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
-	key := req.Header.Get("proxify")
-	response := responses[key]
-	delete(responses, key)
-	ctx.Resp = response
-	return req, response
-}
-
 type Options struct {
 	DNSListenerAddress       string
 	HTTPListenerAddress      string
@@ -62,7 +54,11 @@ func main() {
 	httpproxy.Tr.DialContext = dialer.Dial
 	httpproxy.OnRequest().HandleConnect(goproxy.AlwaysMitm)
 	// httpproxy.OnRequest().DoFunc(OnRequest)
-	go http.ListenAndServe(options.HTTPProxyListenerAddress, httpproxy)
+	go func() {
+		if err := http.ListenAndServe(options.HTTPProxyListenerAddress, httpproxy); err != nil {
+			log.Fatalf("Could not serve proxy: %s\n", err)
+		}
+	}()
 
 	// dns server
 	var domainsToAddresses map[string]string = map[string]string{
@@ -85,9 +81,13 @@ func main() {
 			w.Header().Add(k, strings.Join(v, "; "))
 		}
 		w.WriteHeader(response.StatusCode)
-		io.Copy(w, response.Body)
+		_, _ = io.Copy(w, response.Body)
 	})
-	go http.ListenAndServe(":80", nil)
+	go func() {
+		if err := http.ListenAndServe(":80", nil); err != nil {
+			log.Fatalf("Could not listen and serve: %s\n", err)
+		}
+	}()
 
 	// http client proxy
 	proxyUrl, err := url.Parse(options.HTTPBurpAddress)
@@ -125,8 +125,8 @@ func visit() filepath.WalkFunc {
 		}
 
 		filename := filepath.Base(path)
-		filename = strings.TrimRight(filename, ".txt")
-		filename = strings.TrimRight(filename, ".match")
+		filename = strings.TrimSuffix(filename, ".txt")
+		filename = strings.TrimSuffix(filename, ".match")
 
 		file, err := os.Open(path)
 		if err != nil {
@@ -163,7 +163,6 @@ func visit() filepath.WalkFunc {
 
 		responses[id] = response
 		_, err = httpclient.Do(request)
-
-		return nil
+		return err
 	}
 }

go.mod

@@ -5,6 +5,8 @@ go 1.14
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible // indirect
 	github.com/elazarl/goproxy v0.0.0-20201021153353-00ad82a08272
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/dsl v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.2
 	github.com/projectdiscovery/gologger v1.0.1

go.sum

@@ -10,6 +10,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
@@ -18,6 +20,8 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/dsl v0.0.2 h1:7UysmR2d9CUbmNta8OFYA990FBlc/G7P6s7oNZL/kn4=
 github.com/projectdiscovery/dsl v0.0.2/go.mod h1:LMhpgZjWJ8NfZokFhzyu1blLWhUutCtNA9C6bH80WOo=

internal/runner/options.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"flag"
 	"os"
+	"path"
 
 	"github.com/projectdiscovery/gologger"
 )
@@ -11,6 +12,8 @@ import (
 //nolint:maligned // used once
 type Options struct {
 	OutputDirectory         string
+	Directory               string
+	CertCacheSize           int
 	Verbose                 bool
 	Silent                  bool
 	Version                 bool
@@ -28,9 +31,17 @@ type Options struct {
 }
 
 func ParseOptions() *Options {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		// Almost never here but panic
+		panic(err)
+	}
+
 	options := &Options{}
 	flag.StringVar(&options.OutputDirectory, "output", "logs", "Output Folder")
 	flag.BoolVar(&options.Verbose, "v", false, "Verbose")
+	flag.StringVar(&options.Directory, "config", path.Join(homeDir, ".config", "proxify"), "Directory for storing program information")
+	flag.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache")
 	flag.BoolVar(&options.Silent, "silent", false, "Silent")
 	flag.BoolVar(&options.NoColor, "no-color", true, "No Color")
 	flag.BoolVar(&options.Version, "version", false, "Version")
@@ -46,6 +57,7 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.UpstreamSocks5Proxy, "socks5-proxy", "", "Upstream SOCKS5 Proxy (eg socks5://proxyip:proxyport")
 
 	flag.Parse()
+	_ = os.MkdirAll(options.Directory, os.ModePerm)
 
 	// Read the inputs and configure the logging
 	options.configureOutput()

internal/runner/runner.go

@@ -15,8 +15,10 @@ type Runner struct {
 
 // NewRunner instance
 func NewRunner(options *Options) (*Runner, error) {
-	proxy := proxify.NewProxy(&proxify.Options{
+	proxy, err := proxify.NewProxy(&proxify.Options{
 		Silent:                  options.Silent,
+		Directory:               options.Directory,
+		CertCacheSize:           options.CertCacheSize,
 		Verbose:                 options.Verbose,
 		ListenAddr:              options.ListenAddr,
 		OutputDirectory:         options.OutputDirectory,
@@ -30,6 +32,9 @@ func NewRunner(options *Options) (*Runner, error) {
 		RequestMatchReplaceDSL:  options.RequestMatchReplaceDSL,
 		ResponseMatchReplaceDSL: options.ResponseMatchReplaceDSL,
 	})
+	if err != nil {
+		return nil, err
+	}
 	return &Runner{options: options, proxy: proxy}, nil
 }
 

logger.go

@@ -16,9 +16,8 @@ type OptionsLogger struct {
 }
 
 type OutputData struct {
-	userdata   UserData
-	isResponse bool
-	data       []byte
+	userdata UserData
+	data     []byte
 }
 
 type Logger struct {
@@ -31,7 +30,7 @@ func NewLogger(options *OptionsLogger) *Logger {
 		options:    options,
 		asyncqueue: make(chan OutputData, 1000),
 	}
-	logger.createOutputFolder()
+	_ = logger.createOutputFolder()
 	go logger.AsyncWrite()
 	return logger
 }
@@ -40,7 +39,6 @@ func (l *Logger) createOutputFolder() error {
 	if l.options.OutputFolder == "" {
 		return nil
 	}
-
 	return os.MkdirAll(l.options.OutputFolder, 0755)
 }
 
@@ -59,7 +57,7 @@ func (l *Logger) AsyncWrite() {
 			if outputdata.userdata.match {
 				outputFileName = destFile + ".match.txt"
 			}
-			os.Rename(destFile, outputFileName)
+			_ = os.Rename(destFile, outputFileName)
 		}
 	}
 }
@@ -101,11 +99,3 @@ func (l *Logger) LogResponse(resp *http.Response, userdata UserData) error {
 func (l *Logger) Close() {
 	close(l.asyncqueue)
 }
-
-func fileExists(filename string) bool {
-	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-	return !info.IsDir()
-}