add proxy support

Author: ZORbit01

cmd/root.go

@@ -54,4 +54,7 @@ func init() {
 	rootCmd.Flags().StringSliceP("include", "i", []string{}, "include only domains / ex : -i google.com,facebook.com")
 
 	rootCmd.Flags().IntP("max-concurrency", "m", 1000, "max concurrent tasks / ex: -m 10")
+
+	rootCmd.Flags().StringP("proxy", "p", "", "proxy to use / ex: -p http://proxy.com:8080")
+
 }

core/crawler.go

@@ -85,20 +85,29 @@ type Crawler struct {
 }
 
 func NewCrawler(options *shared.Options) *Crawler {
-	return &Crawler{
+	crawler := Crawler{
 		RootURL:        options.URL,
 		Level:          options.Level,
 		LiveMode:       options.LiveMode,
 		ExportFile:     options.ExportFile,
 		RegexMap:       options.RegexMap,
 		ExcludedStatus: options.StatusResponses,
 		IncludedUrls:   options.IncludedUrls,
-		Client:         &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}},
 		Cache: Cache{
 			Visited: make(map[string]bool),
 		},
 		MaxConcurrency: options.MaxConcurrency,
 	}
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	if options.Proxy != nil {
+		transport.Proxy = http.ProxyURL(options.Proxy)
+	}
+	crawler.Client = &http.Client{
+		Transport: transport,
+	}
+	return &crawler
 }
 
 func (c *Crawler) Fetch(page *webtree.Page) {
@@ -132,9 +141,7 @@ func (c *Crawler) Fetch(page *webtree.Page) {
 func (c *Crawler) ExtractLinks(page *webtree.Page) (links []string) {
 	regex := regexp.MustCompile(GeneralRegex)
 	generalUrlMatches := regex.FindAllString(page.GetData(), -1)
-	for _, link := range generalUrlMatches {
-		links = append(links, link)
-	}
+	links = append(links, generalUrlMatches...)
 	hrefRegex := regexp.MustCompile(HrefRegex)
 	hrefMatches := hrefRegex.FindAllStringSubmatch(page.GetData(), -1)
 	for _, match := range hrefMatches {

shared/options.go

@@ -3,6 +3,7 @@ package shared
 import (
 	"fmt"
 	"net"
+	urltool "net/url"
 	"reflect"
 	"regexp"
 	"strings"
@@ -20,6 +21,7 @@ type Options struct {
 	StatusResponses []int             `name:"exclude codes"`
 	IncludedUrls    []string          `name:"include"`
 	MaxConcurrency  int               `name:"max concurrency"`
+	Proxy           *urltool.URL      `name:"proxy"`
 }
 
 func (o Options) BuildOptionBanner() string {
@@ -112,6 +114,16 @@ func (o Options) BuildOptionBanner() string {
 				banner += color.CyanString("  %s: %s", k, v)
 				banner += "\n"
 			}
+		} else {
+
+			banner += color.RedString("│")
+			banner += color.BlueString(name + ": ")
+			if fmt.Sprintf("%v", value) == "<nil>" {
+				banner += color.CyanString("not set")
+			} else {
+				banner += color.CyanString("%v", value)
+			}
+			banner += "\n"
 		}
 
 	}
@@ -189,6 +201,26 @@ func ValidateThenBuildOption(cmd *cobra.Command) (*Options, error) {
 		return nil, err
 	}
 
+	proxy, err := cmd.Flags().GetString("proxy")
+	if err != nil {
+		return nil, err
+	}
+
+	var parsedProxy *urltool.URL
+	if proxy != "" {
+		parsedProxy, err = urltool.Parse(proxy)
+		if err != nil {
+			return nil, err
+		}
+
+	} else {
+		parsedProxy = nil
+	}
+	// set max concurrency to 1 if live mode is enabled
+	if liveMode {
+		maxConcurrency = 1
+	}
+
 	options := &Options{
 		URL:             url,
 		Level:           level,
@@ -198,6 +230,7 @@ func ValidateThenBuildOption(cmd *cobra.Command) (*Options, error) {
 		StatusResponses: excludedStatus,
 		IncludedUrls:    includedUrls,
 		MaxConcurrency:  maxConcurrency,
+		Proxy:           parsedProxy,
 	}
 	options.ManipulateData()
 	return options, nil