add passive scan and can -b delete junk data

README.md

@@ -18,34 +18,50 @@ go build -v
 ## 使用
 
 ```bash
-└> ./blueming
 Usage of ./blueming:
-  -es string
-    	dirscan filter status(200,206,301,302,400,401,403,404,405,500,501,502,503,504,600,etc.) (default "404")
+  -b	filter output data
+  -crt string
+    	listen cert (default "ca.crt")
   -f string
     	set url file
   -i string
-    	set wordlist index(ex: test.php)
+    	set wordlist index(exp: test.php)
+  -key string
+    	listen key (default "ca.key")
   -l string
     	set log level(trace,debug,info,warn,fatal) (default "debug")
+  -listen string
+    	listen to scan dir (default "127.0.0.1:9099")
   -p string
-    	set download proxy
+    	set proxy, support http proxy(exp: http://localhost:8080)
   -s int
     	set timeout (default 10)
   -t int
-    	set thread (default 10)
+    	set thread (default 100)
   -u string
     	set url
+  -v int
+    	log level
   -w string
     	set wordlist
 ```
 
+> ./blueming -b 可删除output下的垃圾数据(必须使用)
+
 ## TODO
 
 > 基本满足以下要求即可
 
+- [ ] 常见文件泄露扫描 .git .hg .idea .DS_Store ...
+- [x] 开启被动扫描模式，配合httpx自动进行目录扫描(二级、三级、四级...)
 - [x] 通过URL自动生成文件名
 - [x] 根据后缀名将URL定义为对应的文件格式，如zip、tar.gz等
 - [x] 自动下载备份文件，并进行重命名
 - [x] 能够自定义字典
 - [x] 优化内存占用
+- [x] filter.sh 移至程序内部
+- [x] 目录扫描部分添加 页面相似度比较，每个新产生的都会与前面所有的请求进行比较一次(耗时)
+  - 比较时，各网站相互独立，采用协程的方式
+- [x] 采用 GET 请求，查看文件过大时的response
+  - 文件过大导致的超时 则获取 header，比较历史记录中的length
+  - 正常情况，比较body

cmd/blueming/main.go

@@ -6,27 +6,90 @@ import (
 	"github.com/bufsnake/blueming/config"
 	"github.com/bufsnake/blueming/internal/core"
 	"github.com/bufsnake/blueming/pkg/log"
+	"golang.org/x/text/encoding/simplifiedchinese"
 	"io/ioutil"
+	"math"
 	"net/url"
 	"os"
+	"os/exec"
+	"runtime"
 	"strings"
+	"sync"
+	"syscall"
 	"time"
 )
 
 func main() {
 	conf := config.Config{}
-	flag.IntVar(&conf.Thread, "t", 10, "set thread")
+	flag.IntVar(&conf.Thread, "t", 100, "set thread")
 	flag.IntVar(&conf.Timeout, "s", 10, "set timeout")
 	flag.StringVar(&conf.Url, "u", "", "set url")
 	flag.StringVar(&conf.Urlfile, "f", "", "set url file")
 	flag.StringVar(&conf.Loglevel, "l", log.DEBUG, "set log level(trace,debug,info,warn,fatal)")
 	flag.StringVar(&conf.Wordlist, "w", "", "set wordlist")
-	flag.StringVar(&conf.Index, "i", "", "set wordlist index(ex: test.php)")
-	flag.StringVar(&conf.Proxy, "p", "", "set download proxy")
-	flag.StringVar(&conf.ExcludeStatus, "es", "404", "dirscan filter status(200,206,301,302,307,400,401,402,403,404,405,406,424,500,501,502,503,504,600,etc.)")
-	flag.StringVar(&conf.ResultFile, "rf", "", "parse result file")
+	flag.StringVar(&conf.Index, "i", "", "set wordlist index(exp: test.php)")
+	flag.StringVar(&conf.Proxy, "p", "", "set proxy, support http proxy(exp: http://localhost:8080)")
+	flag.StringVar(&conf.Listen, "listen", "127.0.0.1:9099", "listen to scan dir")
+	flag.StringVar(&conf.URLStrs, "urls", "", "set url file")
+	flag.StringVar(&conf.Cert, "crt", "ca.crt", "listen cert")
+	flag.StringVar(&conf.Key, "key", "ca.key", "listen key")
+	flag.BoolVar(&conf.FilterOutput, "b", false, "filter output data")
+	// 暂不考虑
+	//flag.StringVar(&conf.ResultFile, "rf", "", "parse result file")
 	flag.Parse()
+	// 开启多核模式
+	runtime.GOMAXPROCS(runtime.NumCPU() * 3 / 4)
+	// 关闭 GIN Debug模式
+	// 设置工具可打开的文件描述符
+	var rLimit syscall.Rlimit
+	rLimit.Max = 999999
+	rLimit.Cur = 999999
+	if runtime.GOOS == "darwin" {
+		rLimit.Cur = 10240
+	}
+	err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	_ = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 	log.SetLevel(conf.Loglevel)
+	if conf.FilterOutput {
+		// 获取 output 下的所有文件 不包含文件夹
+		allfiles, _ := ioutil.ReadDir("./output")
+		for _,f := range allfiles {
+			if !f.IsDir() {
+				if f.Size() <= 1048576 {
+					err = os.Remove("./output/" + f.Name())
+					if err != nil {
+						log.Fatal(err)
+					}
+				}
+			}
+		}
+
+		wait := sync.WaitGroup{}
+		files, _ := ioutil.ReadDir("./output")
+		fmt.Println("current exist", len(files), "files")
+		go func() {
+			for {
+				fmt.Printf("\r%.2f%%", math.Trunc(((increase/float64(len(files)))*100)*1e2)*1e-2)
+				time.Sleep(1 * time.Second / 10)
+			}
+		}()
+		for _, f := range files {
+			if !f.IsDir() {
+				wait.Add(1)
+				go filter(&wait, strings.ReplaceAll("./output/"+f.Name(), " ", ` `), float64(len(files)))
+			} else {
+				increaseAdd()
+				fmt.Printf("\r%.2f%%", math.Trunc(((increase/float64(len(files)))*100)*1e2)*1e-2)
+			}
+		}
+		wait.Wait()
+		// function filter { if [[ $(file $1 | grep $1": data") == "" && $(file $1 | grep "image data") == "" && $(file $1 | grep "HTML") == "" && $(file $1 | grep "empty") == "" && $(file $1 | grep "JSON") == "" && $(file $1 | grep "text") == "" ]]; then file $1; else rm -rf $1; fi } && filter logs/data.tar.gz
+		os.Exit(1)
+	}
 	urls := []string{}
 	if conf.Url != "" {
 		urls = append(urls, conf.Url)
@@ -88,12 +151,43 @@ func main() {
 			}
 		}
 		return
+	} else if conf.Listen != "" {
+		if conf.Wordlist == "" {
+			log.Fatal("If passive scanning is started, a dictionary must be specified")
+		}
+		if conf.URLStrs == "" {
+			log.Fatal("urls must be specified")
+		}
+		passive := core.NewPassive(conf)
+		err = passive.Start()
+		if err != nil {
+			log.Fatal(err)
+		}
 	} else {
 		flag.Usage()
 		return
 	}
+	// 判断 output 文件夹是否存在
+	if !exists("./output") {
+		log.Info("create output file path")
+		err := os.Mkdir("./output/", os.ModePerm)
+		if err != nil {
+			log.Warn("create output file path error", err)
+			os.Exit(1)
+		}
+	}
+	// 创建 Log 文件夹
+	if !exists("./logs") {
+		log.Info("create logs file path")
+		err := os.Mkdir("./logs/", os.ModePerm)
+		if err != nil {
+			log.Warn("create logs file path error", err)
+			os.Exit(1)
+		}
+	}
+
 	log.Info(len(urls), "个URL,", conf.Thread, "线程,", conf.Timeout, "超时")
-	config.LogFileName = "Log-" + time.Now().Format("2006-01-02 15:04:05")
+	config.LogFileName = "./logs/Log-" + time.Now().Format("2006-01-02 15:04:05")
 	create, err := os.Create(config.LogFileName)
 	if err != nil {
 		log.Warn(err)
@@ -105,3 +199,43 @@ func main() {
 	newCore := core.NewCore(urls, conf)
 	newCore.Core()
 }
+
+func exists(path string) bool {
+	_, err := os.Stat(path) //os.Stat获取文件信息
+	if err != nil {
+		if os.IsExist(err) {
+			return true
+		}
+		return false
+	}
+	return true
+}
+
+var increase float64 = 0
+var inc_l sync.Mutex
+
+func increaseAdd() {
+	inc_l.Lock()
+	defer inc_l.Unlock()
+	increase++
+}
+
+func filter(wait *sync.WaitGroup, filename string, totalcount float64) {
+	defer wait.Done()
+	bin := []string{"-c", "function filter { if [[ $(file $1 | grep $1\": data\") == \"\" && $(file $1 | grep \"image data\") == \"\" && $(file $1 | grep \"HTML\") == \"\" && $(file $1 | grep \"empty\") == \"\" && $(file $1 | grep \"JSON\") == \"\" && $(file $1 | grep \"text\") == \"\" ]]; then file $1; else rm -rf $1; fi } && filter '" + filename + "'"}
+	// 其他的shell环境太烦了
+	run := exec.Command("/bin/zsh", bin...)
+	output, err := run.Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	output, err = simplifiedchinese.GB18030.NewDecoder().Bytes(output)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(output) != 0 {
+		fmt.Print("\r" + string(output))
+	}
+	increaseAdd()
+	fmt.Printf("\r%.2f%%", math.Trunc(((increase/totalcount)*100)*1e2)*1e-2)
+}

config/main.go

@@ -1,16 +1,28 @@
 package config
 
 type Config struct {
-	Thread        int
-	Timeout       int
-	Url           string
-	Urlfile       string
-	Loglevel      string
-	Wordlist      string
-	Index         string
-	Proxy         string
-	ExcludeStatus string
-	ResultFile    string
+	Thread       int
+	Timeout      int
+	Url          string
+	Urlfile      string
+	Loglevel     string
+	Wordlist     string
+	Index        string
+	Proxy        string
+	ResultFile   string
+	FilterOutput bool // 过滤 output 文件夹中的垃圾数据
+	Listen       string
+	URLStrs      string
+	Cert         string
+	Key          string
 }
 
 var LogFileName string
+
+type HTTPStatus struct {
+	URL         string
+	Status      int
+	ContentType string
+	Size        string
+	Body        string
+}

go.mod

@@ -2,4 +2,10 @@ module github.com/bufsnake/blueming
 
 go 1.14
 
-require github.com/logrusorgru/aurora v2.0.3+incompatible
+require (
+	github.com/antlabs/strsim v0.0.2
+	github.com/google/martian v2.1.0+incompatible
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/weppos/publicsuffix-go v0.15.0
+	golang.org/x/text v0.3.0
+)

go.sum

@@ -1,2 +1,20 @@
+github.com/antlabs/strsim v0.0.2 h1:R4qjokEegYTrw+fkcYj3/UndG9Cn136fH+fpw9TIz9k=
+github.com/antlabs/strsim v0.0.2/go.mod h1:95XAAF2dJK9IiZMc0Ue6H9t477/i6fvYoMoeey8sEnc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ=
+github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=