dirscan add domain path scan

.github/workflows/go.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v2
 

.gitignore

@@ -0,0 +1,4 @@
+cmd/blueming/blueming
+cmd/blueming/output
+cmd/blueming/log
+cmd/blueming/wordlists

README.md

@@ -50,6 +50,9 @@ Use "blueming [command] --help" for more information about a command.
 > 基本满足以下要求即可
 
 - [ ] 常见文件泄露扫描 .git .hg .idea .DS_Store ...
+- [x] 提取域名关键字进行目录扫描
+- [ ] 日志文件扫描: 指定扫描地址，计算头一天的日期，根据日期生成字典，可定制日期出现的位置
+  "image$TIME$" "pay.$time$"
 - [x] 开启被动扫描模式，配合httpx自动进行目录扫描(二级、三级、四级...)
 - [x] 通过URL自动生成文件名
 - [x] 根据后缀名将URL定义为对应的文件格式，如zip、tar.gz等

internal/core/core.go

@@ -8,7 +8,9 @@ import (
 	general_file_name "github.com/bufsnake/blueming/pkg/general-file-name"
 	http_request "github.com/bufsnake/blueming/pkg/http-request"
 	"github.com/bufsnake/blueming/pkg/log"
+	"github.com/bufsnake/blueming/pkg/parseip"
 	. "github.com/logrusorgru/aurora"
+	"github.com/weppos/publicsuffix-go/publicsuffix"
 	"io/ioutil"
 	"net/http"
 	url2 "net/url"
@@ -74,6 +76,11 @@ func (c *core) dirscan() {
 		httpr.Add(1)
 		go c.httprequest(&httpr, httpc, nil, c.config.Timeout)
 	}
+	for i := 0; i < len(c.url); i++ {
+		for req, _ := range c.domain_path(c.url[i]) {
+			httpc <- strings.Trim(c.url[i],"/")+"/"+req
+		}
+	}
 	length := general_file_name.InitGeneral(c.wordlist)
 	for w := 0; w < length; w++ {
 		for i := 0; i < len(c.url); i++ {
@@ -266,3 +273,60 @@ func (c *core) httpdownload(wait *sync.WaitGroup, httpd chan config.HTTPStatus)
 		}
 	}
 }
+
+func (c *core) domain_path(urlstr string) map[string]bool {
+	domain_paths := make(map[string]bool, 0)
+	if !isDomain(urlstr) {
+		return domain_paths
+	}
+	parse, err := url2.Parse(urlstr)
+	if err != nil {
+		return domain_paths
+	}
+	if strings.Contains(parse.Host, ":") {
+		parse.Host = strings.Split(parse.Host, ":")[0]
+	}
+	domain, err := publicsuffix.Domain(parse.Host)
+	if err != nil {
+		return domain_paths
+	}
+	parse.Host = strings.ReplaceAll(parse.Host, domain, "")
+	labels := publicsuffix.Labels(parse.Host)
+	for i := 0; i < len(labels); i++ {
+		labels[i] = strings.Trim(labels[i], " \r\n\t")
+		if labels[i] == "" {
+			continue
+		}
+		domain_paths[labels[i]] = true
+		if !strings.Contains(labels[i], "-") {
+			continue
+		}
+		subword := strings.Split(labels[i], "-")
+		for j := 0; j < len(subword); j++ {
+			subword[j] = strings.Trim(subword[j], " \r\n\t")
+			if subword[j] == "" {
+				continue
+			}
+			domain_paths[subword[j]] = true
+		}
+	}
+	return domain_paths
+}
+
+func isDomain(str string) bool {
+	if matched, _ := regexp.MatchString("\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}", str); matched {
+		host := strings.ReplaceAll(strings.ReplaceAll(str, "http://", ""), "https://", "")
+		if strings.Contains(host, "/") {
+			host = strings.Split(host, "/")[0]
+		}
+		if strings.Contains(host, ":") {
+			host = strings.Split(host, ":")[0]
+		}
+		_, _, err := parseip.ParseIP(host)
+		if err != nil {
+			return true
+		}
+		return false
+	}
+	return true
+}