diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d891ae0..57cd752 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,7 +13,7 @@ jobs: - name: Install Golang uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 # 1.18 后会出现tls: server selected unsupported protocol version 301错误 - name: Get Current Date id: date diff --git a/cmd/blueming/main.go b/cmd/blueming/main.go index e618e62..f30720e 100644 --- a/cmd/blueming/main.go +++ b/cmd/blueming/main.go @@ -41,11 +41,12 @@ func main() { conf := config.Config{} var blueming = &cobra.Command{ Use: "blueming", - CompletionOptions: struct { - DisableDefaultCmd bool - DisableNoDescFlag bool - DisableDescriptions bool - }{DisableDefaultCmd: true, DisableNoDescFlag: true, DisableDescriptions: true}, + } + blueming.CompletionOptions = cobra.CompletionOptions{ + DisableDefaultCmd: true, + DisableNoDescFlag: true, + DisableDescriptions: true, + HiddenDefaultCmd: true, } var backupscan = &cobra.Command{ Use: "backupscan", @@ -98,7 +99,7 @@ func main() { backupscan.Flags().IntVarP(&conf.Timeout, "timeout", "s", 10, "set up timeout") backupscan.Flags().StringVarP(&conf.Loglevel, "log-level", "v", log.DEBUG, "set up log level(trace,debug,info,warn,fatal)") backupscan.Flags().BoolVarP(&conf.FilterOutput, "filter-output", "f", false, "empty junk data, must be used for backup scan") - + var dirscan = &cobra.Command{ Use: "dirscan", Short: "dirscan scan", @@ -117,7 +118,7 @@ func main() { dirscan.Flags().IntVarP(&conf.Timeout, "timeout", "s", 10, "set up timeout") dirscan.Flags().StringVarP(&conf.Wordlist, "wordlist", "w", "", "set up wordlist") dirscan.Flags().StringVarP(&conf.Index, "index", "i", "", "set the starting position of the dictionary(-i /test.php)") - + var passive = &cobra.Command{ Use: "passive", Short: "passive scan", @@ -148,7 +149,7 @@ func main() { log.Fatal(err) } log.SetLevel(conf.Loglevel) - + urls := []string{} if conf.Url != "" { urls = append(urls, conf.Url) @@ -168,7 +169,7 @@ func main() { if len(urls) == 0 { return } - + // 判断 output 文件夹是否存在 if !exists("./output") { log.Info("create output file path") @@ -178,7 +179,7 @@ func main() { os.Exit(1) } } - + // 创建 Log 文件夹 if !exists("./logs") { log.Info("create logs file path") @@ -188,9 +189,9 @@ func main() { os.Exit(1) } } - + log.Info(len(urls), "个URL,", conf.Thread, "线程,", conf.Timeout, "超时") - + config.LogFileName = "./logs/Log-" + time.Now().Format("2006-01-02 15:04:05") create, err := os.Create(config.LogFileName) if err != nil { @@ -200,7 +201,7 @@ func main() { if err != nil { log.Warn(err) } - + newCore := core.NewCore(urls, conf) newCore.Core() } diff --git a/go.mod b/go.mod index 4be4f23..e59c5f5 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,22 @@ go 1.14 require ( github.com/antlabs/strsim v0.0.2 + github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/martian v2.1.0+incompatible + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/kr/pretty v0.3.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/spf13/cobra v1.2.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/spf13/cobra v1.4.0 + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.8.1 // indirect + github.com/stretchr/testify v1.7.2 // indirect github.com/weppos/publicsuffix-go v0.15.0 - golang.org/x/text v0.3.5 + golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect + golang.org/x/text v0.3.7 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f667382..559a1b2 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -173,9 +175,14 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -195,6 +202,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -203,7 +211,11 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -213,6 +225,8 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -225,6 +239,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ= github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= @@ -325,6 +341,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -389,7 +407,10 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -398,6 +419,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -561,6 +584,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -570,6 +595,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/core/core.go b/internal/core/core.go index 8a9901d..84a785b 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -12,7 +12,6 @@ import ( . "github.com/logrusorgru/aurora" "github.com/weppos/publicsuffix-go/publicsuffix" "io/ioutil" - "net/http" url2 "net/url" "os" "regexp" @@ -78,7 +77,7 @@ func (c *core) dirscan() { } for i := 0; i < len(c.url); i++ { for req, _ := range c.domain_path(c.url[i]) { - httpc <- strings.Trim(c.url[i],"/")+"/"+req + httpc <- strings.Trim(c.url[i], "/") + "/" + req } } length := general_file_name.InitGeneral(c.wordlist) @@ -165,18 +164,12 @@ func (c *core) httprequest(wait *sync.WaitGroup, httpc chan string, httpd chan c body string err error ) - if c.config.Wordlist == "" { // 备份文件扫描 - status, ct, size, body, err = http_request.HTTPRequest(http.MethodHead, url, c.config.Proxy, timeout) - } else { // 目录扫描 - status, ct, size, body, err = http_request.HTTPRequest(http.MethodGet, url, c.config.Proxy, timeout) - } - log.Trace(status, ct, size, body, err, url) - if err != nil && c.config.Wordlist == "" { - log.Warn(err) - } - if err != nil && strings.Contains(err.Error(), "proxyconnect tcp") { + status, ct, size, body, err = http_request.HTTPRequest(url, c.config.Proxy, timeout) + if err != nil { log.Warn(err) + continue } + log.Trace(status, ct, size, body, err, url) if c.config.Wordlist != "" { if status == 404 || status == 0 { continue @@ -231,21 +224,22 @@ func (c *core) httprequest(wait *sync.WaitGroup, httpc chan string, httpd chan c pr = append(pr, "["+ct+"]") } fmt.Println(pr...) - } else { - if status != 200 && status != 206 { - continue - } - if size == "0B" || size == "0.0B" { - continue - } - matchString, err := regexp.MatchString("application/[-\\w.]+", ct) - if err == nil && matchString { - log.Info(size, ct, url) - httpd <- config.HTTPStatus{ - URL: url, - Size: size, - ContentType: ct, - } + continue + } + if status != 200 && status != 206 { + continue + } + if size == "0B" || size == "0.0B" { + continue + } + var match bool + match, err = regexp.MatchString("application/[-\\w.]+", ct) + if err == nil && match { + log.Info(size, ct, url) + httpd <- config.HTTPStatus{ + URL: url, + Size: size, + ContentType: ct, } } } diff --git a/pkg/http-request/main.go b/pkg/http-request/main.go index 8b15459..3f3547d 100644 --- a/pkg/http-request/main.go +++ b/pkg/http-request/main.go @@ -3,93 +3,78 @@ package http_request import ( "crypto/tls" "fmt" - "github.com/bufsnake/blueming/config" - "github.com/bufsnake/blueming/pkg/log" "github.com/bufsnake/blueming/pkg/useragent" + "io" "io/ioutil" + "mime" "net/http" - url2 "net/url" - "os" + "net/url" "time" ) -func HTTPRequest(method, url, proxyx string, timeout int) (status int, contenttype, size string, body string, err error) { - client := &http.Client{ - Timeout: time.Duration(timeout) * time.Second, - Transport: nil, +func HTTPRequest(urlstr, proxyx string, timeout int) (status int, contenttype, size string, body string, err error) { + cli := &http.Client{ + Timeout: time.Duration(timeout) * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } - transport := &http.Transport{ + transport := http.Transport{ DisableKeepAlives: true, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } if proxyx != "" { - proxy, err := url2.Parse(proxyx) - if err != nil { - return 0, "", "", "", err + transport.Proxy = func(request *http.Request) (*url.URL, error) { + return url.Parse(proxyx) } - transport.Proxy = http.ProxyURL(proxy) - client.Transport = transport - } else { - client.Transport = transport } - req, err := http.NewRequest(method, url, nil) + cli.Transport = &transport + req, err := http.NewRequest(http.MethodGet, urlstr, nil) if err != nil { return 0, "", "0B", "", err } - if method == http.MethodGet { - req.Header.Add("Range", "bytes=0-8100") - } req.Header.Add("Accept", "*/*") - req.Header.Add("Referer", "http://www.baidu.com") + req.Header.Add("Referer", "https://www.baidu.com/") req.Header.Add("Connection", "close") req.Header.Add("Cache-Control", "no-cache") req.Header.Add("User-Agent", useragent.RandomUserAgent()) - do, err := client.Do(req) + res, err := cli.Do(req) if err != nil { return 0, "", "0B", "", err } - defer do.Body.Close() - all, err := ioutil.ReadAll(do.Body) - flag := false + defer res.Body.Close() + if res.StatusCode == 101 { + return 0, "", "0B", "", err + } + resbody := make([]byte, 0) + resbody, err = ioutil.ReadAll(io.LimitReader(res.Body, 1024*1024*2)) if err != nil { - flag = true + return 0, "", "0B", "", err + } + content_length := float64(res.ContentLength) + if content_length < 1 { + content_length = float64(len(resbody)) } - temp := float64(do.ContentLength) SIZE := []string{"B", "K", "M", "G", "T"} i := 0 for { - if temp < 1024 { + if content_length < 1024 { break } - temp = temp / 1024.0 + content_length = content_length / 1024.0 i++ } length := "" if i > len(SIZE) { - length = fmt.Sprintf("%0.1fX", temp) + length = fmt.Sprintf("%0.1fX", content_length) } else { - length = fmt.Sprintf("%0.1f%s", temp, SIZE[i]) + length = fmt.Sprintf("%0.1f%s", content_length, SIZE[i]) } - if do.ContentLength > 104857600 { - file, err := os.OpenFile(config.LogFileName, os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - log.Warn(err) - } - _, err = file.WriteString(do.Header.Get("Content-Type") + " " + length + " " + url + "\n") - if err != nil { - file.Close() - log.Warn(err) - } - file.Close() - return 0, "", "0B", "", err - } - if flag { // GET 获取 body 失败时 - return do.StatusCode, do.Header.Get("Content-Type"), length, "", nil + contenttype, _, err = mime.ParseMediaType(res.Header.Get("Content-Type")) + if err != nil { + contenttype = res.Header.Get("Content-Type") } - return do.StatusCode, do.Header.Get("Content-Type"), length, string(all), nil + return res.StatusCode, contenttype, length, string(resbody), nil }