From d2a3cb29e9d0d9e4771de66127b09e6e44362b35 Mon Sep 17 00:00:00 2001 From: poq79 <273589250@qq.com> Date: Wed, 10 May 2023 17:31:27 +0800 Subject: [PATCH 01/10] home page screenshot --- README.md | 1 + README_ZH.md | 1 + cmd/detect.go | 59 +++++++++++++------- go.mod | 13 ++++- go.sum | 22 ++++++++ utils/screenshot.go | 132 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 20 deletions(-) create mode 100644 utils/screenshot.go diff --git a/README.md b/README.md index 19b311d..918e889 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Flags: tions to specify more than one site. -t, --timeout int Time (in seconds) to wait for response to requests (defa ult 10) + -S --screenshot screenshot the userpage and save Global Flags: -v, --verbose verbose output diff --git a/README_ZH.md b/README_ZH.md index 69d9ea5..20dd3a6 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -43,6 +43,7 @@ Flags: tions to specify more than one site. -t, --timeout int Time (in seconds) to wait for response to requests (defa ult 10) + -S --screenshot screenshot the userpage and save Global Flags: -v, --verbose verbose output diff --git a/cmd/detect.go b/cmd/detect.go index b9e4d74..0fa412f 100644 --- a/cmd/detect.go +++ b/cmd/detect.go @@ -3,30 +3,32 @@ package cmd import ( "DetectDee/utils" "fmt" - "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/tidwall/gjson" "io/ioutil" "regexp" "strings" "sync" "time" + + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/tidwall/gjson" ) type detectArgsType struct { - name []string - site []string - check bool - proxy string - timeout int - isNSFW bool - precisely bool - retry int - file string - google bool - email []string - phone []string + name []string + site []string + check bool + proxy string + timeout int + isNSFW bool + precisely bool + retry int + file string + google bool + email []string + phone []string + screenshot bool //unique bool } @@ -42,6 +44,8 @@ var ( searchInfo = "[+] %-15s %-15s: Please check in %s\n" disableSiteInfo = "[!] %-15s %-15s: data of %s is temporarily unavailable\n" nsfwInfo = "[!] %-15s %-15s: %s is nsfw\n" + safeFolderName = "defaultScreenshotFolder" + existUserPage = false ) func init() { @@ -58,6 +62,7 @@ func init() { detectCmd.Flags().BoolVar(&detectArgs.precisely, "precisely", false, "Check precisely") detectCmd.Flags().StringVarP(&detectArgs.file, "file", "f", "data.json", "Site data file") detectCmd.Flags().BoolVarP(&detectArgs.google, "google", "g", false, "Show google search result") + detectCmd.Flags().BoolVarP(&detectArgs.screenshot, "screenshot", "S", false, "screenshot the userpage and save") //detectCmd.Flags().BoolVar(&detectArgs.unique, "unique", false, "Make new requests client for each site") rootCmd.AddCommand(detectCmd) @@ -292,20 +297,36 @@ func detectUser(name, site string, requestTimes, retryTimes, detectCount int, fl userPage := detectData.Get("userPage").String() if strings.Contains(userPage, "%s") { + existUserPage = true userPage = fmt.Sprintf(userPage, name) + } else { + existUserPage = false + } + if detectArgs.screenshot { + safeFolderName = utils.SanitizeFilename(name + "_userpage") + utils.CreateDirIfNotExists("./" + safeFolderName) } - // precisely mode if !*flag { log.Debugf(nonExistInfo, name, site) return false } else if !detectArgs.precisely { // flag=true && precisely=false - log.Infof(existInfo, name, site, userPage) + if detectArgs.screenshot && existUserPage { + log.Infof(existInfo, name, site, userPage+" [screenshot]") + utils.DoScreenshot(userPage, safeFolderName) + } else { + log.Infof(existInfo, name, site, userPage) + } return false } else if requestTimes == detectCount { // flag=true && precisely=true && last request - log.Infof(existInfo, name, site, userPage) + if detectArgs.screenshot && existUserPage { + log.Infof(existInfo, name, site, userPage+" [screenshot]") + utils.DoScreenshot(userPage, safeFolderName) + } else { + log.Infof(existInfo, name, site, userPage) + } return true } else { return true diff --git a/go.mod b/go.mod index a8e36ed..35bdc68 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,23 @@ require ( github.com/tidwall/gjson v1.14.4 ) +require ( + github.com/chromedp/sysutil v1.0.0 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect +) + require ( github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b + github.com/chromedp/chromedp v0.9.1 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect + golang.org/x/sys v0.6.0 // indirect ) diff --git a/go.sum b/go.sum index 0c34fc3..bb9a5e4 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,33 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b h1:SK9lKpfJgkHu8VzAGGiRWm6IJ8EDo0OVjKqihL7KwEI= +github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA= +github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ= +github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -43,6 +62,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -50,6 +70,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/utils/screenshot.go b/utils/screenshot.go new file mode 100644 index 0000000..283568d --- /dev/null +++ b/utils/screenshot.go @@ -0,0 +1,132 @@ +package utils + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "math" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" +) + +func DoScreenshot(url string, safeFolderName string) { + + opts := append( + chromedp.DefaultExecAllocatorOptions[:], + chromedp.NoDefaultBrowserCheck, + chromedp.Flag("headless", true), + chromedp.Flag("blink-settings", "imagesEnabled=true"), + chromedp.Flag("ignore-certificate-errors", true), + chromedp.Flag("disable-web-security", true), + chromedp.Flag("disable-extensions", true), + chromedp.Flag("disable-default-apps", true), + chromedp.WindowSize(1920, 1080), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("hide-scrollbars", true), + chromedp.Flag("mute-audio", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("no-default-browser-check", true), + chromedp.NoFirstRun, + chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"), + ) + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancel() + + ctx, cancel := chromedp.NewContext( + allocCtx, + chromedp.WithLogf(log.Printf), + ) + defer cancel() + + ctx, cancel = context.WithTimeout(ctx, 50*time.Second) + defer cancel() + + var buf []byte + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(ctx, fullScreenshot(url, 100, &buf)); err != nil { + log.Fatal(err) + } + if err := ioutil.WriteFile("./"+safeFolderName+"/"+getdomain(url)+".png", buf, 0644); err != nil { + log.Fatal(err) + } + // log.Println("screenshot") + +} + +func fullScreenshot(urlstr string, quality int64, res *[]byte) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Navigate(urlstr), + //chromedp.WaitVisible("style"), + chromedp.Sleep(20 * time.Second), + //chromedp.OuterHTML(`document.querySelector("body")`, &htmlContent, chromedp.ByJSPath), + chromedp.ActionFunc(func(ctx context.Context) error { + _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + if err != nil { + return err + } + + width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + + err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + WithScreenOrientation(&emulation.ScreenOrientation{ + Type: emulation.OrientationTypePortraitPrimary, + Angle: 0, + }). + Do(ctx) + if err != nil { + return err + } + + *res, err = page.CaptureScreenshot(). + WithQuality(quality). + WithClip(&page.Viewport{ + X: contentSize.X, + Y: contentSize.Y, + Width: contentSize.Width, + Height: contentSize.Height, + Scale: 1, + }).Do(ctx) + if err != nil { + return err + } + return nil + }), + } +} + +func getdomain(urlstr string) string { + u, err := url.Parse(urlstr) + if err != nil { + fmt.Println(err) + return "error" + } + return u.Hostname() +} + +func SanitizeFilename(filename string) string { + re := regexp.MustCompile(`[\\/:*?"<>|]|`) + filename = re.ReplaceAllString(filename, "") + filename = strings.TrimSpace(filename) + return filename +} + +func CreateDirIfNotExists(dirName string) error { + if _, err := os.Stat(dirName); os.IsNotExist(err) { + err := os.MkdirAll(dirName, 0755) + if err != nil { + return err + } + // fmt.Printf("Folder \"%s\" created.\n", dirName) + } else { + // fmt.Printf("Folder \"%s\" already exists.\n", dirName) + } + return nil +} From d8c7a0a6bb33554e392972d64e30fb8492addb89 Mon Sep 17 00:00:00 2001 From: poq79 <273589250@qq.com> Date: Sat, 13 May 2023 15:23:15 +0800 Subject: [PATCH 02/10] Reconstruct screenshot module --- README.md | 1 - README_ZH.md | 1 - cmd/detect.go | 104 ++++++++++----------- cmd/smirror.go | 212 +++++++++++++++++++++++++++++++++++++++++++ data.json | 3 +- go.mod | 2 +- go.sum | 3 +- utils/writeToFile.go | 20 ++++ 8 files changed, 288 insertions(+), 58 deletions(-) create mode 100644 cmd/smirror.go create mode 100644 utils/writeToFile.go diff --git a/README.md b/README.md index 918e889..19b311d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ Flags: tions to specify more than one site. -t, --timeout int Time (in seconds) to wait for response to requests (defa ult 10) - -S --screenshot screenshot the userpage and save Global Flags: -v, --verbose verbose output diff --git a/README_ZH.md b/README_ZH.md index 20dd3a6..69d9ea5 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -43,7 +43,6 @@ Flags: tions to specify more than one site. -t, --timeout int Time (in seconds) to wait for response to requests (defa ult 10) - -S --screenshot screenshot the userpage and save Global Flags: -v, --verbose verbose output diff --git a/cmd/detect.go b/cmd/detect.go index 0fa412f..8059a3b 100644 --- a/cmd/detect.go +++ b/cmd/detect.go @@ -3,49 +3,49 @@ package cmd import ( "DetectDee/utils" "fmt" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/tidwall/gjson" "io/ioutil" "regexp" "strings" "sync" "time" - - "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/tidwall/gjson" ) type detectArgsType struct { - name []string - site []string - check bool - proxy string - timeout int - isNSFW bool - precisely bool - retry int - file string - google bool - email []string - phone []string - screenshot bool + name []string + site []string + check bool + proxy string + timeout int + isNSFW bool + precisely bool + retry int + file string + google bool + email []string + phone []string + output string //unique bool } var ( - detectArgs detectArgsType - wg sync.WaitGroup - nonSiteData = "[-] There is no site data of %s\n" - existInfo = "[+] %-15s %-15s: %s\n" - nonExistInfo = "[-] %-15s %-15s: non exists\n" - reqErrorInfo = "[!] %-15s %-15s: %s requests error, retry %d/%d\n" - sleepMap = make(map[string]int64) - sleepChannel = make(map[string]chan bool) - searchInfo = "[+] %-15s %-15s: Please check in %s\n" - disableSiteInfo = "[!] %-15s %-15s: data of %s is temporarily unavailable\n" - nsfwInfo = "[!] %-15s %-15s: %s is nsfw\n" - safeFolderName = "defaultScreenshotFolder" - existUserPage = false + detectArgs detectArgsType + wg sync.WaitGroup + nonSiteData = "[-] There is no site data of %s\n" + existInfo = "[+] %-15s %-15s: %s\n" + nonExistInfo = "[-] %-15s %-15s: non exists\n" + reqErrorInfo = "[!] %-15s %-15s: %s requests error, retry %d/%d\n" + sleepMap = make(map[string]int64) + sleepChannel = make(map[string]chan bool) + searchInfo = "[+] %-15s %-15s: Please check in %s\n" + disableSiteInfo = "[!] %-15s %-15s: data of %s is temporarily unavailable\n" + detectCompletedInfo = "[+] Detect completed, save to %s\n" + nsfwInfo = "[!] %-15s %-15s: %s is nsfw\n" + writeContent = make(chan string) + writeDone = make(chan bool) ) func init() { @@ -62,7 +62,7 @@ func init() { detectCmd.Flags().BoolVar(&detectArgs.precisely, "precisely", false, "Check precisely") detectCmd.Flags().StringVarP(&detectArgs.file, "file", "f", "data.json", "Site data file") detectCmd.Flags().BoolVarP(&detectArgs.google, "google", "g", false, "Show google search result") - detectCmd.Flags().BoolVarP(&detectArgs.screenshot, "screenshot", "S", false, "screenshot the userpage and save") + detectCmd.Flags().StringVarP(&detectArgs.output, "output", "o", "result.txt", "Result file") //detectCmd.Flags().BoolVar(&detectArgs.unique, "unique", false, "Make new requests client for each site") rootCmd.AddCommand(detectCmd) @@ -76,12 +76,23 @@ var detectCmd = &cobra.Command{ } func detect(_ *cobra.Command, _ []string) { - log.Infoln("Detect for", detectArgs.name) + if len(detectArgs.name) != 0 { + log.Infoln("Detect for", detectArgs.name) + } + if len(detectArgs.email) != 0 { + log.Infoln("Detect for", detectArgs.email) + } + if len(detectArgs.phone) != 0 { + log.Infoln("Detect for", detectArgs.phone) + } + if Verbose { log.Infoln("Debug Mode") log.SetLevel(log.DebugLevel) } + go utils.WriteToFile(detectArgs.output, writeContent, writeDone) + log.Debugln(detectArgs) siteData, err := ioutil.ReadFile(detectArgs.file) if err != nil { @@ -139,7 +150,10 @@ func detect(_ *cobra.Command, _ []string) { } } } + wg.Wait() + writeDone <- true + log.Infof(detectCompletedInfo, detectArgs.output) } func detectSite(name, site, nameType string, siteBody gjson.Result) { @@ -297,36 +311,22 @@ func detectUser(name, site string, requestTimes, retryTimes, detectCount int, fl userPage := detectData.Get("userPage").String() if strings.Contains(userPage, "%s") { - existUserPage = true userPage = fmt.Sprintf(userPage, name) - } else { - existUserPage = false - } - if detectArgs.screenshot { - safeFolderName = utils.SanitizeFilename(name + "_userpage") - utils.CreateDirIfNotExists("./" + safeFolderName) } + // precisely mode if !*flag { log.Debugf(nonExistInfo, name, site) return false } else if !detectArgs.precisely { // flag=true && precisely=false - if detectArgs.screenshot && existUserPage { - log.Infof(existInfo, name, site, userPage+" [screenshot]") - utils.DoScreenshot(userPage, safeFolderName) - } else { - log.Infof(existInfo, name, site, userPage) - } + log.Infof(existInfo, name, site, userPage) + writeContent <- fmt.Sprintf(existInfo, name, site, userPage) return false } else if requestTimes == detectCount { // flag=true && precisely=true && last request - if detectArgs.screenshot && existUserPage { - log.Infof(existInfo, name, site, userPage+" [screenshot]") - utils.DoScreenshot(userPage, safeFolderName) - } else { - log.Infof(existInfo, name, site, userPage) - } + log.Infof(existInfo, name, site, userPage) + writeContent <- fmt.Sprintf(existInfo, name, site, userPage) return true } else { return true diff --git a/cmd/smirror.go b/cmd/smirror.go new file mode 100644 index 0000000..92d9a30 --- /dev/null +++ b/cmd/smirror.go @@ -0,0 +1,212 @@ +package cmd + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "math" + "net/url" + "os" + "strings" + "sync" + "time" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type mirrorArgsType struct { + proxy string + chrome bool + thread int + timeout int + path string + file string + result string +} + +var ( + mirrorArgs mirrorArgsType + mirrorWg sync.WaitGroup + targets = make(chan string, 1000) + execPathNotFound = "[-] Chrome executable file not found, please install Chrome or specify the chrome.exe path with --path" + urlFileReadError = "[-] failed to read targets file:%v" + createResultFolderError = "[-] failed to create result folder:%v" +) + +func init() { + mirrorCmd.Flags().StringVar(&mirrorArgs.proxy, "proxy", "", "Make requests over a proxy. e.g. socks5://127.0.0.1:1080") + mirrorCmd.Flags().StringVar(&mirrorArgs.path, "path", "", "Chrome ExecPath") + mirrorCmd.Flags().StringVarP(&mirrorArgs.result, "result", "r", "screenshots", "Folder path of the screenshot") + mirrorCmd.Flags().BoolVar(&mirrorArgs.chrome, "chrome", false, "Show chrome") + mirrorCmd.Flags().IntVarP(&mirrorArgs.thread, "thread", "t", 3, "Chrome number") + mirrorCmd.Flags().StringVarP(&mirrorArgs.file, "file", "f", "result.txt", "Url list file") + mirrorCmd.Flags().IntVar(&mirrorArgs.timeout, "timeout", 60, "Timeout") + rootCmd.AddCommand(mirrorCmd) +} + +var mirrorCmd = &cobra.Command{ + Use: "mirror", + Short: "mirror data.json using github", + Long: ``, + Run: mirror, +} + +func mirror(_ *cobra.Command, _ []string) { + if Verbose { + log.Infoln("Debug Mode") + log.SetLevel(log.DebugLevel) + } + file, err := os.Open(mirrorArgs.file) + if err != nil { + log.Errorf(urlFileReadError, err) + return + } + defer file.Close() + err = CreateDirIfNotExists(mirrorArgs.result) + if err != nil { + log.Errorf(createResultFolderError, err) + return + } + log.Infoln("[+] starting screenshot tasks") + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if scanner.Text() != "" { + targets <- scanner.Text() + } + } + for i := 1; i < mirrorArgs.thread+1; i++ { + go navigate(i) + mirrorWg.Add(1) + } + mirrorWg.Wait() +} + +func navigate(workerNum int) { + defer mirrorWg.Done() + + // create context + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", !mirrorArgs.chrome), + chromedp.ProxyServer(mirrorArgs.proxy), + chromedp.Flag("mute-audio", true), + chromedp.IgnoreCertErrors, + chromedp.DisableGPU, + chromedp.NoFirstRun, + chromedp.ExecPath(mirrorArgs.path), + chromedp.WindowSize(1920, 1080), + chromedp.NoDefaultBrowserCheck, + chromedp.NoSandbox, + ) + + if mirrorArgs.proxy != "" { + opts = append(opts, chromedp.Flag("proxy-server", mirrorArgs.proxy)) + } + + ctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + + ctx, cancel = context.WithTimeout(ctx, time.Duration(mirrorArgs.timeout)*time.Second) + + ctx, cancel = chromedp.NewContext(ctx) + + // Check + err := chromedp.Run( + ctx, + chromedp.Tasks{ + chromedp.Navigate("about:blank"), + }, + ) + if err != nil { + if strings.Contains(err.Error(), "executable file not found") { + log.Fatalln(execPathNotFound) + } else { + log.Fatalln(err) + } + } + + // Run + for len(targets) > 0 { + url := <-targets + log.Debugf("[Worker%2d] starting screenshot task %s,remaining tasks:%d\n", workerNum, url, len(targets)) + var buf []byte + if err := chromedp.Run(ctx, fullScreenshot(url, 100, &buf)); err != nil { + log.Errorf("[-] Failed to take (URL:%s) screenshot: %v", url, err) + continue + } + pngFilePath := fmt.Sprintf("./%s/%s.png", mirrorArgs.result, getDomain(url)) + if err := ioutil.WriteFile(pngFilePath, buf, 0644); err != nil { + log.Errorf("[-] Failed to write file %v", err) + continue + } + log.Infof("[+] screenshot success %s", url) + log.Debugf("[Worker%2d] finished screenshot task %s,remaining tasks:%d\n", workerNum, url, len(targets)) + + } + + defer cancel() +} + +func fullScreenshot(urlstr string, quality int64, res *[]byte) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Navigate(urlstr), + //chromedp.WaitVisible("style"), + chromedp.Sleep(5 * time.Second), + //chromedp.OuterHTML(`document.querySelector("body")`, &htmlContent, chromedp.ByJSPath), + chromedp.ActionFunc(func(ctx context.Context) error { + _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + if err != nil { + return err + } + + width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + + err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + WithScreenOrientation(&emulation.ScreenOrientation{ + Type: emulation.OrientationTypePortraitPrimary, + Angle: 0, + }). + Do(ctx) + if err != nil { + return err + } + + *res, err = page.CaptureScreenshot(). + WithQuality(quality). + WithClip(&page.Viewport{ + X: contentSize.X, + Y: contentSize.Y, + Width: contentSize.Width, + Height: contentSize.Height, + Scale: 1, + }).Do(ctx) + if err != nil { + return err + } + return nil + }), + } +} + +func getDomain(urlstr string) string { + u, err := url.Parse(urlstr) + if err != nil { + fmt.Println(err) + return "getdomain_error" + } + return u.Hostname() +} + +func CreateDirIfNotExists(dirName string) error { + if _, err := os.Stat(dirName); os.IsNotExist(err) { + err := os.MkdirAll(dirName, 0755) + if err != nil { + return err + } + } + return nil +} diff --git a/data.json b/data.json index 5713c47..eb50922 100644 --- a/data.json +++ b/data.json @@ -1459,7 +1459,8 @@ "header": { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" }, - "userPage": "http://www.xsssql.com/article/author/%s" + "userPage": "http://www.xsssql.com/article/author/%s", + "status": false } ], "login": { diff --git a/go.mod b/go.mod index 35bdc68..b7f9307 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( ) require ( + github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect @@ -21,7 +22,6 @@ require ( require ( github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b github.com/chromedp/chromedp v0.9.1 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index bb9a5e4..104f003 100644 --- a/go.sum +++ b/go.sum @@ -2,9 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY= github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b h1:SK9lKpfJgkHu8VzAGGiRWm6IJ8EDo0OVjKqihL7KwEI= -github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA= github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= diff --git a/utils/writeToFile.go b/utils/writeToFile.go new file mode 100644 index 0000000..0420619 --- /dev/null +++ b/utils/writeToFile.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "os" +) + +func WriteToFile(filename string, content chan string, done chan bool) { + file, _ := os.Create(filename) + defer file.Close() + + for { + select { + case line := <-content: + _, _ = fmt.Fprint(file, line) + case <-done: + return + } + } +} From c2e6bf7da82fcca78e22011c9f767e7e7a1acbcc Mon Sep 17 00:00:00 2001 From: poq79 <273589250@qq.com> Date: Sat, 13 May 2023 15:25:18 +0800 Subject: [PATCH 03/10] remove old files --- utils/screenshot.go | 132 -------------------------------------------- 1 file changed, 132 deletions(-) delete mode 100644 utils/screenshot.go diff --git a/utils/screenshot.go b/utils/screenshot.go deleted file mode 100644 index 283568d..0000000 --- a/utils/screenshot.go +++ /dev/null @@ -1,132 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "io/ioutil" - "log" - "math" - "net/url" - "os" - "regexp" - "strings" - "time" - - "github.com/chromedp/cdproto/emulation" - "github.com/chromedp/cdproto/page" - "github.com/chromedp/chromedp" -) - -func DoScreenshot(url string, safeFolderName string) { - - opts := append( - chromedp.DefaultExecAllocatorOptions[:], - chromedp.NoDefaultBrowserCheck, - chromedp.Flag("headless", true), - chromedp.Flag("blink-settings", "imagesEnabled=true"), - chromedp.Flag("ignore-certificate-errors", true), - chromedp.Flag("disable-web-security", true), - chromedp.Flag("disable-extensions", true), - chromedp.Flag("disable-default-apps", true), - chromedp.WindowSize(1920, 1080), - chromedp.Flag("disable-gpu", true), - chromedp.Flag("hide-scrollbars", true), - chromedp.Flag("mute-audio", true), - chromedp.Flag("no-sandbox", true), - chromedp.Flag("no-default-browser-check", true), - chromedp.NoFirstRun, - chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"), - ) - allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) - defer cancel() - - ctx, cancel := chromedp.NewContext( - allocCtx, - chromedp.WithLogf(log.Printf), - ) - defer cancel() - - ctx, cancel = context.WithTimeout(ctx, 50*time.Second) - defer cancel() - - var buf []byte - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(ctx, fullScreenshot(url, 100, &buf)); err != nil { - log.Fatal(err) - } - if err := ioutil.WriteFile("./"+safeFolderName+"/"+getdomain(url)+".png", buf, 0644); err != nil { - log.Fatal(err) - } - // log.Println("screenshot") - -} - -func fullScreenshot(urlstr string, quality int64, res *[]byte) chromedp.Tasks { - return chromedp.Tasks{ - chromedp.Navigate(urlstr), - //chromedp.WaitVisible("style"), - chromedp.Sleep(20 * time.Second), - //chromedp.OuterHTML(`document.querySelector("body")`, &htmlContent, chromedp.ByJSPath), - chromedp.ActionFunc(func(ctx context.Context) error { - _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) - if err != nil { - return err - } - - width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) - - err = emulation.SetDeviceMetricsOverride(width, height, 1, false). - WithScreenOrientation(&emulation.ScreenOrientation{ - Type: emulation.OrientationTypePortraitPrimary, - Angle: 0, - }). - Do(ctx) - if err != nil { - return err - } - - *res, err = page.CaptureScreenshot(). - WithQuality(quality). - WithClip(&page.Viewport{ - X: contentSize.X, - Y: contentSize.Y, - Width: contentSize.Width, - Height: contentSize.Height, - Scale: 1, - }).Do(ctx) - if err != nil { - return err - } - return nil - }), - } -} - -func getdomain(urlstr string) string { - u, err := url.Parse(urlstr) - if err != nil { - fmt.Println(err) - return "error" - } - return u.Hostname() -} - -func SanitizeFilename(filename string) string { - re := regexp.MustCompile(`[\\/:*?"<>|]|`) - filename = re.ReplaceAllString(filename, "") - filename = strings.TrimSpace(filename) - return filename -} - -func CreateDirIfNotExists(dirName string) error { - if _, err := os.Stat(dirName); os.IsNotExist(err) { - err := os.MkdirAll(dirName, 0755) - if err != nil { - return err - } - // fmt.Printf("Folder \"%s\" created.\n", dirName) - } else { - // fmt.Printf("Folder \"%s\" already exists.\n", dirName) - } - return nil -} From b758f3decf3d424f837024a9ac168130de1902c6 Mon Sep 17 00:00:00 2001 From: poq79 <273589250@qq.com> Date: Sat, 13 May 2023 15:35:44 +0800 Subject: [PATCH 04/10] modify args --- cmd/smirror.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/smirror.go b/cmd/smirror.go index 92d9a30..d8989f3 100644 --- a/cmd/smirror.go +++ b/cmd/smirror.go @@ -41,7 +41,7 @@ var ( func init() { mirrorCmd.Flags().StringVar(&mirrorArgs.proxy, "proxy", "", "Make requests over a proxy. e.g. socks5://127.0.0.1:1080") mirrorCmd.Flags().StringVar(&mirrorArgs.path, "path", "", "Chrome ExecPath") - mirrorCmd.Flags().StringVarP(&mirrorArgs.result, "result", "r", "screenshots", "Folder path of the screenshot") + mirrorCmd.Flags().StringVarP(&mirrorArgs.result, "dir", "d", "screenshots", "Folder path of the screenshot") mirrorCmd.Flags().BoolVar(&mirrorArgs.chrome, "chrome", false, "Show chrome") mirrorCmd.Flags().IntVarP(&mirrorArgs.thread, "thread", "t", 3, "Chrome number") mirrorCmd.Flags().StringVarP(&mirrorArgs.file, "file", "f", "result.txt", "Url list file") From 8848a1dc15b583a57a491b7f69e3b39769e4baab Mon Sep 17 00:00:00 2001 From: poq79 <273589250@qq.com> Date: Sat, 13 May 2023 17:07:11 +0800 Subject: [PATCH 05/10] rename module --- cmd/{smirror.go => screenshot.go} | 58 +++++++++++++++---------------- 1 file changed, 29 insertions(+), 29 deletions(-) rename cmd/{smirror.go => screenshot.go} (70%) diff --git a/cmd/smirror.go b/cmd/screenshot.go similarity index 70% rename from cmd/smirror.go rename to cmd/screenshot.go index d8989f3..51eee13 100644 --- a/cmd/smirror.go +++ b/cmd/screenshot.go @@ -19,7 +19,7 @@ import ( "github.com/spf13/cobra" ) -type mirrorArgsType struct { +type screenshotArgsType struct { proxy string chrome bool thread int @@ -30,8 +30,8 @@ type mirrorArgsType struct { } var ( - mirrorArgs mirrorArgsType - mirrorWg sync.WaitGroup + screenshotArgs screenshotArgsType + screenshotWg sync.WaitGroup targets = make(chan string, 1000) execPathNotFound = "[-] Chrome executable file not found, please install Chrome or specify the chrome.exe path with --path" urlFileReadError = "[-] failed to read targets file:%v" @@ -39,35 +39,35 @@ var ( ) func init() { - mirrorCmd.Flags().StringVar(&mirrorArgs.proxy, "proxy", "", "Make requests over a proxy. e.g. socks5://127.0.0.1:1080") - mirrorCmd.Flags().StringVar(&mirrorArgs.path, "path", "", "Chrome ExecPath") - mirrorCmd.Flags().StringVarP(&mirrorArgs.result, "dir", "d", "screenshots", "Folder path of the screenshot") - mirrorCmd.Flags().BoolVar(&mirrorArgs.chrome, "chrome", false, "Show chrome") - mirrorCmd.Flags().IntVarP(&mirrorArgs.thread, "thread", "t", 3, "Chrome number") - mirrorCmd.Flags().StringVarP(&mirrorArgs.file, "file", "f", "result.txt", "Url list file") - mirrorCmd.Flags().IntVar(&mirrorArgs.timeout, "timeout", 60, "Timeout") - rootCmd.AddCommand(mirrorCmd) + screenshotCmd.Flags().StringVar(&screenshotArgs.proxy, "proxy", "", "Make requests over a proxy. e.g. socks5://127.0.0.1:1080") + screenshotCmd.Flags().StringVar(&screenshotArgs.path, "path", "", "Chrome ExecPath") + screenshotCmd.Flags().StringVarP(&screenshotArgs.result, "dir", "d", "screenshots", "Folder path of the screenshot") + screenshotCmd.Flags().BoolVar(&screenshotArgs.chrome, "chrome", false, "Show chrome") + screenshotCmd.Flags().IntVarP(&screenshotArgs.thread, "thread", "t", 3, "Chrome number") + screenshotCmd.Flags().StringVarP(&screenshotArgs.file, "file", "f", "result.txt", "Url list file") + screenshotCmd.Flags().IntVar(&screenshotArgs.timeout, "timeout", 60, "Timeout") + rootCmd.AddCommand(screenshotCmd) } -var mirrorCmd = &cobra.Command{ - Use: "mirror", - Short: "mirror data.json using github", +var screenshotCmd = &cobra.Command{ + Use: "screenshot", + Short: "screenshot data.json using github", Long: ``, - Run: mirror, + Run: screenshot, } -func mirror(_ *cobra.Command, _ []string) { +func screenshot(_ *cobra.Command, _ []string) { if Verbose { log.Infoln("Debug Mode") log.SetLevel(log.DebugLevel) } - file, err := os.Open(mirrorArgs.file) + file, err := os.Open(screenshotArgs.file) if err != nil { log.Errorf(urlFileReadError, err) return } defer file.Close() - err = CreateDirIfNotExists(mirrorArgs.result) + err = CreateDirIfNotExists(screenshotArgs.result) if err != nil { log.Errorf(createResultFolderError, err) return @@ -80,37 +80,37 @@ func mirror(_ *cobra.Command, _ []string) { targets <- scanner.Text() } } - for i := 1; i < mirrorArgs.thread+1; i++ { + for i := 1; i < screenshotArgs.thread+1; i++ { go navigate(i) - mirrorWg.Add(1) + screenshotWg.Add(1) } - mirrorWg.Wait() + screenshotWg.Wait() } func navigate(workerNum int) { - defer mirrorWg.Done() + defer screenshotWg.Done() // create context opts := append(chromedp.DefaultExecAllocatorOptions[:], - chromedp.Flag("headless", !mirrorArgs.chrome), - chromedp.ProxyServer(mirrorArgs.proxy), + chromedp.Flag("headless", !screenshotArgs.chrome), + chromedp.ProxyServer(screenshotArgs.proxy), chromedp.Flag("mute-audio", true), chromedp.IgnoreCertErrors, chromedp.DisableGPU, chromedp.NoFirstRun, - chromedp.ExecPath(mirrorArgs.path), + chromedp.ExecPath(screenshotArgs.path), chromedp.WindowSize(1920, 1080), chromedp.NoDefaultBrowserCheck, chromedp.NoSandbox, ) - if mirrorArgs.proxy != "" { - opts = append(opts, chromedp.Flag("proxy-server", mirrorArgs.proxy)) + if screenshotArgs.proxy != "" { + opts = append(opts, chromedp.Flag("proxy-server", screenshotArgs.proxy)) } ctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) - ctx, cancel = context.WithTimeout(ctx, time.Duration(mirrorArgs.timeout)*time.Second) + ctx, cancel = context.WithTimeout(ctx, time.Duration(screenshotArgs.timeout)*time.Second) ctx, cancel = chromedp.NewContext(ctx) @@ -138,7 +138,7 @@ func navigate(workerNum int) { log.Errorf("[-] Failed to take (URL:%s) screenshot: %v", url, err) continue } - pngFilePath := fmt.Sprintf("./%s/%s.png", mirrorArgs.result, getDomain(url)) + pngFilePath := fmt.Sprintf("./%s/%s.png", screenshotArgs.result, getDomain(url)) if err := ioutil.WriteFile(pngFilePath, buf, 0644); err != nil { log.Errorf("[-] Failed to write file %v", err) continue From b0855a2c9582beb7545a77669baded1e2d8824bc Mon Sep 17 00:00:00 2001 From: lxy <1418676300@qq.com> Date: Sat, 13 May 2023 03:07:29 -0700 Subject: [PATCH 06/10] pref: add parse detect output file Made improvement - add parse detect output file - fix target channel - fix sleep --- .gitignore | 2 ++ .idea/DetectDee.iml | 4 +++- cmd/detect.go | 5 ++-- cmd/screenshot.go | 58 ++++++++++++++++++++++++++++----------------- go.mod | 6 ++--- go.sum | 34 ++++---------------------- 6 files changed, 50 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 1932579..eee0705 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out +result.txt # Dependency directories (remove the comment below to include it) # vendor/ @@ -21,3 +22,4 @@ pack/ # Go workspace file go.work +screenshots/ diff --git a/.idea/DetectDee.iml b/.idea/DetectDee.iml index 5e764c4..9942e69 100644 --- a/.idea/DetectDee.iml +++ b/.idea/DetectDee.iml @@ -2,7 +2,9 @@ - + + + diff --git a/cmd/detect.go b/cmd/detect.go index 8059a3b..74f3571 100644 --- a/cmd/detect.go +++ b/cmd/detect.go @@ -36,6 +36,7 @@ var ( wg sync.WaitGroup nonSiteData = "[-] There is no site data of %s\n" existInfo = "[+] %-15s %-15s: %s\n" + existOutputInfo = "%-15s %-15s: %s\n" nonExistInfo = "[-] %-15s %-15s: non exists\n" reqErrorInfo = "[!] %-15s %-15s: %s requests error, retry %d/%d\n" sleepMap = make(map[string]int64) @@ -321,12 +322,12 @@ func detectUser(name, site string, requestTimes, retryTimes, detectCount int, fl } else if !detectArgs.precisely { // flag=true && precisely=false log.Infof(existInfo, name, site, userPage) - writeContent <- fmt.Sprintf(existInfo, name, site, userPage) + writeContent <- fmt.Sprintf(existOutputInfo, name, site, userPage) return false } else if requestTimes == detectCount { // flag=true && precisely=true && last request log.Infof(existInfo, name, site, userPage) - writeContent <- fmt.Sprintf(existInfo, name, site, userPage) + writeContent <- fmt.Sprintf(existOutputInfo, name, site, userPage) return true } else { return true diff --git a/cmd/screenshot.go b/cmd/screenshot.go index 51eee13..d9e96b9 100644 --- a/cmd/screenshot.go +++ b/cmd/screenshot.go @@ -32,7 +32,7 @@ type screenshotArgsType struct { var ( screenshotArgs screenshotArgsType screenshotWg sync.WaitGroup - targets = make(chan string, 1000) + targets = make(chan []string, 10) execPathNotFound = "[-] Chrome executable file not found, please install Chrome or specify the chrome.exe path with --path" urlFileReadError = "[-] failed to read targets file:%v" createResultFolderError = "[-] failed to create result folder:%v" @@ -61,25 +61,16 @@ func screenshot(_ *cobra.Command, _ []string) { log.Infoln("Debug Mode") log.SetLevel(log.DebugLevel) } - file, err := os.Open(screenshotArgs.file) - if err != nil { - log.Errorf(urlFileReadError, err) - return - } - defer file.Close() - err = CreateDirIfNotExists(screenshotArgs.result) + + go parseFile(screenshotArgs.file) + + err := CreateDirIfNotExists(screenshotArgs.result) if err != nil { log.Errorf(createResultFolderError, err) return } log.Infoln("[+] starting screenshot tasks") - scanner := bufio.NewScanner(file) - for scanner.Scan() { - if scanner.Text() != "" { - targets <- scanner.Text() - } - } for i := 1; i < screenshotArgs.thread+1; i++ { go navigate(i) screenshotWg.Add(1) @@ -87,6 +78,26 @@ func screenshot(_ *cobra.Command, _ []string) { screenshotWg.Wait() } +func parseFile(file string) { + f, err := os.Open(file) + if err != nil { + log.Errorf(urlFileReadError, err) + return + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if scanner.Text() != "" { + tmp := strings.Split(scanner.Text(), ": ") + name := strings.Split(strings.TrimSpace(tmp[0]), " ") + + target := []string{fmt.Sprintf("%s-%s", name[0], name[len(name)-1]), tmp[1]} + targets <- target + } + } +} + func navigate(workerNum int) { defer screenshotWg.Done() @@ -131,20 +142,23 @@ func navigate(workerNum int) { // Run for len(targets) > 0 { - url := <-targets - log.Debugf("[Worker%2d] starting screenshot task %s,remaining tasks:%d\n", workerNum, url, len(targets)) + target := <-targets + name := target[0] + urlStr := target[1] + + log.Debugf("[Worker%2d] starting screenshot task %s,remaining tasks:%d\n", workerNum, urlStr, len(targets)) var buf []byte - if err := chromedp.Run(ctx, fullScreenshot(url, 100, &buf)); err != nil { - log.Errorf("[-] Failed to take (URL:%s) screenshot: %v", url, err) + if err := chromedp.Run(ctx, fullScreenshot(urlStr, 100, &buf)); err != nil { + log.Errorf("[-] Failed to take (URL:%s) screenshot: %v", urlStr, err) continue } - pngFilePath := fmt.Sprintf("./%s/%s.png", screenshotArgs.result, getDomain(url)) + pngFilePath := fmt.Sprintf("./%s/%s.png", screenshotArgs.result, name+"-"+getDomain(urlStr)) if err := ioutil.WriteFile(pngFilePath, buf, 0644); err != nil { log.Errorf("[-] Failed to write file %v", err) continue } - log.Infof("[+] screenshot success %s", url) - log.Debugf("[Worker%2d] finished screenshot task %s,remaining tasks:%d\n", workerNum, url, len(targets)) + log.Infof("[+] screenshot success %s", urlStr) + log.Debugf("[Worker%2d] finished screenshot task %s,remaining tasks:%d\n", workerNum, urlStr, len(targets)) } @@ -155,7 +169,7 @@ func fullScreenshot(urlstr string, quality int64, res *[]byte) chromedp.Tasks { return chromedp.Tasks{ chromedp.Navigate(urlstr), //chromedp.WaitVisible("style"), - chromedp.Sleep(5 * time.Second), + //chromedp.Sleep(5 * time.Second), //chromedp.OuterHTML(`document.querySelector("body")`, &htmlContent, chromedp.ByJSPath), chromedp.ActionFunc(func(ctx context.Context) error { _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) diff --git a/go.mod b/go.mod index b7f9307..c731d95 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module DetectDee go 1.18 require ( - github.com/PuerkitoBio/goquery v1.8.1 + github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b + github.com/chromedp/chromedp v0.9.1 github.com/go-resty/resty/v2 v2.7.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.7.0 @@ -11,7 +12,6 @@ require ( ) require ( - github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect @@ -21,8 +21,6 @@ require ( ) require ( - github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/chromedp/chromedp v0.9.1 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 104f003..37f91b6 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,6 @@ -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY= github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b h1:SK9lKpfJgkHu8VzAGGiRWm6IJ8EDo0OVjKqihL7KwEI= +github.com/chromedp/cdproto v0.0.0-20230506233603-4ea4c6dc2e5b/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA= github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= @@ -24,9 +21,11 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -46,43 +45,18 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 133561f905cc973d8e069f8aa43d0181a08ebd8d Mon Sep 17 00:00:00 2001 From: lxy <1418676300@qq.com> Date: Sat, 13 May 2023 03:30:03 -0700 Subject: [PATCH 07/10] pref: add parse detect output file Made improvement - add parse detect output file - fix target channel - fix sleep --- cmd/screenshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/screenshot.go b/cmd/screenshot.go index d9e96b9..b9c8de7 100644 --- a/cmd/screenshot.go +++ b/cmd/screenshot.go @@ -169,7 +169,7 @@ func fullScreenshot(urlstr string, quality int64, res *[]byte) chromedp.Tasks { return chromedp.Tasks{ chromedp.Navigate(urlstr), //chromedp.WaitVisible("style"), - //chromedp.Sleep(5 * time.Second), + chromedp.Sleep(1 * time.Second), //chromedp.OuterHTML(`document.querySelector("body")`, &htmlContent, chromedp.ByJSPath), chromedp.ActionFunc(func(ctx context.Context) error { _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) From f5fcf2fba1a33ce76ac29294a5db1687ce9bd52f Mon Sep 17 00:00:00 2001 From: lxy <1418676300@qq.com> Date: Sat, 13 May 2023 03:31:06 -0700 Subject: [PATCH 08/10] pref: add parse detect output file Made improvement - add parse detect output file - fix target channel - fix sleep --- cmd/screenshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/screenshot.go b/cmd/screenshot.go index b9c8de7..96e6053 100644 --- a/cmd/screenshot.go +++ b/cmd/screenshot.go @@ -152,7 +152,7 @@ func navigate(workerNum int) { log.Errorf("[-] Failed to take (URL:%s) screenshot: %v", urlStr, err) continue } - pngFilePath := fmt.Sprintf("./%s/%s.png", screenshotArgs.result, name+"-"+getDomain(urlStr)) + pngFilePath := fmt.Sprintf("./%s/%s.png", screenshotArgs.result, name) if err := ioutil.WriteFile(pngFilePath, buf, 0644); err != nil { log.Errorf("[-] Failed to write file %v", err) continue From 6c2c851d980fc5ce2b00ea969efcec153706590c Mon Sep 17 00:00:00 2001 From: lxy <1418676300@qq.com> Date: Sat, 13 May 2023 07:00:05 -0700 Subject: [PATCH 09/10] pref: add parse detect output file Made improvement - add parse detect output file - fix target channel - fix sleep --- cmd/screenshot.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/screenshot.go b/cmd/screenshot.go index 96e6053..e3f1fcd 100644 --- a/cmd/screenshot.go +++ b/cmd/screenshot.go @@ -121,15 +121,15 @@ func navigate(workerNum int) { ctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) - ctx, cancel = context.WithTimeout(ctx, time.Duration(screenshotArgs.timeout)*time.Second) + //ctx, cancel = context.WithTimeout(ctx, time.Duration(screenshotArgs.timeout)*time.Second) ctx, cancel = chromedp.NewContext(ctx) - // Check + // Check & Make Browser not close err := chromedp.Run( ctx, chromedp.Tasks{ - chromedp.Navigate("about:blank"), + chromedp.Navigate("https://github.com/piaolin"), }, ) if err != nil { @@ -148,6 +148,9 @@ func navigate(workerNum int) { log.Debugf("[Worker%2d] starting screenshot task %s,remaining tasks:%d\n", workerNum, urlStr, len(targets)) var buf []byte + //ctx, cancel = chromedp.NewContext( + // ctx, + //) if err := chromedp.Run(ctx, fullScreenshot(urlStr, 100, &buf)); err != nil { log.Errorf("[-] Failed to take (URL:%s) screenshot: %v", urlStr, err) continue @@ -157,9 +160,13 @@ func navigate(workerNum int) { log.Errorf("[-] Failed to write file %v", err) continue } + + //if err := page.Close().Do(cdp.WithExecutor(ctx, chromedp.FromContext(ctx).Target)); err != nil { + // log.Errorln(err) + //} + log.Infof("[+] screenshot success %s", urlStr) log.Debugf("[Worker%2d] finished screenshot task %s,remaining tasks:%d\n", workerNum, urlStr, len(targets)) - } defer cancel() From 8db35d2dd6d8bfdc408a99f00f212c732176f80a Mon Sep 17 00:00:00 2001 From: lxy <1418676300@qq.com> Date: Sat, 13 May 2023 07:22:36 -0700 Subject: [PATCH 10/10] pref: add parse detect output file Made improvement - add parse detect output file - fix target channel - fix sleep - update README --- README.md | 29 +++++++++++++++++++ README_ZH.md | 30 +++++++++++++++++++- cmd/screenshot.go | 72 +++++++++++++++++++++++------------------------ 3 files changed, 93 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 19b311d..7eead07 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ DetectDee: Hunt down social media accounts by **username, email or phone** across [social networks](site.md) ![example.gif](https://s2.loli.net/2023/04/30/FZ1QtKoGud4xVPW.gif) +![screen.jpg](https://s2.loli.net/2023/05/13/XzV4EGKrbkURHQg.jpg) ## Feat - Includes sites frequently used by **CyberSecurity practitioners** - Hunt down social media accounts by **username, email or phone** @@ -85,7 +86,35 @@ To search in specified site: ```shell ./DetectDee detect -n piaolin -s github,v2ex ``` +### Screenshot +The screenshot function is used to screenshot the results of detect. Note that this function requires: +- Chrome +- A period of time +- A bit of memory usage +```shell +Usage: + DetectDee screenshot [flags] + +Flags: + --chrome Show chrome + -d, --dir string Folder path of the screenshot (default "screenshots") + -f, --file string Url list file (default "result.txt") + -h, --help help for screenshot + --path string Chrome ExecPath + --proxy string Make requests over a proxy. e.g. socks5://127.0.0.1:1080 + -t, --thread int Chrome number (default 3) + --timeout int Timeout (default 60) + +Global Flags: + -v, --verbose verbose output +``` +Screenshot the results of detect +```shell +./DetectDee screenshot +``` +![result.jpg](https://s2.loli.net/2023/05/13/OWRDnU98TyCdceN.jpg) +![screen.jpg](https://s2.loli.net/2023/05/13/XzV4EGKrbkURHQg.jpg) ## Contributing We would love to have you help us with the development of DetectDee. Each and every contribution is greatly valued! diff --git a/README_ZH.md b/README_ZH.md index 69d9ea5..5355e61 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -3,6 +3,7 @@ 神探狄仁杰: 在[社交网络](site.md)上通过**用户名，电子邮件或电话**搜索社交媒体账户 ![example.gif](https://s2.loli.net/2023/04/30/FZ1QtKoGud4xVPW.gif) +![screen.jpg](https://s2.loli.net/2023/05/13/XzV4EGKrbkURHQg.jpg) ## 特性 - 集成**网络安全从业者**常用网站 - 通过**用户名，电子邮件或电话**查找社交媒体账户 @@ -81,7 +82,34 @@ Global Flags: ```shell ./DetectDee detect -n piaolin -s github,v2ex ``` - +### 截屏 +截屏功能用于将探测得来的结果进行访问截屏，请注意该功能需要： +- Chrome +- 一定时间 +- 一定内存占用 +```shell +Usage: + DetectDee screenshot [flags] + +Flags: + --chrome Show chrome + -d, --dir string Folder path of the screenshot (default "screenshots") + -f, --file string Url list file (default "result.txt") + -h, --help help for screenshot + --path string Chrome ExecPath + --proxy string Make requests over a proxy. e.g. socks5://127.0.0.1:1080 + -t, --thread int Chrome number (default 3) + --timeout int Timeout (default 60) + +Global Flags: + -v, --verbose verbose output +``` +对detect得到结果进行截屏 +```shell +./DetectDee screenshot +``` +![result.jpg](https://s2.loli.net/2023/05/13/OWRDnU98TyCdceN.jpg) +![screen.jpg](https://s2.loli.net/2023/05/13/XzV4EGKrbkURHQg.jpg) ## 贡献 希望您能帮助我们开发“神探狄仁杰”，您的每一份提交都会得到我的重视。 diff --git a/cmd/screenshot.go b/cmd/screenshot.go index e3f1fcd..00a072b 100644 --- a/cmd/screenshot.go +++ b/cmd/screenshot.go @@ -5,15 +5,12 @@ import ( "context" "fmt" "io/ioutil" - "math" "net/url" "os" "strings" "sync" "time" - "github.com/chromedp/cdproto/emulation" - "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -51,7 +48,7 @@ func init() { var screenshotCmd = &cobra.Command{ Use: "screenshot", - Short: "screenshot data.json using github", + Short: "screenshot result with chrome headless", Long: ``, Run: screenshot, } @@ -172,44 +169,45 @@ func navigate(workerNum int) { defer cancel() } -func fullScreenshot(urlstr string, quality int64, res *[]byte) chromedp.Tasks { +func fullScreenshot(urlstr string, quality int, res *[]byte) chromedp.Tasks { return chromedp.Tasks{ chromedp.Navigate(urlstr), //chromedp.WaitVisible("style"), chromedp.Sleep(1 * time.Second), //chromedp.OuterHTML(`document.querySelector("body")`, &htmlContent, chromedp.ByJSPath), - chromedp.ActionFunc(func(ctx context.Context) error { - _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) - if err != nil { - return err - } - - width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) - - err = emulation.SetDeviceMetricsOverride(width, height, 1, false). - WithScreenOrientation(&emulation.ScreenOrientation{ - Type: emulation.OrientationTypePortraitPrimary, - Angle: 0, - }). - Do(ctx) - if err != nil { - return err - } - - *res, err = page.CaptureScreenshot(). - WithQuality(quality). - WithClip(&page.Viewport{ - X: contentSize.X, - Y: contentSize.Y, - Width: contentSize.Width, - Height: contentSize.Height, - Scale: 1, - }).Do(ctx) - if err != nil { - return err - } - return nil - }), + chromedp.FullScreenshot(res, quality), + //chromedp.ActionFunc(func(ctx context.Context) error { + // _, _, _, _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + // if err != nil { + // return err + // } + // + // width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + // + // err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + // WithScreenOrientation(&emulation.ScreenOrientation{ + // Type: emulation.OrientationTypePortraitPrimary, + // Angle: 0, + // }). + // Do(ctx) + // if err != nil { + // return err + // } + // + // *res, err = page.CaptureScreenshot(). + // WithQuality(quality). + // WithClip(&page.Viewport{ + // X: contentSize.X, + // Y: contentSize.Y, + // Width: contentSize.Width, + // Height: contentSize.Height, + // Scale: 1, + // }).Do(ctx) + // if err != nil { + // return err + // } + // return nil + //}), } }