【网站内容安全检测】之1：获取网站所有链接sitemap数据

不多BB，直接上代码：
main.go

package mainimport ("bufio""crypto/tls""fmt""io""net/http""net/url""os""strings""sync""time"_ "net/http/pprof""log""github.com/PuerkitoBio/goquery""github.com/schollz/progressbar/v3"
)type WebCrawler struct {startURLs     []stringbaseDomains   map[string]boolvisitedURLs   sync.MapurlsToVisit   chan stringsemaphore     chan struct{}timeout       time.DurationverifySSL     boolclient        *http.ClientprogressBar   *progressbar.ProgressBarwg            sync.WaitGroup
}func NewWebCrawler(startURLs []string, maxConnections int, timeout int, verifySSL bool) *WebCrawler {baseDomains := make(map[string]bool)for _, u := range startURLs {parsed, _ := url.Parse(u)baseDomains[parsed.Host] = true}return &WebCrawler{startURLs:   startURLs,baseDomains: baseDomains,urlsToVisit: make(chan string, 1000),semaphore:   make(chan struct{}, maxConnections),timeout:     time.Duration(timeout) * time.Second,verifySSL:   verifySSL,}
}func (c *WebCrawler) initClient() {tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: !c.verifySSL},}c.client = &http.Client{Timeout:   c.timeout,Transport: tr,}
}func (c *WebCrawler) normalizeURL(rawURL string, baseURL string) (string, error) {base, err := url.Parse(baseURL)if err != nil || base == nil {return "", fmt.Errorf("invalid base URL: %v", err)}u, err := url.Parse(rawURL)if err != nil || u == nil {return "", fmt.Errorf("invalid URL: %v", err)}return base.ResolveReference(u).String(), nil
}func (c *WebCrawler) isValidURL(rawURL string) bool {parsed, err := url.Parse(rawURL)if err != nil {return false}if parsed.Scheme != "http" && parsed.Scheme != "https" {return false}if !c.baseDomains[parsed.Host] {return false}extensions := []string{".jpg", ".jpeg", ".png", ".gif", ".pdf", ".zip"}for _, ext := range extensions {if strings.HasSuffix(strings.ToLower(parsed.Path), ext) {return false}}return true
}func (c *WebCrawler) fetchURL(url string) (string, error) {c.semaphore <- struct{}{}defer func() { <-c.semaphore }()req, err := http.NewRequest("GET", url, nil)if err != nil {return "", fmt.Errorf("request creation failed: %v", err)}req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")resp, err := c.client.Do(req)if err != nil {if strings.Contains(err.Error(), "no such host") {return "", fmt.Errorf("DNS lookup failed for %s", url)}return "", fmt.Errorf("request failed: %v", err)}defer resp.Body.Close()if resp.StatusCode != 200 {return "", fmt.Errorf("non-200 status: %d", resp.StatusCode)}if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {return "", fmt.Errorf("non-HTML content type: %s", resp.Header.Get("Content-Type"))}body, err := io.ReadAll(resp.Body)if err != nil {return "", fmt.Errorf("error reading response: %v", err)}return string(body), nil
}func (c *WebCrawler) parseLinks(html string, baseURL string) []string {doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))if err != nil {log.Printf("Error parsing HTML: %v", err)return nil}var links []stringdoc.Find("a[href]").Each(func(i int, s *goquery.Selection) {href, exists := s.Attr("href")if !exists || strings.HasPrefix(href, "javascript:") || href == "#" {return}normalized, err := c.normalizeURL(href, baseURL)if err != nil {log.Printf("Error normalizing URL %s: %v", href, err)return}if c.isValidURL(normalized) {links = append(links, normalized)}})doc.Find("[src]").Each(func(i int, s *goquery.Selection) {src, exists := s.Attr("src")if !exists || strings.HasPrefix(src, "data:") {return}normalized, err := c.normalizeURL(src, baseURL)if err != nil {log.Printf("Error normalizing URL %s: %v", src, err)return}if c.isValidURL(normalized) {links = append(links, normalized)}})return links
}func (c *WebCrawler) processURL(url string) {defer c.wg.Done()if _, exists := c.visitedURLs.Load(url); exists {return}c.visitedURLs.Store(url, true)html, err := c.fetchURL(url)if err != nil {fmt.Printf("Error fetching %s: %v\n", url, err)return}newLinks := c.parseLinks(html, url)for _, link := range newLinks {if _, exists := c.visitedURLs.Load(link); !exists {c.urlsToVisit <- link}}if c.progressBar != nil {c.progressBar.Add(1)}
}func (c *WebCrawler) crawl() {c.initClient()c.progressBar = progressbar.Default(-1, "爬取进度")defer c.progressBar.Close()for _, url := range c.startURLs {c.wg.Add(1)go c.processURL(url)}go func() {for newURL := range c.urlsToVisit {if _, exists := c.visitedURLs.Load(newURL); !exists {c.wg.Add(1)go c.processURL(newURL)}}}()c.wg.Wait()
}func (c *WebCrawler) saveResults(filename string) {file, err := os.Create(filename)if err != nil {fmt.Printf("Error creating file: %v\n", err)return}defer file.Close()c.visitedURLs.Range(func(key, _ interface{}) bool {file.WriteString(key.(string) + "\n")return true})
}func (c *WebCrawler) run() {startTime := time.Now()c.crawl()elapsed := time.Since(startTime)fmt.Printf("\n爬取完成！\n")// 修复语法错误：添加缺少的括号和逗号visitedCount := 0c.visitedURLs.Range(func(key, _ interface{}) bool {visitedCount++return true})fmt.Printf("共爬取 %d 个URL\n", visitedCount)fmt.Printf("用时: %.2f 秒\n", elapsed.Seconds())outputFile := "multi_domain_links.txt"c.saveResults(outputFile)fmt.Printf("结果已保存到 %s\n", outputFile)
}func main() {go func() {log.Println(http.ListenAndServe("localhost:6060", nil))}()if len(os.Args) < 2 {fmt.Println("用法: go run web_crawler.go <URL文件路径> [verify_ssl]")fmt.Println("例如: go run web_crawler.go urls.txt")fmt.Println("或: go run web_crawler.go urls.txt true")return}urlFile := os.Args[1]file, err := os.Open(urlFile)if err != nil {fmt.Printf("错误：文件 %s 不存在\n", urlFile)return}defer file.Close()var startURLs []stringscanner := bufio.NewScanner(file)for scanner.Scan() {if url := strings.TrimSpace(scanner.Text()); url != "" {startURLs = append(startURLs, url)}}if len(startURLs) == 0 {fmt.Println("错误：URL文件为空")return}verifySSL := falseif len(os.Args) > 2 {verifySSL = os.Args[2] == "true"}crawler := NewWebCrawler(startURLs, 50, 20, verifySSL)// 添加开始运行提示fmt.Printf("开始爬取网站，起始URL数量: %d，是否验证SSL: %v\n", len(startURLs), verifySSL)crawler.run()
}

go.mod

module webcrawlergo 1.24.4require (github.com/PuerkitoBio/goquery v1.10.3 // indirectgithub.com/andybalholm/cascadia v1.3.3 // indirectgithub.com/mattn/go-sqlite3 v1.14.28 // indirectgithub.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirectgithub.com/rivo/uniseg v0.4.7 // indirectgithub.com/schollz/progressbar/v3 v3.18.0 // indirectgolang.org/x/net v0.39.0 // indirectgolang.org/x/sys v0.32.0 // indirectgolang.org/x/term v0.31.0 // indirect
)