// URLs implements a breadth-first search webcrawler based on the // example given in section 8.6 of The Go Programming Language. package main import ( "flag" "fmt" "log" "net/url" "runtime/pprof" "strings" "time" ) func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) maxConcurrency := flag.Int("c", 0, "Maximum number of concurrent queue pushes") startRawURL := flag.String("url", "", "Entry-point URL") maxURLs := flag.Int("max", 0, "Maximum number of URLs to collect (omitted or 0 means no limit)") maxDepth := flag.Int("depth", 0, "Maximum URL depth (omitted or 0 means no limit)") flag.Parse() if *maxConcurrency == 0 { log.Fatal("Missing -c argument") } if *maxConcurrency < 1 { log.Fatalf("Invalid -c argument: %d", *maxConcurrency) } if *startRawURL == "" { log.Fatal("Missing -url argument") } if *maxURLs < 0 { log.Fatalf("Invalid -max argument: %d", *maxURLs) } if *maxDepth < 0 { log.Fatalf("Invalid -depth argument: %d", *maxDepth) } startURL, err := url.Parse(*startRawURL) if err != nil { log.Fatal(err) } getLeakProfile(func() { classic(*startURL, *maxConcurrency, *maxURLs, *maxDepth) }) } // getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, // and writes it to stdout. func getLeakProfile(leakySnippet func()) { prof := pprof.Lookup("goroutineleak") defer func() { time.Sleep(2 * time.Second) var content strings.Builder prof.WriteTo(&content, 2) // Ignore non leaked goroutines leaks := strings.SplitSeq(content.String(), "\n\n") for leak := range leaks { if strings.Contains(leak, "(leaked)") { fmt.Println(leak + "\n") } } }() leakySnippet() }