// URLs implements a breadth-first search webcrawler based on the // example given in section 8.6 of The Go Programming Language. package main import ( "context" "errors" "fmt" "log" "net/url" "os" "runtime/pprof" "strings" "time" "github.com/urfave/cli/v3" ) const shortcodeFilename = "urls.csv" func main() { if os.Getenv("GOEXPERIMENT") != "goroutineleakprofile" { log.Fatal("Missing GOEXPERIMENT=goroutineleakprofile environment setting") } // Setting shorfile helps especially for when we log errors // without returning them. log.SetFlags(log.LstdFlags | log.Lshortfile) cmd := &cli.Command{ Usage: "A configurable web crawler", Commands: []*cli.Command{ { Name: "get", Usage: "Crawl the target", MutuallyExclusiveFlags: []cli.MutuallyExclusiveFlags{ { Required: true, Flags: [][]cli.Flag{ { &cli.StringFlag{ Name: "shortcode", Usage: "Specify target using a shortcode", }, }, { &cli.StringFlag{ Name: "url", Usage: "Specify target using a URL", }, }, }, }, }, Flags: []cli.Flag{ &cli.IntFlag{ Name: "concurrency", Aliases: []string{"c"}, Usage: "Allowable number of concurrent URL fetches", // Default concurrency setting is 1. Value: 1, }, &cli.IntFlag{ Name: "maxurls", Aliases: []string{"m"}, Usage: "Maximum number of URLs to collect", DefaultText: "no limit", }, &cli.IntFlag{ Name: "depth", Aliases: []string{"d"}, Usage: "Maximum URL depth", DefaultText: "no limit", }, }, Action: func(ctx context.Context, cmd *cli.Command) error { if shortcode := cmd.String("shortcode"); shortcode != "" { rawURL, err := getURLFromShortcode(shortcodeFilename, shortcode) if err != nil { return err } u, err := convertToURL(rawURL) if err != nil { return err } fmt.Printf("URL: %s\n", &u) classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) return nil } if rawURL := cmd.String("url"); rawURL != "" { u, err := convertToURL(rawURL) if err != nil { return err } fmt.Printf("URL: %s\n", &u) classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) return nil } // We shouldn't reach this // code, but let's at least // document our intentions. return errors.New("url and shortcode arguments should be mutually exclusive") }, }, { Name: "shortcode", Usage: "Configure shortcodes", Action: func(ctx context.Context, cmd *cli.Command) error { fmt.Println("shortcode!") return nil }, }, }, } if err := cmd.Run(context.Background(), os.Args); err != nil { fmt.Printf("\n%v\n", err) os.Exit(1) } } // convertToURL parses the given rawURL into a [url.URL]. If the // rawURL is missing a scheme, "https://" is prepended before parsing. // // Return the parsed URL, along with any error. func convertToURL(rawURL string) (url.URL, error) { if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { rawURL = "https://" + rawURL fmt.Printf("start url: %s\n", rawURL) } u, err := url.Parse(rawURL) if err != nil { return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) } return *u, nil } // getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, // and writes it to stdout. func getLeakProfile(leakySnippet func()) { prof := pprof.Lookup("goroutineleak") defer func() { time.Sleep(2 * time.Second) var content strings.Builder prof.WriteTo(&content, 2) // Ignore non leaked goroutines leaks := strings.SplitSeq(content.String(), "\n\n") for leak := range leaks { if strings.Contains(leak, "(leaked)") { fmt.Println(leak + "\n") } } }() leakySnippet() }