package main import ( "context" "fmt" "net/url" "runtime/pprof" "strings" "time" "github.com/urfave/cli/v3" ) // run processes the given command-line configuration and then runs // the web crawler proper. func run(ctx context.Context, cmd *cli.Command) error { rawURL := cmd.String("url") // If rawURL is empty, it // means that we supplied // --shortcode instead, so use // that. if rawURL == "" { var err error shortcode := cmd.String("shortcode") rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode) if err != nil { return err } } u, err := convertToURL(rawURL) if err != nil { return err } getLeakProfile(func() { classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) }) return nil } // convertToURL parses the given rawURL into a [url.URL]. If the // rawURL is missing a scheme, "https://" is prepended before parsing. // // Return the parsed URL, along with any error. func convertToURL(rawURL string) (url.URL, error) { if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { rawURL = "https://" + rawURL fmt.Printf("start url: %s\n", rawURL) } u, err := url.Parse(rawURL) if err != nil { return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) } return *u, nil } // getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, // and writes it to stdout. func getLeakProfile(leakySnippet func()) { prof := pprof.Lookup("goroutineleak") defer func() { time.Sleep(2 * time.Second) var content strings.Builder prof.WriteTo(&content, 2) // Ignore non leaked goroutines leaks := strings.SplitSeq(content.String(), "\n\n") for leak := range leaks { if strings.Contains(leak, "(leaked)") { fmt.Println(leak + "\n") } } }() leakySnippet() }