package main import ( "context" "errors" "fmt" "io/fs" "net/url" "os" "runtime/pprof" "strings" "time" "github.com/urfave/cli/v3" ) // run processes the given command-line configuration and then runs // the web crawler proper. func run(ctx context.Context, cmd *cli.Command) error { rawURL := cmd.String("url") // If rawURL is empty, it // means that we supplied // --shortcode instead, so use // that. if rawURL == "" { var err error shortcode := cmd.String("shortcode") rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode) if err != nil { return err } } u, err := convertToURL(rawURL) if err != nil { return err } var seen map[url.URL]int getLeakProfile(func() { seen = classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) }) // Generate a sitemap. fmt.Println("Generating sitemap...") sitemap, err := toSitemap(seen, cmd.Int("depth"), cmd.Int("maxurls")) if err != nil { return fmt.Errorf("creating sitemap: %w", err) } if err := os.Mkdir("xml", 0750); err != nil && !errors.Is(err, fs.ErrExist) { return fmt.Errorf("making sitemap dir: %w", err) } xmlFilename := fmt.Sprintf("xml/%s.xml", u.Host) if err := os.WriteFile(xmlFilename, []byte(sitemap), 0666); err != nil { return fmt.Errorf("writing sitemap: %w", err) } fmt.Printf("Wrote sitemap to %s\n", xmlFilename) return nil } // convertToURL parses the given rawURL into a [url.URL]. If the // rawURL is missing a scheme, "https://" is prepended before parsing. // // Return the parsed URL, along with any error. func convertToURL(rawURL string) (url.URL, error) { if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { rawURL = "https://" + rawURL fmt.Printf("start url: %s\n", rawURL) } u, err := url.Parse(rawURL) if err != nil { return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) } return *u, nil } // getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, // and writes it to stdout. func getLeakProfile(leakySnippet func()) { prof := pprof.Lookup("goroutineleak") defer func() { time.Sleep(2 * time.Second) var content strings.Builder prof.WriteTo(&content, 2) // Ignore non leaked goroutines leaks := strings.SplitSeq(content.String(), "\n\n") for leak := range leaks { if strings.Contains(leak, "(leaked)") { fmt.Println(leak + "\n") } } }() leakySnippet() }