// URLs implements a breadth-first search webcrawler based on the // example given in section 8.6 of The Go Programming Language. package main import ( "context" "fmt" "log" "os" "github.com/urfave/cli/v3" ) const shortcodeFilename = "urls.csv" func main() { if os.Getenv("GOEXPERIMENT") != "goroutineleakprofile" { log.Fatal("Missing GOEXPERIMENT=goroutineleakprofile environment setting") } // Setting shorfile helps especially for when we log errors // without returning them. log.SetFlags(log.LstdFlags | log.Lshortfile) cmd := &cli.Command{ Usage: "A configurable web crawler", Commands: []*cli.Command{ { Name: "get", Usage: "Crawl the target", MutuallyExclusiveFlags: []cli.MutuallyExclusiveFlags{ { Required: true, Flags: [][]cli.Flag{ { &cli.StringFlag{ Name: "shortcode", Usage: "Specify target using a shortcode", }, }, { &cli.StringFlag{ Name: "url", Usage: "Specify target using a URL", }, }, }, }, }, Flags: []cli.Flag{ &cli.IntFlag{ Name: "concurrency", Aliases: []string{"c"}, Usage: "Allowable number of concurrent URL fetches", // Default concurrency setting is 1. Value: 1, }, &cli.IntFlag{ Name: "maxurls", Aliases: []string{"m"}, Usage: "Maximum number of URLs to collect", DefaultText: "no limit", }, &cli.IntFlag{ Name: "depth", Aliases: []string{"d"}, Usage: "Maximum URL depth", DefaultText: "no limit", }, }, Action: run, }, { Name: "shortcode", Usage: "Configure shortcodes", Action: func(ctx context.Context, cmd *cli.Command) error { fmt.Println("shortcode!") return nil }, }, }, } if err := cmd.Run(context.Background(), os.Args); err != nil { fmt.Printf("\n%v\n", err) os.Exit(1) } }