diff options
Diffstat (limited to 'main.go')
| -rw-r--r-- | main.go | 80 |
1 files changed, 23 insertions, 57 deletions
@@ -3,7 +3,7 @@ package main import ( - "flag" + "context" "fmt" "log" "net/url" @@ -12,7 +12,7 @@ import ( "strings" "time" - "git.brandonirizarry.xyz/hamlet" + "github.com/urfave/cli/v3" ) func main() { @@ -24,65 +24,31 @@ func main() { // without returning them. log.SetFlags(log.LstdFlags | log.Lshortfile) - maxConcurrency := flag.Int("c", 0, "Maximum number of concurrent queue pushes") - urlArg := flag.String("url", "", "Entry-point URL") - maxURLs := flag.Int("max", 0, "Maximum number of URLs to collect (omitted or 0 means no limit)") - maxDepth := flag.Int("depth", 0, "Maximum URL depth (omitted or 0 means no limit)") - shortcode := flag.String("shortcode", "", "URL shortcode") - shortcodeFilename := flag.String("scfile", "urls.csv", "Shortcode CSV file") - - flag.Parse() - - // Vet the given CLI arguments for things like negative or - // missing values. - if *maxConcurrency == 0 { - log.Fatal("Missing -c argument") - } - - if *maxConcurrency < 1 { - log.Fatalf("Invalid -c argument: %d", *maxConcurrency) - } - - if *maxURLs < 0 { - log.Fatalf("Invalid -max argument: %d", *maxURLs) - } - - if *maxDepth < 0 { - log.Fatalf("Invalid -depth argument: %d", *maxDepth) - } - - // Use hamlet to vet the validity of the given command-line - // argument configuration provided by the user. - shortcodePresent := hamlet.Present(*shortcode) - urlArgPresent := hamlet.Present(*urlArg) - if hamlet.Xor(urlArgPresent, shortcodePresent).Deny() { - log.Fatal("Need exactly one of -url or -shortcode") - } - - if hamlet.If(hamlet.Present(*shortcodeFilename), shortcodePresent).Deny() { - log.Fatal("-scfile implies use of -shortcode") + cmd := &cli.Command{ + Usage: "A configurable web crawler", + Commands: []*cli.Command{ + { + Name: "get", + Usage: "Crawl the target", + Action: func(ctx context.Context, cmd *cli.Command) error { + fmt.Println("get!") + return nil + }, + }, + { + Name: "shortcode", + Usage: "Configure shortcodes", + Action: func(ctx context.Context, cmd *cli.Command) error { + fmt.Println("shortcode!") + return nil + }, + }, + }, } - startRawURL := *urlArg - - if shortcodePresent.Assert() { - var err error - startRawURL, err = getURLFromShortcode(*shortcodeFilename, *shortcode) - if err != nil { - log.Fatal(err) - } - } - - startURL, err := convertToURL(startRawURL) - if err != nil { + if err := cmd.Run(context.Background(), os.Args); err != nil { log.Fatal(err) } - - // Our web crawlers use concurrency: check if any goroutines - // have leaked. - getLeakProfile(func() { - classic(startURL, *maxConcurrency, *maxURLs, *maxDepth) - }) } // convertToURL parses the given rawURL into a [url.URL]. If the |