From ed605f2643b369a14042527209ceef16e43bbc44 Mon Sep 17 00:00:00 2001 From: demo Date: Fri, 12 Jun 2026 11:30:40 -0400 Subject: refactor: move main business logic into run.go --- main.go | 69 -------------------------------------------------------- run.go | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 69 deletions(-) create mode 100644 run.go diff --git a/main.go b/main.go index cfe7b3a..ceeab89 100644 --- a/main.go +++ b/main.go @@ -6,11 +6,7 @@ import ( "context" "fmt" "log" - "net/url" "os" - "runtime/pprof" - "strings" - "time" "github.com/urfave/cli/v3" ) @@ -91,68 +87,3 @@ func main() { os.Exit(1) } } - -func run(ctx context.Context, cmd *cli.Command) error { - rawURL := cmd.String("url") - - // If rawURL is empty, it - // means that we supplied - // --shortcode instead, so use - // that. - if rawURL == "" { - var err error - shortcode := cmd.String("shortcode") - - rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode) - if err != nil { - return err - } - } - - u, err := convertToURL(rawURL) - if err != nil { - return err - } - - classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) - return nil -} - -// convertToURL parses the given rawURL into a [url.URL]. If the -// rawURL is missing a scheme, "https://" is prepended before parsing. -// -// Return the parsed URL, along with any error. -func convertToURL(rawURL string) (url.URL, error) { - if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { - rawURL = "https://" + rawURL - fmt.Printf("start url: %s\n", rawURL) - } - - u, err := url.Parse(rawURL) - if err != nil { - return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) - } - - return *u, nil -} - -// getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, -// and writes it to stdout. -func getLeakProfile(leakySnippet func()) { - prof := pprof.Lookup("goroutineleak") - defer func() { - time.Sleep(2 * time.Second) - var content strings.Builder - - prof.WriteTo(&content, 2) - // Ignore non leaked goroutines - leaks := strings.SplitSeq(content.String(), "\n\n") - for leak := range leaks { - if strings.Contains(leak, "(leaked)") { - fmt.Println(leak + "\n") - } - } - }() - - leakySnippet() -} diff --git a/run.go b/run.go new file mode 100644 index 0000000..cfa7829 --- /dev/null +++ b/run.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "runtime/pprof" + "strings" + "time" + + "github.com/urfave/cli/v3" +) + +// run processes the given command-line configuration and then runs +// the web crawler proper. +func run(ctx context.Context, cmd *cli.Command) error { + rawURL := cmd.String("url") + + // If rawURL is empty, it + // means that we supplied + // --shortcode instead, so use + // that. + if rawURL == "" { + var err error + shortcode := cmd.String("shortcode") + + rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode) + if err != nil { + return err + } + } + + u, err := convertToURL(rawURL) + if err != nil { + return err + } + + classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) + return nil +} + +// convertToURL parses the given rawURL into a [url.URL]. If the +// rawURL is missing a scheme, "https://" is prepended before parsing. +// +// Return the parsed URL, along with any error. +func convertToURL(rawURL string) (url.URL, error) { + if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { + rawURL = "https://" + rawURL + fmt.Printf("start url: %s\n", rawURL) + } + + u, err := url.Parse(rawURL) + if err != nil { + return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) + } + + return *u, nil +} + +// getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, +// and writes it to stdout. +func getLeakProfile(leakySnippet func()) { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(2 * time.Second) + var content strings.Builder + + prof.WriteTo(&content, 2) + // Ignore non leaked goroutines + leaks := strings.SplitSeq(content.String(), "\n\n") + for leak := range leaks { + if strings.Contains(leak, "(leaked)") { + fmt.Println(leak + "\n") + } + } + }() + + leakySnippet() +} -- cgit v1.2.3