diff options
| author | demo <demo@antix1> | 2026-06-12 11:30:40 -0400 |
|---|---|---|
| committer | demo <demo@antix1> | 2026-06-12 11:30:40 -0400 |
| commit | ed605f2643b369a14042527209ceef16e43bbc44 (patch) | |
| tree | 6fd296b9cc069c2eb0199e4518854e5330520a2e | |
| parent | 1e8f0e20ee895394209f070b05137b1bec609534 (diff) | |
refactor: move main business logic into run.go
| -rw-r--r-- | main.go | 69 | ||||
| -rw-r--r-- | run.go | 79 |
2 files changed, 79 insertions, 69 deletions
@@ -6,11 +6,7 @@ import ( "context" "fmt" "log" - "net/url" "os" - "runtime/pprof" - "strings" - "time" "github.com/urfave/cli/v3" ) @@ -91,68 +87,3 @@ func main() { os.Exit(1) } } - -func run(ctx context.Context, cmd *cli.Command) error { - rawURL := cmd.String("url") - - // If rawURL is empty, it - // means that we supplied - // --shortcode instead, so use - // that. - if rawURL == "" { - var err error - shortcode := cmd.String("shortcode") - - rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode) - if err != nil { - return err - } - } - - u, err := convertToURL(rawURL) - if err != nil { - return err - } - - classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) - return nil -} - -// convertToURL parses the given rawURL into a [url.URL]. If the -// rawURL is missing a scheme, "https://" is prepended before parsing. -// -// Return the parsed URL, along with any error. -func convertToURL(rawURL string) (url.URL, error) { - if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { - rawURL = "https://" + rawURL - fmt.Printf("start url: %s\n", rawURL) - } - - u, err := url.Parse(rawURL) - if err != nil { - return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) - } - - return *u, nil -} - -// getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, -// and writes it to stdout. -func getLeakProfile(leakySnippet func()) { - prof := pprof.Lookup("goroutineleak") - defer func() { - time.Sleep(2 * time.Second) - var content strings.Builder - - prof.WriteTo(&content, 2) - // Ignore non leaked goroutines - leaks := strings.SplitSeq(content.String(), "\n\n") - for leak := range leaks { - if strings.Contains(leak, "(leaked)") { - fmt.Println(leak + "\n") - } - } - }() - - leakySnippet() -} @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "runtime/pprof" + "strings" + "time" + + "github.com/urfave/cli/v3" +) + +// run processes the given command-line configuration and then runs +// the web crawler proper. +func run(ctx context.Context, cmd *cli.Command) error { + rawURL := cmd.String("url") + + // If rawURL is empty, it + // means that we supplied + // --shortcode instead, so use + // that. + if rawURL == "" { + var err error + shortcode := cmd.String("shortcode") + + rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode) + if err != nil { + return err + } + } + + u, err := convertToURL(rawURL) + if err != nil { + return err + } + + classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth")) + return nil +} + +// convertToURL parses the given rawURL into a [url.URL]. If the +// rawURL is missing a scheme, "https://" is prepended before parsing. +// +// Return the parsed URL, along with any error. +func convertToURL(rawURL string) (url.URL, error) { + if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { + rawURL = "https://" + rawURL + fmt.Printf("start url: %s\n", rawURL) + } + + u, err := url.Parse(rawURL) + if err != nil { + return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err) + } + + return *u, nil +} + +// getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile, +// and writes it to stdout. +func getLeakProfile(leakySnippet func()) { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(2 * time.Second) + var content strings.Builder + + prof.WriteTo(&content, 2) + // Ignore non leaked goroutines + leaks := strings.SplitSeq(content.String(), "\n\n") + for leak := range leaks { + if strings.Contains(leak, "(leaked)") { + fmt.Println(leak + "\n") + } + } + }() + + leakySnippet() +} |
