1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
package main
import (
"context"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"runtime/pprof"
"strings"
"time"
"github.com/urfave/cli/v3"
)
// run processes the given command-line configuration and then runs
// the web crawler proper.
func run(ctx context.Context, cmd *cli.Command) error {
rawURL := cmd.String("url")
// If rawURL is empty, it
// means that we supplied
// --shortcode instead, so use
// that.
if rawURL == "" {
var err error
shortcode := cmd.String("shortcode")
rawURL, err = getURLFromShortcode(shortcodeFilename, shortcode)
if err != nil {
return err
}
}
u, err := convertToURL(rawURL)
if err != nil {
return err
}
var seen map[url.URL]int
getLeakProfile(func() {
seen = classic(u, cmd.Int("concurrency"), cmd.Int("maxurls"), cmd.Int("depth"))
})
// Generate a sitemap.
fmt.Println("Generating sitemap...")
sitemap, err := toSitemap(seen, cmd.Int("depth"), cmd.Int("maxurls"))
if err != nil {
return fmt.Errorf("creating sitemap: %w", err)
}
if err := os.Mkdir("xml", 0750); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("making sitemap dir: %w", err)
}
xmlFilename := fmt.Sprintf("xml/%s.xml", u.Host)
if err := os.WriteFile(xmlFilename, []byte(sitemap), 0666); err != nil {
return fmt.Errorf("writing sitemap: %w", err)
}
fmt.Printf("Wrote sitemap to %s\n", xmlFilename)
return nil
}
// convertToURL parses the given rawURL into a [url.URL]. If the
// rawURL is missing a scheme, "https://" is prepended before parsing.
//
// Return the parsed URL, along with any error.
func convertToURL(rawURL string) (url.URL, error) {
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
rawURL = "https://" + rawURL
fmt.Printf("start url: %s\n", rawURL)
}
u, err := url.Parse(rawURL)
if err != nil {
return url.URL{}, fmt.Errorf("can't parse %s: %w", rawURL, err)
}
return *u, nil
}
// getLeakProfile runs a leaky program snippet, extracts the goroutine leak profile,
// and writes it to stdout.
func getLeakProfile(leakySnippet func()) {
prof := pprof.Lookup("goroutineleak")
defer func() {
time.Sleep(2 * time.Second)
var content strings.Builder
prof.WriteTo(&content, 2)
// Ignore non leaked goroutines
leaks := strings.SplitSeq(content.String(), "\n\n")
for leak := range leaks {
if strings.Contains(leak, "(leaked)") {
fmt.Println(leak + "\n")
}
}
}()
leakySnippet()
}
|
