blob: db53d69925fa648d0ff325c90a55706c5c3aabdd (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
|
package main
import (
"flag"
"fmt"
"log"
"net/url"
)
func main() {
maxConcurrency := flag.Int("c", 0, "Maximum number of concurrent queue pushes")
startRawURL := flag.String("url", "", "Entry-point URL")
flag.Parse()
if *maxConcurrency == 0 {
log.Fatal("Missing -c argument")
}
if *startRawURL == "" {
log.Fatal("Missing -url argument")
}
startURL, err := url.Parse(*startRawURL)
if err != nil {
log.Fatal(err)
}
/* Main logic starts here. */
worklist := make(chan []url.URL)
go func() {
worklist <- []url.URL{*startURL}
}()
seen := make(map[url.URL]bool)
for list := range worklist {
for _, u := range list {
if !seen[u] {
fmt.Printf("%s\n", &u)
seen[u] = true
go func() {
worklist <- crawl(u)
}()
}
}
}
}
func crawl(u url.URL) []url.URL {
doc, err := fetch(u)
if err != nil {
log.Print(err)
return nil
}
return findURLs(u, doc)
}
|
