diff options
Diffstat (limited to 'classic.go')
| -rw-r--r-- | classic.go | 21 |
1 files changed, 2 insertions, 19 deletions
@@ -6,13 +6,12 @@ import ( "fmt" "log" "net/url" - "os" "sync" "golang.org/x/net/html" ) -func classic(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) { +func classic(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) map[url.URL]int { worklist := make(chan []packet) var numPendingSends int @@ -75,23 +74,7 @@ loop: cancel() wg.Wait() - // FIXME: eventually, when all crawlers terminate properly, we - // can move this out to main: that is, all crawlers will - // return the same seen map that will be processed by this - // code. - fmt.Println("Generating sitemap...") - sitemap, err := toSitemap(seen, maxDepth, maxURLs) - if err != nil { - log.Fatal(err) - } - - xmlFilename := fmt.Sprintf("xml/%s.xml", startURL.Host) - - if err := os.WriteFile(xmlFilename, []byte(sitemap), 0666); err != nil { - log.Fatal(err) - } - - fmt.Printf("Wrote sitemap to %s\n", xmlFilename) + return seen } func getBatch(u url.URL) []url.URL { |