From 104577455dcc0d45d0c0b6b9f07221beb1eb455f Mon Sep 17 00:00:00 2001 From: demo Date: Fri, 12 Jun 2026 11:45:59 -0400 Subject: refactor: move sitemap-creation to 'run' and fix missing xml dir --- classic.go | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) (limited to 'classic.go') diff --git a/classic.go b/classic.go index 4a0bcc5..1e264a2 100644 --- a/classic.go +++ b/classic.go @@ -6,13 +6,12 @@ import ( "fmt" "log" "net/url" - "os" "sync" "golang.org/x/net/html" ) -func classic(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) { +func classic(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) map[url.URL]int { worklist := make(chan []packet) var numPendingSends int @@ -75,23 +74,7 @@ loop: cancel() wg.Wait() - // FIXME: eventually, when all crawlers terminate properly, we - // can move this out to main: that is, all crawlers will - // return the same seen map that will be processed by this - // code. - fmt.Println("Generating sitemap...") - sitemap, err := toSitemap(seen, maxDepth, maxURLs) - if err != nil { - log.Fatal(err) - } - - xmlFilename := fmt.Sprintf("xml/%s.xml", startURL.Host) - - if err := os.WriteFile(xmlFilename, []byte(sitemap), 0666); err != nil { - log.Fatal(err) - } - - fmt.Printf("Wrote sitemap to %s\n", xmlFilename) + return seen } func getBatch(u url.URL) []url.URL { -- cgit v1.2.3