diff options
| author | demo <demo@antix1> | 2026-05-23 10:34:11 -0400 |
|---|---|---|
| committer | demo <demo@antix1> | 2026-05-23 10:34:11 -0400 |
| commit | 2d1d02c95493b02d566c9f38c2c9cda8715eee3f (patch) | |
| tree | 19fe9157a742573edc7f71e28abcdab6ac38a55f /main.go | |
| parent | bc4948442d8eb571ca94fd79898c19b6c204d591 (diff) | |
feat: implement cancellation
I've also added 1., 2., etc. to the printed list of URLs.
Diffstat (limited to 'main.go')
| -rw-r--r-- | main.go | 23 |
1 files changed, 20 insertions, 3 deletions
@@ -1,6 +1,7 @@ package main import ( + "context" "flag" "fmt" "log" @@ -33,26 +34,42 @@ func main() { }() sem := make(chan struct{}, *maxConcurrency) + ctx, cancel := context.WithCancel(context.Background()) seen := make(map[url.URL]bool) + i := 1 + +loop: for list := range worklist { for _, u := range list { + if len(seen) == 30 { + cancel() + break loop + } + if !seen[u] { - fmt.Printf("%s\n", &u) + fmt.Printf("%d. %s\n", i, &u) + i++ seen[u] = true go func() { sem <- struct{}{} defer func() { <-sem }() - worklist <- crawl(u) + worklist <- crawl(ctx, u) }() } } } } -func crawl(u url.URL) []url.URL { +func crawl(ctx context.Context, u url.URL) []url.URL { + select { + case <-ctx.Done(): + return nil + default: + } + doc, err := fetch(u) if err != nil { log.Print(err) |
