summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--workers.go22
1 files changed, 11 insertions, 11 deletions
diff --git a/workers.go b/workers.go
index 02d7d7d..db22986 100644
--- a/workers.go
+++ b/workers.go
@@ -33,23 +33,23 @@ func workers(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) {
// the urls channel (unseen links.)
for i := range maxConcurrency {
wg.Go(func() {
- for p := range packets {
- batch := getBatch(p.url)
-
- // Convert URLs to Packets. In the
- // process, bump up the depth by 1.
- ps := convertToPackets(batch, p.depth+1)
-
+ for {
select {
case <-ctx.Done():
fmt.Printf("exiting early %d\n", i+1)
- return
- default:
+ case p, ok := <-packets:
+ if !ok {
+ fmt.Printf("exiting because packets is closed %d\n", i+1)
+ }
+
+ batch := getBatch(p.url)
+
+ // Convert URLs to Packets. In the
+ // process, bump up the depth by 1.
+ ps := convertToPackets(batch, p.depth+1)
go func() { worklist <- ps }()
}
}
-
- fmt.Printf("terminating %d\n", i+1)
})
}