diff options
| -rw-r--r-- | workers.go | 22 |
1 files changed, 11 insertions, 11 deletions
@@ -33,23 +33,23 @@ func workers(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) { // the urls channel (unseen links.) for i := range maxConcurrency { wg.Go(func() { - for p := range packets { - batch := getBatch(p.url) - - // Convert URLs to Packets. In the - // process, bump up the depth by 1. - ps := convertToPackets(batch, p.depth+1) - + for { select { case <-ctx.Done(): fmt.Printf("exiting early %d\n", i+1) - return - default: + case p, ok := <-packets: + if !ok { + fmt.Printf("exiting because packets is closed %d\n", i+1) + } + + batch := getBatch(p.url) + + // Convert URLs to Packets. In the + // process, bump up the depth by 1. + ps := convertToPackets(batch, p.depth+1) go func() { worklist <- ps }() } } - - fmt.Printf("terminating %d\n", i+1) }) } |
