feat: update the classic crawler to track depth via packets

1 files changed, 14 insertions, 9 deletions

diff --git a/classic.go b/classic.go
index 0b2bfa8..bd689fc 100644
--- a/classic.go
+++ b/classic.go
@@ -9,16 +9,18 @@ import (
 )
 
 func classic(startURL url.URL, maxConcurrency, maxURLs int) {
-	worklist := make(chan []url.URL)
+	worklist := make(chan []packet)
 	var numPendingSends int
 
 	numPendingSends++
 	go func() {
-		worklist <- []url.URL{startURL}
+		startPacket := packet{startURL, 0}
+		worklist <- []packet{startPacket}
 	}()
 
-	// Crawl the web concurrently.
-	seen := make(map[url.URL]bool)
+	// Crawl the web concurrently. Map URLs to their depth (i.e
+	// how many links we have to work through to find the URL.)
+	seen := make(map[url.URL]int)
 	count := 1
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -28,12 +30,12 @@ func classic(startURL url.URL, maxConcurrency, maxURLs int) {
 loop:
 	for ; numPendingSends > 0; numPendingSends-- {
 		batch := <-worklist
-		for _, u := range batch {
-			if !seen[u] {
-				fmt.Printf("%d. %s\n", count, &u)
+		for _, p := range batch {
+			if _, ok := seen[p.url]; !ok {
+				fmt.Printf("%d. %s\n", count, &p)
 				count++
 
-				seen[u] = true
+				seen[p.url] = p.depth
 				if len(seen) == maxURLs {
 					break loop
 				}
@@ -46,7 +48,10 @@ loop:
 					select {
 					case <-ctx.Done():
 						return
-					case worklist <- getBatch(u):
+					default:
+						batch := getBatch(p.url)
+						ps := convertToPackets(batch, p.depth+1)
+						worklist <- ps
 					}
 				})
 			}