refactor: delete 'workers' version of webcrawler

1 files changed, 0 insertions, 99 deletions

diff --git a/workers.go b/workers.go
deleted file mode 100644
index db22986..0000000
--- a/workers.go
+++ /dev/null
@@ -1,99 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/url"
-	"sync"
-)
-
-/*
-   CHANNEL/GOROUTINE NOTES
-
-   main goroutine:
-   - manages urls channel.
-*/
-
-// workers launches a worker queue for crawling a given Web domain.
-func workers(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) {
-	worklist := make(chan []packet)
-
-	// Unseen URLs.
-	packets := make(chan packet)
-
-	go func() {
-		startPacket := packet{startURL, 0}
-		worklist <- []packet{startPacket}
-	}()
-
-	var wg sync.WaitGroup
-	ctx, cancel := context.WithCancel(context.Background())
-
-	// Create maxConcurrency worker goroutines to demultiplex from
-	// the urls channel (unseen links.)
-	for i := range maxConcurrency {
-		wg.Go(func() {
-			for {
-				select {
-				case <-ctx.Done():
-					fmt.Printf("exiting early %d\n", i+1)
-				case p, ok := <-packets:
-					if !ok {
-						fmt.Printf("exiting because packets is closed %d\n", i+1)
-					}
-
-					batch := getBatch(p.url)
-
-					// Convert URLs to Packets. In the
-					// process, bump up the depth by 1.
-					ps := convertToPackets(batch, p.depth+1)
-					go func() { worklist <- ps }()
-				}
-			}
-		})
-	}
-
-	// The main goroutine deduplicates worklist items and sends
-	// unseen ones to the crawlers in a fan-out fashion.
-	seen := make(map[url.URL]int)
-
-	// Used to prettify the running URL listing.
-	count := 1
-
-loop:
-	for batch := range worklist {
-		for _, p := range batch {
-			// We're tracking _depth_ with the seen-map
-			// now, so any unseen URL doesn't have any
-			// depth-entry registered yet.
-			if _, ok := seen[p.url]; !ok {
-				fmt.Printf("%d. %s\n", count, p)
-				count++
-
-				seen[p.url] = p.depth
-
-				if len(seen) == maxURLs {
-					break loop
-				}
-
-				// This works similarly to the classic
-				// example: if we're at the max depth,
-				// don't tell any of the workers to
-				// fetch new URLs.
-				if maxDepth > 0 && p.depth == maxDepth {
-					continue
-				}
-
-				packets <- p
-			}
-		}
-	}
-
-	// We're done writing to the packets channel, so close it.
-	close(packets)
-
-	// There are some in-flight workers as of this point, so
-	// signal a cancel to them.
-	cancel()
-	wg.Wait()
-}