feat: add semaphore to throttle concurrent GET requests

1 files changed, 5 insertions, 0 deletions

diff --git a/main.go b/main.go
index db53d69..ed203d5 100644
--- a/main.go
+++ b/main.go
@@ -32,6 +32,8 @@ func main() {
 		worklist <- []url.URL{*startURL}
 	}()
 
+	sem := make(chan struct{}, *maxConcurrency)
+
 	seen := make(map[url.URL]bool)
 	for list := range worklist {
 		for _, u := range list {
@@ -40,6 +42,9 @@ func main() {
 				seen[u] = true
 
 				go func() {
+					sem <- struct{}{}
+					defer func() { <-sem }()
+
 					worklist <- crawl(u)
 				}()
 			}