// URLs implements a breadth-first search webcrawler based on the
// example given in section 8.6 of The Go Programming Language.
package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"sync"
)

func main() {
	maxConcurrency := flag.Int("c", 0, "Maximum number of concurrent queue pushes")
	startRawURL := flag.String("url", "", "Entry-point URL")
	maxURLs := flag.Int("max", 0, "Maximum number of URLs to collect (omitted or 0 means no limit)")

	flag.Parse()

	if *maxConcurrency == 0 {
		log.Fatal("Missing -c argument")
	}

	if *maxConcurrency < 1 {
		log.Fatalf("Invalid -c argument: %d", *maxConcurrency)
	}

	if *startRawURL == "" {
		log.Fatal("Missing -url argument")
	}

	if *maxURLs < 0 {
		log.Fatalf("Invalid -max argument: %d", *maxURLs)
	}

	startURL, err := url.Parse(*startRawURL)
	if err != nil {
		log.Fatal(err)
	}

	pool(*startURL, *maxConcurrency, *maxURLs)
}

func pool(startURL url.URL, maxConcurrency, maxURLs int) {
	var wg sync.WaitGroup
	urls := make(chan url.URL)
	outlets := make([]<-chan []url.URL, maxConcurrency)

	for i := range maxConcurrency {
		outlets[i] = worker(&wg, i+1, urls)
	}

	out := fanIn(outlets...)

	go func() {
		urls <- startURL
	}()

	seen := make(map[url.URL]bool)
	for batch := range out {
		for _, u := range batch {
			if !seen[u] {
				seen[u] = true

				go func() {
					urls <- u
				}()

				fmt.Printf("(%d) %s\n", len(seen), &u)
			}
		}
	}

	wg.Wait()
}

func worker(wg *sync.WaitGroup, id int, urls <-chan url.URL) <-chan []url.URL {
	out := make(chan []url.URL)

	wg.Go(func() {
		defer func() { close(out) }()

		for u := range urls {
			//fmt.Printf("(%d) starting %s\n", id, &u)

			doc, err := fetch(u)
			if err != nil {
				log.Print(err)
			}

			batch := findURLs(u, doc)
			//	fmt.Printf("(%d) finished %s\n", id, &u)

			out <- batch
		}
	})

	return out
}

func fanIn(chans ...<-chan []url.URL) <-chan []url.URL {
	out := make(chan []url.URL)
	var wg sync.WaitGroup

	for _, ch := range chans {
		wg.Go(func() {
			for batch := range ch {
				out <- batch
			}
		})
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}