summaryrefslogtreecommitdiff
path: root/classic.go
diff options
context:
space:
mode:
Diffstat (limited to 'classic.go')
-rw-r--r--classic.go14
1 files changed, 12 insertions, 2 deletions
diff --git a/classic.go b/classic.go
index 653f8f8..19450c1 100644
--- a/classic.go
+++ b/classic.go
@@ -1,11 +1,14 @@
package main
import (
+ "bytes"
"context"
"fmt"
"log"
"net/url"
"sync"
+
+ "golang.org/x/net/html"
)
func classic(startURL url.URL, maxConcurrency, maxURLs, maxDepth int) {
@@ -73,12 +76,19 @@ loop:
}
func getBatch(u url.URL) []url.URL {
- doc, err := fetch(u)
+ htmlBytes, err := fetch(u)
+ if err != nil {
+ log.Print(err)
+ return nil
+ }
+
+ htmlDoc, err := html.Parse(bytes.NewReader(htmlBytes))
if err != nil {
log.Print(err)
+ return nil
}
- batch := findURLs(u, doc)
+ batch := findURLs(u, htmlDoc)
return batch
}