refactor: move html document creation to getBatch

Also, if there are errors, I log them and simply return a nil slice.

1 files changed, 5 insertions, 6 deletions

diff --git a/fetch.go b/fetch.go
index f81f327..446e3fd 100644
--- a/fetch.go
+++ b/fetch.go
@@ -2,10 +2,9 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
-
-	"golang.org/x/net/html"
 )
 
 // fetch makes a GET request to refURL, returning the HTML contents of
@@ -14,7 +13,7 @@ import (
 // A [url.URL] type is used for refURL to simplify recursive or else
 // repeated use of this function when crawling webpages to, say, build
 // a sitemap.
-func fetch(refURL url.URL) (*html.Node, error) {
+func fetch(refURL url.URL) ([]byte, error) {
 	rawURL := refURL.String()
 
 	// For now we leave the client unconfigured.
@@ -35,10 +34,10 @@ func fetch(refURL url.URL) (*html.Node, error) {
 		return nil, fmt.Errorf("status for %s for %s: %s", http.MethodGet, rawURL, resp.Status)
 	}
 
-	htmlDoc, err := html.Parse(resp.Body)
+	htmlBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("can't parse response body: %w", err)
+		return nil, fmt.Errorf("can't read reponse body into byte buffer")
 	}
 
-	return htmlDoc, nil
+	return htmlBytes, nil
 }