diff options
Diffstat (limited to 'findurls.go')
| -rw-r--r-- | findurls.go | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/findurls.go b/findurls.go index 6bc95d7..8f84936 100644 --- a/findurls.go +++ b/findurls.go @@ -9,7 +9,10 @@ import ( ) // findURLs combs htmlDoc for anchor-tag URLs. The refURL parameter is -// used to resolve each found URL into an absolute URL. +// used to resolve each found URL into an absolute URL, and also +// ensures that we don't crawl outside refURL's parent Web domain. +// +// Return the slice of URLs. func findURLs(refURL url.URL, htmlDoc *html.Node) []url.URL { // Used for collecting all the URLs we find. var rawURLs []url.URL |