findurls.go


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

package main

import (
	"log"
	"net/url"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

// findURLs combs htmlDoc for anchor-tag URLs. The refURL parameter is
// used to resolve each found URL into an absolute URL.
func findURLs(refURL url.URL, htmlDoc *html.Node) []url.URL {
	// Used for collecting all the URLs we find.
	var rawURLs []url.URL

	for node := range htmlDoc.Descendants() {
		if node.Type == html.ElementNode {
			switch node.DataAtom {
			case atom.A:
				for _, attr := range node.Attr {
					if attr.Key == "href" {
						subURL, err := refURL.Parse(attr.Val)
						if err != nil {
							log.Printf("bad URL: %s", attr.Val)
							continue
						}

						// Filter external
						// URLs from the final
						// result.
						if subURL.Hostname() != refURL.Hostname() {
							continue
						}

						rawURLs = append(rawURLs, *subURL)
					}
				}
			}
		}
	}

	return rawURLs
}