summaryrefslogtreecommitdiff
path: root/packet.go
diff options
context:
space:
mode:
Diffstat (limited to 'packet.go')
-rw-r--r--packet.go35
1 files changed, 35 insertions, 0 deletions
diff --git a/packet.go b/packet.go
new file mode 100644
index 0000000..5b3a5ba
--- /dev/null
+++ b/packet.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+ "fmt"
+ "net/url"
+)
+
+// packet accrues data as it passes through our concurrent
+// pipeline. Formerly the web crawler only transmitted [url.URL]'s,
+// but usingn a compound data type allows us to add URL
+// depth-tracking.
+type packet struct {
+ url url.URL
+ depth int
+}
+
+// String implements the Stringer interface. We need this mainly
+// because a [url.URL]'s String method only works when that URL is a
+// pointer.
+func (p packet) String() string {
+ return fmt.Sprintf("[%d] %s", p.depth, &p.url)
+}
+
+// convertToPackets converts the batch of URLs to a slice of packet
+// structs, configuring each one with the given depth.
+func convertToPackets(batch []url.URL, depth int) []packet {
+ var ps []packet
+
+ for _, u := range batch {
+ newPacket := packet{u, depth}
+ ps = append(ps, newPacket)
+ }
+
+ return ps
+}