summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordemo <demo@antix1>2026-05-27 11:59:08 -0400
committerdemo <demo@antix1>2026-05-27 11:59:40 -0400
commit81ee780aca0aa0d9ce6999a23ef94e986307e060 (patch)
tree18165a8619727c94c4361134ed9d9730f337c1d4
parent1190174edda07fea3f956b58a782eaff2d2213b8 (diff)
refactor: move packet definitions to their own file
I also decided to make the packet datatype package private.
-rw-r--r--packet.go35
-rw-r--r--workers.go37
2 files changed, 39 insertions, 33 deletions
diff --git a/packet.go b/packet.go
new file mode 100644
index 0000000..5b3a5ba
--- /dev/null
+++ b/packet.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+ "fmt"
+ "net/url"
+)
+
+// packet accrues data as it passes through our concurrent
+// pipeline. Formerly the web crawler only transmitted [url.URL]'s,
+// but usingn a compound data type allows us to add URL
+// depth-tracking.
+type packet struct {
+ url url.URL
+ depth int
+}
+
+// String implements the Stringer interface. We need this mainly
+// because a [url.URL]'s String method only works when that URL is a
+// pointer.
+func (p packet) String() string {
+ return fmt.Sprintf("[%d] %s", p.depth, &p.url)
+}
+
+// convertToPackets converts the batch of URLs to a slice of packet
+// structs, configuring each one with the given depth.
+func convertToPackets(batch []url.URL, depth int) []packet {
+ var ps []packet
+
+ for _, u := range batch {
+ newPacket := packet{u, depth}
+ ps = append(ps, newPacket)
+ }
+
+ return ps
+}
diff --git a/workers.go b/workers.go
index 43e398d..4d646fd 100644
--- a/workers.go
+++ b/workers.go
@@ -14,32 +14,16 @@ import (
- manages urls channel.
*/
-// Packet accrues data as it passes through our concurrent
-// pipeline. Formerly the web crawler only transmitted [url.URL]'s,
-// but usingn a compound data type allows us to add URL
-// depth-tracking.
-type Packet struct {
- url url.URL
- depth int
-}
-
-// String implements the Stringer interface. We need this mainly
-// because a [url.URL]'s String method only works when that URL is a
-// pointer.
-func (p Packet) String() string {
- return fmt.Sprintf("[%d] %s", p.depth, &p.url)
-}
-
// workers launches a worker queue for crawling a given Web domain.
func workers(startURL url.URL, maxConcurrency, maxURLs int) {
- worklist := make(chan []Packet)
+ worklist := make(chan []packet)
// Unseen URLs.
- packets := make(chan Packet)
+ packets := make(chan packet)
go func() {
- startPacket := Packet{startURL, 0}
- worklist <- []Packet{startPacket}
+ startPacket := packet{startURL, 0}
+ worklist <- []packet{startPacket}
}()
var wg sync.WaitGroup
@@ -105,16 +89,3 @@ loop:
cancel()
wg.Wait()
}
-
-// convertToPackets converts the batch of URLs to a slice of Packet
-// structs, configuring each one with the given depth.
-func convertToPackets(batch []url.URL, depth int) []Packet {
- var ps []Packet
-
- for _, u := range batch {
- newPacket := Packet{u, depth}
- ps = append(ps, newPacket)
- }
-
- return ps
-}