diff options
| author | demo <demo@antix1> | 2026-05-27 11:59:08 -0400 |
|---|---|---|
| committer | demo <demo@antix1> | 2026-05-27 11:59:40 -0400 |
| commit | 81ee780aca0aa0d9ce6999a23ef94e986307e060 (patch) | |
| tree | 18165a8619727c94c4361134ed9d9730f337c1d4 | |
| parent | 1190174edda07fea3f956b58a782eaff2d2213b8 (diff) | |
refactor: move packet definitions to their own file
I also decided to make the packet datatype package private.
| -rw-r--r-- | packet.go | 35 | ||||
| -rw-r--r-- | workers.go | 37 |
2 files changed, 39 insertions, 33 deletions
diff --git a/packet.go b/packet.go new file mode 100644 index 0000000..5b3a5ba --- /dev/null +++ b/packet.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "net/url" +) + +// packet accrues data as it passes through our concurrent +// pipeline. Formerly the web crawler only transmitted [url.URL]'s, +// but usingn a compound data type allows us to add URL +// depth-tracking. +type packet struct { + url url.URL + depth int +} + +// String implements the Stringer interface. We need this mainly +// because a [url.URL]'s String method only works when that URL is a +// pointer. +func (p packet) String() string { + return fmt.Sprintf("[%d] %s", p.depth, &p.url) +} + +// convertToPackets converts the batch of URLs to a slice of packet +// structs, configuring each one with the given depth. +func convertToPackets(batch []url.URL, depth int) []packet { + var ps []packet + + for _, u := range batch { + newPacket := packet{u, depth} + ps = append(ps, newPacket) + } + + return ps +} @@ -14,32 +14,16 @@ import ( - manages urls channel. */ -// Packet accrues data as it passes through our concurrent -// pipeline. Formerly the web crawler only transmitted [url.URL]'s, -// but usingn a compound data type allows us to add URL -// depth-tracking. -type Packet struct { - url url.URL - depth int -} - -// String implements the Stringer interface. We need this mainly -// because a [url.URL]'s String method only works when that URL is a -// pointer. -func (p Packet) String() string { - return fmt.Sprintf("[%d] %s", p.depth, &p.url) -} - // workers launches a worker queue for crawling a given Web domain. func workers(startURL url.URL, maxConcurrency, maxURLs int) { - worklist := make(chan []Packet) + worklist := make(chan []packet) // Unseen URLs. - packets := make(chan Packet) + packets := make(chan packet) go func() { - startPacket := Packet{startURL, 0} - worklist <- []Packet{startPacket} + startPacket := packet{startURL, 0} + worklist <- []packet{startPacket} }() var wg sync.WaitGroup @@ -105,16 +89,3 @@ loop: cancel() wg.Wait() } - -// convertToPackets converts the batch of URLs to a slice of Packet -// structs, configuring each one with the given depth. -func convertToPackets(batch []url.URL, depth int) []Packet { - var ps []Packet - - for _, u := range batch { - newPacket := Packet{u, depth} - ps = append(ps, newPacket) - } - - return ps -} |
