forked from alistairking/bgpfinder
-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathrouteviews_finder.go
241 lines (207 loc) · 6.29 KB
/
routeviews_finder.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
package bgpfinder
import (
"fmt"
"regexp"
"strings"
"sync"
"time"
"github.com/alistairking/bgpfinder/internal/scraper"
)
type rvDumpType struct {
DumpType DumpType
Duration time.Duration
URL string
Regexp *regexp.Regexp
}
const (
ROUTEVIEWS = "routeviews"
RouteviewsArchiveUrl = "https://archive.routeviews.org/"
)
var (
RouteviewsProject = Project{Name: ROUTEVIEWS}
ROUTEVIEWS_DUMP_TYPES = map[DumpType]rvDumpType{
DumpTypeRib: {
DumpType: DumpTypeRib,
Duration: time.Hour, // ish
URL: "RIBS",
Regexp: regexp.MustCompile(`^rib\.(\d{8}\.\d{4})\.bz2$`),
},
DumpTypeUpdates: {
DumpType: DumpTypeUpdates,
Duration: time.Minute * 15,
URL: "UPDATES",
Regexp: regexp.MustCompile(`^updates\.(\d{8}\.\d{4})\.bz2$`),
},
}
)
// RouteViewsFinder implements the Finder interface
// TODO: refactor a this common caching-finder code out so that RIS and PCH can use it
type RouteViewsFinder struct {
// Cache of collectors
mu *sync.RWMutex
collectors []Collector
collectorsErr error // set if collectors is nil, nil otherwise
}
func NewRouteViewsFinder() *RouteViewsFinder {
f := &RouteViewsFinder{
mu: &sync.RWMutex{},
}
// TODO: turn this into a goroutine that periodically
// refreshes collector list (and handles transient failures)?
c, err := f.getCollectors()
f.collectors = c
f.collectorsErr = err
return f
}
// Projects Retrieves a list of supported projects
func (f *RouteViewsFinder) Projects() ([]Project, error) {
return []Project{RouteviewsProject}, nil
}
// Project Retrieves a specific project by name
func (f *RouteViewsFinder) Project(name string) (Project, error) {
if name == "" || name == ROUTEVIEWS {
return RouteviewsProject, nil
}
return Project{}, nil
}
// Collectors gets a list of collectors for a given project
func (f *RouteViewsFinder) Collectors(project string) ([]Collector, error) {
if project != "" && project != ROUTEVIEWS {
return nil, nil
}
f.mu.RLock()
defer f.mu.RUnlock()
return f.collectors, f.collectorsErr
}
// Collector Gets a specific collector by name
func (f *RouteViewsFinder) Collector(name string) (Collector, error) {
if f.collectorsErr != nil {
return Collector{}, f.collectorsErr
}
f.mu.RLock()
defer f.mu.RUnlock()
// TODO: add a map to avoid the linear search
for _, c := range f.collectors {
if c.Name == name {
return c, nil
}
}
// not found
return Collector{}, fmt.Errorf("collector not found: %+v", name)
}
// getCollectors fetches all collectors from RouteviewsArchiveUrl
func (f *RouteViewsFinder) getCollectors() ([]Collector, error) {
// If we could find a Go rsync client (not a wrapper) we could just do
// `rsync archive.routeviews.org::` and do some light parsing on the
// output.
links, err := scraper.ScrapeLinks(RouteviewsArchiveUrl)
if err != nil {
return nil, fmt.Errorf("failed to get collector list: %v", err)
}
var collectors []Collector
for _, link := range links {
if !strings.HasSuffix(link, "/bgpdata") {
continue
}
link = strings.TrimSuffix(link, "/bgpdata")
link = strings.TrimPrefix(link, "/")
// Handle the only special case for now. This is needed because collector.Name is used in other places
if link == "" {
link = "route-views2"
}
collectors = append(collectors, Collector{
Project: RouteviewsProject,
Name: link,
})
}
return collectors, nil
}
// getCollectorURL constructs the collector URL from collector name
func (f *RouteViewsFinder) getCollectorURL(collector Collector) string {
// usually a collector's url is https://archive.routeviews.org/<collector.Name>bgpdata/
// but for route-views2, the url is https://archive.routeviews.org/bgpdata/
CollectorNameOverride := map[string]string{
"route-views2": "",
}
if override, exists := CollectorNameOverride[collector.Name]; exists {
return RouteviewsArchiveUrl + override + "bgpdata/"
}
return RouteviewsArchiveUrl + collector.Name + "/bgpdata/"
}
// Find BGP dumps matching the specified query
func (f *RouteViewsFinder) Find(query Query) ([]BGPDump, error) {
var results []BGPDump
var allowedPrefixes []string
if query.DumpType == DumpTypeRib || query.DumpType == DumpTypeAny {
allowedPrefixes = append(allowedPrefixes, "rib.")
}
if query.DumpType == DumpTypeUpdates || query.DumpType == DumpTypeAny {
allowedPrefixes = append(allowedPrefixes, "updates.")
}
for _, collector := range query.Collectors {
// baseURL: https://archive.routeviews.org/<collector_name>/bgpdata/
baseURL := f.getCollectorURL(collector)
// monthDirs: YYYY.MM/
monthDirs, err := scraper.ScrapeLinks(baseURL)
if err != nil {
return nil, fmt.Errorf("failed to get month list from %s: %v", baseURL, err)
}
for _, monthDir := range monthDirs {
date, err := time.Parse("2006.01", strings.TrimSuffix(monthDir, "/"))
if err != nil {
// Skip directories that don't match the expected format
continue
}
if monthInRange(date, query) {
for _, prefix := range allowedPrefixes {
finalDir := baseURL + monthDir
if prefix == "rib." {
finalDir += "RIBS/"
} else {
finalDir += "UPDATES/"
}
dumps, err := f.scrapeFilesFromDir(finalDir, prefix, collector, query)
if err != nil {
fmt.Printf("Warning: failed to process %s: %v\n", finalDir, err)
continue
}
results = append(results, dumps...)
}
}
}
}
return results, nil
}
func (f *RouteViewsFinder) scrapeFilesFromDir(dir string, prefix string, collector Collector, query Query) ([]BGPDump, error) {
fmt.Println("Scraping ", dir)
var results []BGPDump
files, err := scraper.ScrapeLinks(dir)
if err != nil {
return nil, fmt.Errorf("failed to get file list from %s: %v", dir, err)
}
for _, file := range files {
if !strings.HasPrefix(file, prefix) {
continue
}
// file: updates.20150801.0000.bz2
parts := strings.Split(strings.TrimSuffix(file, ".bz2"), ".")
if len(parts) != 3 {
continue
}
// Parse both date and time parts in UTC
timestamp, err := time.Parse("20060102.1504", parts[1]+"."+parts[2])
if err != nil {
continue
}
if dateInRange(timestamp, query) {
results = append(results, BGPDump{
URL: dir + file,
Collector: collector,
Duration: getDurationFromPrefix(prefix),
DumpType: getDumpTypeFromPrefix(prefix),
Timestamp: timestamp.Unix(),
})
}
}
return results, nil
}