-
Notifications
You must be signed in to change notification settings - Fork 18
/
run_all_crawlers.sh
64 lines (51 loc) · 3.09 KB
/
run_all_crawlers.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
echo "Starting all crawlers"
date
# MANRS
python3 -m iyp.crawlers.manrs.members
# AS Names
python3 -m iyp.crawlers.ripe.as_names
python3 -m iyp.crawlers.bgptools.as_names
python3 -m iyp.crawlers.emileaben.as_names
# AS Peers
python3 -m iyp.crawlers.inetintel.siblings_asdb
# Rankings
python3 -m iyp.crawlers.apnic.eyeball
python3 -m iyp.crawlers.caida.asrank
python3 -m iyp.crawlers.ihr.country_dependency
# BGP data
python3 -m iyp.crawlers.bgpkit.pfx2asn
python3 -m iyp.crawlers.bgpkit.as2rel
python3 -m iyp.crawlers.bgpkit.peerstats
python3 -m iyp.crawlers.ripe.roa
# IHR
python3 -m iyp.crawlers.ihr.local_hegemony
python3 -m iyp.crawlers.ihr.rov
# DNS
python3 -m iyp.crawlers.tranco.top1M
python3 -m iyp.crawlers.cloudflare.top100
#BGP.Tools tags, and anycast prefixes
python3 -m iyp.crawlers.bgptools.tags
python3 -m iyp.crawlers.bgptools.anycast_prefixes
#PeeringDB
python3 -m iyp.crawlers.peeringdb.org
python3 -m iyp.crawlers.peeringdb.ix
# Delegated files
python3 -m iyp.crawlers.nro.delegated_stats
# URL data
python3 -m iyp.crawlers.citizenlab.urldb
# OONI
python3 -m iyp.crawlers.ooni.webconnectivity
python3 -m iyp.crawlers.ooni.facebookmessenger
python3 -m iyp.crawlers.ooni.signal
python3 -m iyp.crawlers.ooni.telegram
python3 -m iyp.crawlers.ooni.whatsapp
python3 -m iyp.crawlers.ooni.httpheaderfieldmanipulation
python3 -m iyp.crawlers.ooni.httpinvalidrequestline
python3 -m iyp.crawlers.ooni.psiphon
python3 -m iyp.crawlers.ooni.riseupvpn
python3 -m iyp.crawlers.ooni.stunreachability
python3 -m iyp.crawlers.ooni.tor
python3 -m iyp.crawlers.ooni.torsf
python3 -m iyp.crawlers.ooni.vanillator
echo "All crawlers finished"
date