diff --git a/.github/readme-images/maxmind-geolite-downloads-screenshot.png b/.github/readme-images/maxmind-geolite-downloads-screenshot.png new file mode 100644 index 0000000..9506845 Binary files /dev/null and b/.github/readme-images/maxmind-geolite-downloads-screenshot.png differ diff --git a/README.md b/README.md index ba0c8d1..519a628 100644 --- a/README.md +++ b/README.md @@ -111,3 +111,37 @@ echo '{"ip": "1.1.1.1"}' | ./zannotate --rdns --geoasn --geoasn-database=/path- ```json {"ip":"1.1.1.1","zannotate":{"geoasn":{"asn":13335,"org":"CLOUDFLARENET"},"rdns":{"domain_names":["one.one.one.one"]}}} ``` +# Acquiring Datasets + +> [!NOTE] +> URLs and instructions may change over time. These are up-to-date as of September 2025. +Below are instructions for getting datasets from the below providers. + +### MaxMind GeoLite ASN and City (Formerly GeoIP2) + +1. [Sign-up form](https://www.maxmind.com/en/geolite2/signup) for MaxMind GeoLite Access +2. Login to your account +3. Go to the "GeoIP / GeoLite" > "Download files" section and download the zip files for either GeoLite ASN or GeoLite City +datasets. + +![GeoLite Download Page](.github/readme-images/maxmind-geolite-downloads-screenshot.png) + +4. Unzip, place the `.mmdb` files somewhere and test with the below. + +#### MaxMind GeoLite City +```shell +echo "171.67.71.209" | ./zannotate --geoip2 --geoip2-database=./path-to-geolite2.mmdb +``` + +```json +{"ip":"171.67.71.209","geoip2":{"city":{"name":"Vallejo","id":5405380},"country":{"name":"United States","code":"US","id":6252001},"continent":{"name":"North America","code":"NA","id":6255149},"postal":{"code":"94590"},"latlong":{"accuracy_radius":50,"latitude":38.1043,"longitude":-122.2442,"metro_code":807,"time_zone":"America/Los_Angeles"},"represented_country":{},"registered_country":{"name":"United States","code":"US","id":6252001},"metadata":{}}} +``` + +#### MaxMind GeoLite ASN +```shell + echo "171.67.71.209" | ./zannotate --geoasn --geoasn-database=/path-to-asn-file.mmdb +``` + +```json +{"ip":"171.67.71.209","geoasn":{"asn":32,"org":"STANFORD"}} +``` \ No newline at end of file diff --git a/geoipasn.go b/geolite-asn.go similarity index 50% rename from geoipasn.go rename to geolite-asn.go index e9fbea9..70f28c2 100644 --- a/geoipasn.go +++ b/geolite-asn.go @@ -24,46 +24,46 @@ import ( log "github.com/sirupsen/logrus" ) -type GeoIPASNOutput struct { +// Note - MaxMind's GeoLite databases used be known as GeoIP2. + +type GeoLiteASNOutput struct { ASN uint `json:"asn,omitempty"` ASNOrg string `json:"org,omitempty"` } -type GeoIPASNAnnotatorFactory struct { +type GeoLiteASNAnnotatorFactory struct { BasePluginConf Path string - Mode string } -type GeoIPASNAnnotator struct { - Factory *GeoIPASNAnnotatorFactory +type GeoLiteASNAnnotator struct { + Factory *GeoLiteASNAnnotatorFactory Reader *geoip2.Reader Id int } -func (fact *GeoIPASNAnnotatorFactory) AddFlags(flags *flag.FlagSet) { - flags.BoolVar(&fact.Enabled, "geoasn", false, "annotate with Maxmind Geolite ASN data") +func (fact *GeoLiteASNAnnotatorFactory) AddFlags(flags *flag.FlagSet) { + flags.BoolVar(&fact.Enabled, "geoasn", false, "annotate with Maxmind GeoLite ASN data") flags.StringVar(&fact.Path, "geoasn-database", "", "path to Maxmind ASN database") - flags.StringVar(&fact.Mode, "geoasn-mode", "mmap", "how to open database: mmap or memory") flags.IntVar(&fact.Threads, "geoasn-threads", 5, "how many geoASN processing threads to use") } -func (fact *GeoIPASNAnnotatorFactory) IsEnabled() bool { +func (fact *GeoLiteASNAnnotatorFactory) IsEnabled() bool { return fact.Enabled } -func (fact *GeoIPASNAnnotatorFactory) GetWorkers() int { +func (fact *GeoLiteASNAnnotatorFactory) GetWorkers() int { return fact.Threads } -func (fact *GeoIPASNAnnotatorFactory) MakeAnnotator(i int) Annotator { - var v GeoIPASNAnnotator +func (fact *GeoLiteASNAnnotatorFactory) MakeAnnotator(i int) Annotator { + var v GeoLiteASNAnnotator v.Factory = fact v.Id = i return &v } -func (fact *GeoIPASNAnnotatorFactory) Initialize(conf *GlobalConf) error { +func (fact *GeoLiteASNAnnotatorFactory) Initialize(_ *GlobalConf) error { if fact.Path == "" { log.Fatal("no GeoIP ASN database provided") } @@ -71,54 +71,43 @@ func (fact *GeoIPASNAnnotatorFactory) Initialize(conf *GlobalConf) error { return nil } -func (fact *GeoIPASNAnnotatorFactory) Close() error { +func (fact *GeoLiteASNAnnotatorFactory) Close() error { return nil } -func (anno *GeoIPASNAnnotator) Initialize() error { - switch anno.Factory.Mode { - case "memory": - bytes, err := os.ReadFile(anno.Factory.Path) - if err != nil { - return fmt.Errorf("unable to open maxmind geoIP ASN database (memory): %w", err) - } - db, err := geoip2.FromBytes(bytes) - if err != nil { - return fmt.Errorf("unable to parse maxmind geoIP ASN database: %w", err) - } - anno.Reader = db - case "mmap": - db, err := geoip2.Open(anno.Factory.Path) - if err != nil { - return fmt.Errorf("unable to load maxmind geoIP ASN database: %w", err) - } - anno.Reader = db - default: - return fmt.Errorf("unrecognized geoIP ASN mode: %s", anno.Factory.Mode) +func (anno *GeoLiteASNAnnotator) Initialize() error { + bytes, err := os.ReadFile(anno.Factory.Path) + if err != nil { + return fmt.Errorf("unable to open maxmind GeoLite ASN database (memory): %w", err) + } + db, err := geoip2.FromBytes(bytes) + if err != nil { + return fmt.Errorf("unable to parse maxmind GeoLite ASN database: %w", err) } + anno.Reader = db return nil } -func (anno *GeoIPASNAnnotator) GetFieldName() string { +func (anno *GeoLiteASNAnnotator) GetFieldName() string { return "geoasn" } -func (anno *GeoIPASNAnnotator) Annotate(ip net.IP) interface{} { +func (anno *GeoLiteASNAnnotator) Annotate(ip net.IP) interface{} { record, err := anno.Reader.ASN(ip) if err != nil { - return &GeoIPASNOutput{} + return &GeoLiteASNOutput{} } - return &GeoIPASNOutput{ + return &GeoLiteASNOutput{ ASN: record.AutonomousSystemNumber, ASNOrg: record.AutonomousSystemOrganization, } } -func (anno *GeoIPASNAnnotator) Close() error { +func (anno *GeoLiteASNAnnotator) Close() error { return anno.Reader.Close() } func init() { - f := new(GeoIPASNAnnotatorFactory) + f := new(GeoLiteASNAnnotatorFactory) RegisterAnnotator(f) } diff --git a/geoipasn_test.go b/geolite-asn_test.go similarity index 86% rename from geoipasn_test.go rename to geolite-asn_test.go index 2665708..0562150 100644 --- a/geoipasn_test.go +++ b/geolite-asn_test.go @@ -24,35 +24,34 @@ func TestGeoIPASNAnnotator(t *testing.T) { tests := []struct { testName string ipAddr net.IP - expectedResult *GeoIPASNOutput + expectedResult *GeoLiteASNOutput }{ { testName: "Positive Test Case, IPv4", ipAddr: net.ParseIP("1.1.1.1"), - expectedResult: &GeoIPASNOutput{ + expectedResult: &GeoLiteASNOutput{ ASN: 13335, ASNOrg: "CLOUDFLARENET", }, }, { testName: "Positive Test Case, IPv6", ipAddr: net.ParseIP("2606:4700:4700::1111"), - expectedResult: &GeoIPASNOutput{ + expectedResult: &GeoLiteASNOutput{ ASN: 13335, ASNOrg: "CLOUDFLARENET", }, }, { testName: "Negative Test Case, Invalid IP", ipAddr: net.ParseIP("999.999.999.999"), - expectedResult: &GeoIPASNOutput{}, + expectedResult: &GeoLiteASNOutput{}, }, { testName: "Negative Test Case, Private IP", ipAddr: net.ParseIP("127.0.0.1"), - expectedResult: &GeoIPASNOutput{}, + expectedResult: &GeoLiteASNOutput{}, }, } - factory := &GeoIPASNAnnotatorFactory{ + factory := &GeoLiteASNAnnotatorFactory{ Path: "./data-snapshots/geolite2_asn.mmdb", - Mode: "mmap", } err := factory.Initialize(nil) if err != nil { @@ -60,7 +59,7 @@ func TestGeoIPASNAnnotator(t *testing.T) { } for _, tt := range tests { t.Run(tt.testName, func(t *testing.T) { - annotator := factory.MakeAnnotator(0).(*GeoIPASNAnnotator) + annotator := factory.MakeAnnotator(0).(*GeoLiteASNAnnotator) err = annotator.Initialize() if err != nil { t.Fatalf("Failed to initialize annotator: %v", err) diff --git a/geoip2.go b/geolite.go similarity index 100% rename from geoip2.go rename to geolite.go