Last active
August 29, 2015 14:02
-
-
Save ShawnMilo/bed1bd88f53493755640 to your computer and use it in GitHub Desktop.
Convert IP to zip & country code via MaxMind (GeoLite) data in RAM, no DB Required. Proof of concept.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"encoding/csv" | |
"encoding/json" | |
"fmt" | |
"io" | |
"log" | |
"math" | |
"net/http" | |
"os" | |
"sort" | |
"strconv" | |
"strings" | |
) | |
// Output is for JSON marshalling. | |
type Output struct { | |
Zip string | |
Country string | |
IP string | |
} | |
// location stores the postal code and country code | |
type location struct { | |
zip string | |
country string | |
} | |
// record contains integers for an IPv4 address | |
// range and the associated zip code. | |
type record struct { | |
start int64 | |
end int64 | |
location | |
} | |
// recSorter is a list of record structs with the sort.Interface implemented. | |
// This means sort.Sort() can be called on it and it will sort by the "start" field. | |
type recSorter []record | |
func (s recSorter) Len() int { return len(s) } | |
func (s recSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } | |
func (s recSorter) Less(i, j int) bool { return s[i].start < s[j].start } | |
// Global slice of all records, to be sorted and queried via binary search. | |
var records = make(recSorter, 5e6) | |
// ipToInt converts a string IPv4 address to an int64. | |
func ipToInt(ip string) (int64, error) { | |
num := float64(0) // the return value | |
const parts = float64(3) // this minus index of dotted quad is the exponent | |
for i, val := range strings.Split(ip, ".") { | |
mult := math.Pow(float64(256), parts-float64(i)) | |
part, err := strconv.ParseFloat(val, 64) | |
if err != nil { | |
return int64(0), err | |
} | |
num += (part * mult) | |
} | |
return int64(num), nil | |
} | |
// init loads the files into memory. | |
func init() { | |
loc, err := os.Open("GeoLiteCity-Location.csv") | |
if err != nil { | |
log.Fatal("Couldn't open location file.", err) | |
} | |
defer loc.Close() | |
reader := csv.NewReader(loc) | |
reader.FieldsPerRecord = 9 | |
locZip := make(map[string]location, 9e5) | |
for { | |
rec, err := reader.Read() | |
if err != nil { | |
if err == io.EOF { | |
break | |
} | |
if strings.Contains(err.Error(), "wrong number of fields in line") { | |
// Skip copyright notice (invalid CSV record). | |
continue | |
} | |
log.Fatal(err) | |
} | |
if rec[4] == "" { | |
// Skip records with no zip code; waste of space. | |
continue | |
} | |
locZip[rec[0]] = location{rec[4], rec[1]} | |
} | |
blocks, err := os.Open("GeoLiteCity-Blocks.csv") | |
if err != nil { | |
log.Fatal("Couldn't open block file.", err) | |
} | |
defer blocks.Close() | |
reader = csv.NewReader(blocks) | |
reader.FieldsPerRecord = 3 | |
for { | |
rec, err := reader.Read() | |
if err != nil { | |
if err == io.EOF { | |
break | |
} | |
if strings.Contains(err.Error(), "wrong number of fields in line") { | |
// Skip copyright notice (invalid CSV record). | |
continue | |
} | |
log.Fatal(err) | |
} | |
start, err := strconv.ParseInt(rec[0], 10, 64) | |
if err != nil { | |
// Skip invalid (non-digit) data. Known to happen on header record. | |
continue | |
} | |
end, err := strconv.ParseInt(rec[1], 10, 64) | |
if err != nil { | |
// Skip invalid (non-digit) data. Known to happen on header record. | |
continue | |
} | |
val := locZip[rec[2]] | |
if val.zip == "" { | |
// Don't waste our time or memory on blank records. | |
continue | |
} | |
records = append(records, record{start, end, val}) | |
} | |
sort.Sort(records) | |
} | |
// ipToZip accepts a string IPv4 address and returns a record. | |
func ipToZip(ip string) record { | |
num, err := ipToInt(ip) | |
if err != nil { | |
// Must be an invalid zip. | |
return record{} | |
} | |
start := 0 | |
end := len(records) - 1 | |
i := 0 | |
for { | |
i++ | |
pos := ((end - start) / 2) + start | |
rec := records[pos] | |
if rec.start <= num && num <= rec.end { | |
return rec | |
} | |
if rec.end < num { | |
start = pos + 1 | |
} else { | |
end = pos - 1 | |
} | |
if end < start { | |
// Our data just doesn't have this one. Sorry. | |
break | |
} | |
} | |
return record{} | |
} | |
func main() { | |
http.HandleFunc("/", handler) | |
err := http.ListenAndServe(":2626", nil) | |
if err != nil { | |
log.Fatal(err) | |
} | |
log.Println("Listening on port 2626.") | |
} | |
// handler receives the requests and returns the responses. | |
func handler(w http.ResponseWriter, r *http.Request) { | |
r.ParseForm() | |
ip := r.FormValue("ip") | |
if ip == "" { | |
ip = r.Form.Get("ip") | |
} | |
rec := ipToZip(ip) | |
out := Output{rec.zip, rec.country, ip} | |
j, err := json.Marshal(out) | |
var ret string | |
if err != nil { | |
ret = "{}" | |
log.Println(err) | |
} else { | |
ret = string(j) | |
} | |
log.Printf("%s\n", ret) | |
fmt.Fprintf(w, ret) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I'm not convinced this is a good approach, but it was fun to try. The idea is that if you don't have a database set up, and don't need one for anything else, you can convert IPs to zip codes. If it doesn't consume too much RAM for you, then it's really convenient.
Also, to update the data you only have to download the new CSV files and restart the program.