getipfromlog.py

# -*- coding=utf-8 -*-

# Update 2-12
# Add spam IP list and ignore the spam IPs
# Add a time delay for ipinfo API - There is a limit usage for the ipinfo api: 1000/day for free, if you want more, you can buy from the website
# You can edit and add your spam ips list file yourself, remember separate your ip address with other information by one space.

import urllib
import json
import os
import time
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

# change current director to your files directory
os.chdir("/var/www/logs")

url="http://ipinfo.io/"
ipdetails = []

# get the details of this ip
def ip_detail(ip):
        data=urllib.urlopen(url+ip+"/json").read()
        datadict=json.loads(data)
        # it seems that some ips don't have any org information
        if "org" in datadict:
                return ip+u" \t该IP的详细信息为:\t" +str(datadict["country"]) + " -- " + str(datadict["region"]) + " -- " + str(datadict["city"]) + " -- " + str(datadict["org"])+"\t"+time.strftime("%d/%m - %H:%M:%S")+"\n"
        else:
                return ip+u" \t该IP的详细信息为:\t" +str(datadict["country"]) + " -- " + str(datadict["region"]) + " -- " + str(datadict["city"]) + " -- no org details\t"+time.strftime("%d/%m - %H:%M:%S")+"\n"
  
# get all ips from the log file
def getips(filename):
        ips = []
        with open(filename,"r") as flog:
                for i in flog.readlines():
                        if(i!="\n"):
                                ip = i.split(" ")[0]
                                ips.append(ip)
        return list(set(ips))

# output the details into the output file
def output(filename,listname):
        with open(filename,"a") as wip:
                wip.write("\n".join(listname))

if __name__=="__main__":
        allips = getips("nginx-access.log")
        oldips = getips("ipdetails.txt")
        spamips = getips("spamips.txt")
        errorips = []
        eips = getips("errorips.txt")
        print len(allips)
        print len(oldips)
        print len(eips)
        eips.extend(oldips)
        eips.extend(spamips)
        print len(eips)
        newips = [i for i in allips if i not in eips]
        # get the new ips that we never looked before
        print len(newips)
        for i in newips:
                time.sleep(0.01)
                # protection for connection
                try:
                        ipdetails.append(ip_detail(i))
                except Exception,e:
                        print i
                        print e
                        errorips.append(i+" " +str(e))
                        # store the error ones and save it into a single file
        output("ipdetails.txt",ipdetails)
        output("errorips.txt",errorips)