send_malware_to_es.py

#! /usr/bin/env python2
# -*- coding: utf-8 -*-
import sys
import urllib2
from elasticsearch import Elasticsearch


class ElasticSearch:

    def __init__(self, host, port):
        self.es = Elasticsearch([{'host': host, 'port': port}])
        self.es.search("*")  # Check connection

    def index(self, json, index):
        self.es.index(index=index, doc_type='ip_or_url', body=json)

    def delete(self, index):
        self.es.indices.delete(index)


class UrlParser(object):

    def __init__(self, Es):
        self.Es = Es

    def download(self):
        try:
            response = urllib2.urlopen(self.url).readlines()
        except:
            print "Impossible de télécharger la liste."
            sys.exit(2)
        print "Liste téléchargée"
        return response

    def delete_old(self):
        try:
            Es.delete(self.index)
        except:
            pass


class AlienVault(UrlParser):

    def __init__(self, Es):
        UrlParser.__init__(self, Es)
        self.type = "AlienVault"
        self.url = "https://reputation.alienvault.com/reputation.generic"
        self.index = "alienvault_blacklist"

    def parse(self):
        self.delete_old()
        response = UrlParser.download(self)
        for line in response[8:]:
            Es.index({"ip": line.split(" ")[0]}, self.index)


class CyberCrime(UrlParser):

    def __init__(self, Es):
        UrlParser.__init__(self, Es)
        self.type = "CyberCrime"
        self.url = "http://cybercrime-tracker.net/all.php"
        self.index = "cybercrime_blacklist"

    def parse(self):
        self.delete_old()
        response = UrlParser.download(self)
        for line in response[0].split("<br />"):
            Es.index({"url": line}, self.index)

if __name__ == "__main__":
    try:
        Es = ElasticSearch("localhost", 9200)
    except:
        print "Unable to connect to ElasticSearch"
        sys.exit(1)
    parser = CyberCrime(Es)
    parser.parse()
    parser = AlienVault(Es)
    parser.parse()