jjsantanna

gzcat raw-daily-2019-07-17.json.gz |jq "{data, ip_str, port, location, asn}" -n > shodan_20190717_lessfields.json

cat shodan.json  | jq -r  '. | {ip: .ip_str, port: .port, cc: .location.country_code3, data: .data} | @json' > /tmp/shodan1.txt

-----------
shodan_raw_filename='shodan_output.json.gz'

import json
import gzip

jjsantanna

.DS_Store
*.log
*.aux
*.dvi
*.lof
*.lot
*.bit
*.idx
*.glo
*.bbl

jjsantanna

mypath ="/Users/santannajj/Desktop"

import os
import pandas as pd
import numpy as np


for file in os.listdir(mypath):
    if file.endswith(".log"):
        path_file = os.path.join(mypath, file)

jjsantanna

#!/bin/bash
rm all_summaries.csv; ls |grep .log |while read file; do echo $file; tail -1 $file >> all_summaries.csv; done

jjsantanna

def top_n_dataframe(n,dataframe_field):
    
    top_n = n
    field_name = dataframe_field.name
    top  = dataframe_field.value_counts()[:top_n].to_frame().reset_index()

    new_row = pd.DataFrame(data = {
        'hits' : [ dataframe_field.value_counts()[top_n:].sum()],
        field_name : ['others'],
    })

jjsantanna

import glob #for reading recursively
import pandas as pd

def read_NL_dataset(output_dir):
	all_files = glob.glob(output_dir+'/<FILE_NAMEish>*.hdf5') 
	df_all = pd.DataFrame()
	print(all_files)
    
	for all_file in all_files:
	    df_temp = pd.read_hdf(all_file, 'shodan')

jjsantanna

### CRAWLER wikipedia for reserved IPs
import cfscrape
from lxml import etree
import pandas as pd
url="https://en.wikipedia.org/wiki/Reserved_IP_addresses"
scraper = cfscrape.create_scraper()
scraped_html=scraper.get(url).content
tables = pd.read_html(scraped_html) # Returns list of all tables on page
reserved_ipv4=tables[0][0].drop(0)
reserved_ipv6=tables[1][0].drop(0)

jjsantanna

Notes to buy a motocicle

Types of motorcycle:

• Sport Touring
• Adventure (ADV)
• Standard
• (Super)Sport
• Touring
• Cafe racer
• Roadster

jjsantanna

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import plotly
plotly.offline.init_notebook_mode()

# Data for plotting
t = np.arange(0.0, 2.0, 0.01)
s = 1 + np.sin(2 * np.pi * t)

jjsantanna

df['ip']=df['column'].str.extract('([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)')

##########################
########### IP to hostname
import socket
import pandas as pd

def ip2hostname (df_series_ip):
    df_output = pd.DataFrame()
    for ip in df_series_ip: