Clean up and test links for Water Chronology Map and Climate Commons

clean.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re

# Climate Commons (available stories from WayBack Machine)

base = "https://web.archive.org/web/20160822185508/http://climatecommons.earthjournalism.net/map/data/load_stories.php"

res = requests.get(base).json()

def test_url(record):
	try:
		if requests.get(record['url'], timeout=5).status_code  == 200:
			return {
				"url": record['url'],
				"category": record['category'],
				"latitude": record['latitude'],
				"longitude": record['longitude']
			}
		else:
			return None
	except Exception as e:
		print(e)
		return None

records = [test_url(x) for x in res]


df = pd.DataFrame(filter(None, records))
df.to_csv("~/Desktop/cc-records.csv", index=False)

# Water Conflict Chronology map

ref_links = "http://www.worldwater.org/conflict/refs.html"
ref_html = requests.get(ref_links).content
REF_SOUP = BeautifulSoup(ref_html, 'html.parser')

base = "http://www.worldwater.org/conflict/php/maplist.php?jstr={%22region%22:%22%%22,%22conftype%22:%22%%22,%22epoch%22:%222010,2020%22,%22timestamp%22:1558793884322,%22search%22:%22%22}&_=1558793812356"
res = requests.get(base).content
soup = BeautifulSoup(res, 'html.parser')
records = soup.find_all('div')

def format_entry(record):
	href_list = record.find_all('a', href=True)
	results = []

	def _relative_link(a):
		rel_link = a.get('href').split('#')[-1]
		return rel_link.strip('"').strip("\\")

	references = [_relative_link(a) for a in href_list]
	print(references)

	for r in references:
		p = REF_SOUP.find("a", {"id": r}).find_parent('p')
		a_list = p.find_all('a', attrs={'href': re.compile("^http*")})
		for a in a_list:
			results.append(a.get('href'))

	return results


def clean_everything(records):
	results = []
	for record in records:
		try:
			href_list = format_entry(record)
			for href in href_list:
				if requests.get(href, timeout=5).status_code  == 200:
					results.append({'url': href})
		except Exception, e:
			print(e)
	df = pd.DataFrame(results)
	df.to_csv("~/Desktop/water-records.csv", index=False)
	return df

climate-commons.csv

water.csv