Scraping CENTCOM's PRs with Python

links_collections.json

["http://www.centcom.mil/en/news/articles/jan.-9-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-8-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/u.s.-military-begins-training-iraqi-forces-to-take-on-isil", "http://www.centcom.mil/en/news/articles/jan.-6-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-5-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-4-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-14-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/gen.-austin-sends-letter-to-servicemembers-families-following-cyber-vandali", "http://www.centcom.mil/en/news/articles/jan.-13-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/statement-from-u.s.-central-command-regarding-twitter-youtube-compromise", "http://www.centcom.mil/en/news/articles/jan.-12-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-11-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-5-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-4-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/more-than-200-sunni-tribesmen-join-isf-to-fight-isil", "http://www.centcom.mil/en/news/articles/jan.-2-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jan.-1-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/obama-hagel-mark-end-of-operation-enduring-freedom", "http://www.centcom.mil/en/news/articles/dec.-26-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/dec.-25-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jordanian-pilot-not-downed-by-isil", "http://www.centcom.mil/en/news/articles/uscentcom-commanders-holiday-message-2014", "http://www.centcom.mil/en/news/articles/dec.-24-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/dec.-23-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/dec.-17-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/isaf-condemns-peshwar-school-attack", "http://www.centcom.mil/en/news/articles/dec.-15-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/dec.-12-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/dec.-10-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/jobs-not-over-hagel-tells-troops-in-afghanistan", "http://www.centcom.mil/en/news/articles/dec.-3-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/dec.-1-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/pilot-killed-in-f-16-crash-in-middle-east", "http://www.centcom.mil/en/news/articles/nov.-28-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/uscentcom-commanders-thanksgiving-message", "http://www.centcom.mil/en/news/articles/nov.-26-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/nov.-14-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/u.s.-central-command-hosts-coalition-planning-conference-in-support-of-coun", "http://www.centcom.mil/en/news/articles/nov.-12-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/nov.-10-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/centcom-chief-notes-significant-progress-in-isil-fight", "http://www.centcom.mil/en/news/articles/nov.-7-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-31-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-30-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-29-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-28-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/marines-brits-turn-over-helmand-bases-to-afghan-forces", "http://www.centcom.mil/en/news/articles/oct.-27-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-21-military-airstrikes-continue-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-20-military-airstrikes-continue-against-isil-in-syria-and-in-iraq", "http://www.centcom.mil/en/news/articles/u.s.-military-conducts-aerial-resupply-of-kurdish-forces-fighting-isil", "http://www.centcom.mil/en/news/articles/oct.-18-military-airstrikes-against-isil-in-syria-and-in-iraq-in-support-of", "http://www.centcom.mil/en/news/articles/anti-isil-strategy-working-needs-patience-austin-says", "http://www.centcom.mil/en/news/articles/oct.-17-military-airstrikes-in-support-of-operation-inherent-resolve", "http://www.centcom.mil/en/news/articles/oct.-12-u.s.-military-conducts-airstrikes-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-11-u.s.-military-conducts-airstrikes-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-10-u.s.-military-conducts-airstrikes-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct-9-u.s.-military-conducts-airstrikes-against-isil-in-syria-and-iraq", "http://www.centcom.mil/en/news/articles/oct.-9-u.s.-military-conduct-airstrikes-against-isil-in-syria1", "http://www.centcom.mil/en/news/articles/oct-8-u.s.-military-partner-nations-conduct-airstrikes-against-isil", "http://www.centcom.mil/en/news/articles/campbell-afghanistan-drawdown-remains-on-schedule", "http://www.centcom.mil/en/news/articles/decisive-air-power-thwarts-isils-capabilities", "http://www.centcom.mil/en/news/articles/oct.-2-u.s.-military-partner-nations-conduct-airstrikes-against-isil", "http://www.centcom.mil/en/news/articles/oct.-1-u.s.-military-partner-nation-conduct-airstrikes-against-isil", "http://www.centcom.mil/en/news/articles/marines-stand-up-special-purpose-marine-air-ground-task-force", "http://www.centcom.mil/en/news/articles/sept.-30-u.s.-military-partner-nations-conduct-airstrikes-against-isil", "http://www.centcom.mil/en/news/articles/joint-staff-director-arab-nations-bring-professionalism-to-isil-fight", "http://www.centcom.mil/en/news/articles/obama-asks-all-nations-to-join-fight-against-isil", "http://www.centcom.mil/en/news/articles/afghan-isaf-leaders-discuss-gender-integration-into-ansf", "http://www.centcom.mil/en/news/articles/u.s.-military-conducts-airstrikes-against-isil-in-iraq1", "http://www.centcom.mil/en/news/articles/u.s.-military-partner-nations-conduct-airstrikes-against-isil-in-syria", "http://www.centcom.mil/en/news/articles/sept.-24-u.s.-military-conducts-airstrikes-against-isil-in-iraq-and-syria", "http://www.centcom.mil/en/news/articles/sept.-18-u.s.-military-conducts-airstrikes-against-isil-in-iraq", "http://www.centcom.mil/en/news/articles/sept-17-us-military-conducts-airstrikes-against-isil-in-iraq", "http://www.centcom.mil/en/news/articles/exercise-regional-cooperation-2014-kicks-off", "http://www.centcom.mil/en/news/articles/sept-16-us-military-conducts-airstrikes-against-isil-southwest-of-baghdad", "http://www.centcom.mil/en/news/articles/u.s.-military-conducts-airstrikes-against-isil-near-sinjar-and-southwest-of", "http://www.centcom.mil/en/news/articles/sept.-13-u.s.-military-conducts-airstrikes-against-isil-near-the-mosul-dam", "http://www.centcom.mil/en/news/articles/sept.-7-u.s.-military-conducts-airstrikes-against-isil", "http://www.centcom.mil/en/news/articles/07-us-military-conducts-airstrikes-against-isil-in-support-of-haditha-dam", "http://www.centcom.mil/en/news/articles/sept.-6-u.s.-military-conducts-airstrikes-in-support-of-defense-of-irbil", "http://www.centcom.mil/en/news/articles/sept.-5-u.s.-military-conducts-airstrikes-against-isil", "http://www.centcom.mil/en/news/articles/sept.-4-u.s.-military-conducts-airstrikes-against-isil-in-support-of-mosul", "http://www.centcom.mil/en/news/articles/u.s.-military-conducts-airstrike-against-isil-near-the-mosul-dam", "http://www.centcom.mil/en/news/articles/aug-27-u.s.-military-conducts-airstrikes-against-isil-near-irbil-and-mosul", "http://www.centcom.mil/en/news/articles/aug-26-u.s.-military-conducts-airstrikes-against-isil-near-irbil", "http://www.centcom.mil/en/news/articles/aug-24-us-military-conducts-airstrikes-against-isil-near-irbil-and-mosul", "http://www.centcom.mil/en/news/articles/aug.-23-u.s.-military-conducts-airstrike-against-isil-near-mosul-dam", "http://www.centcom.mil/en/news/articles/aug.-22-u.s.-military-conducts-airstrikes-against-isil-near-mosul-dam", "http://www.centcom.mil/en/news/articles/aug.-21-u.s.-military-conducts-airstrikes-against-isil-near-mosul-dam", "http://www.centcom.mil/en/news/articles/us-military-conducts-airstrikes-against-isil-near-irbil-the-mosul-dam", "http://www.centcom.mil/en/news/articles/aug.-15-u.s.-military-conducts-airstrikes-against-isil-near-sinjar", "http://www.centcom.mil/en/news/articles/us-assessment-team-receives-a-warm-welcome-on-mt.-sinjar", "http://www.centcom.mil/en/news/articles/u.s.-military-conducts-airstrike-against-isil-northeast-of-irbil", "http://www.centcom.mil/en/news/articles/uscentcom-deputy-commander-helps-open-v.a.-hospital-annex", "http://www.centcom.mil/en/news/articles/aug-13-update-on-humanitarian-assistance-operations-near-sinjar"]

script.py

from bs4 import BeautifulSoup
from urllib2 import urlopen
import json 

base_url = "http://www.centcom.mil"
links_collection = []

def make_soup(url):
  html = urlopen(url).read()
  return BeautifulSoup(html, "lxml")

def get_links(section_url):
  soup = make_soup(section_url)

  # Define where the data is in the page 
  table = soup.find("table", "blog")
  tds = table.findAll("td", "contentheading")

  # Grab all the links
  for td in tds:
    links_collection.append(base_url + td.a["href"])

  # Output the whole think in JSON file
  with open("links_collections.json", 'w') as outfile:
    json.dump(links_collection, outfile)

if __name__ == '__main__':

  # Scrape home-page
  url_to_scrape = ("http://www.centcom.mil/en/news")
  
  # Scrape following pages
  pages = get_links(url_to_scrape)
  for i in range(0, 165, 11):
    url_to_scrape = ("http://www.centcom.mil/en/news/P" + str(i))
    pages = get_links(url_to_scrape)