Skip to content

Instantly share code, notes, and snippets.

@markcam1
Last active April 25, 2019 18:18
Show Gist options
  • Save markcam1/405e8ed12d155f64c20613656349c4f9 to your computer and use it in GitHub Desktop.
Save markcam1/405e8ed12d155f64c20613656349c4f9 to your computer and use it in GitHub Desktop.
Python get URL, and parse HTML string
# GREAT SCHOOL API: https://www.greatschools.org/api/docs/school-search/
import requests
import json
import logging
import getpass
import bs4
#logging config:
logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s - %(message)s')
logging.debug('Start of program')
KEY='X0X0X0X0X0X0X0X0X0X0X'
STATE='CA'
QUERY='Alameda'
LIMIT='2'
PARAMS = {'key':KEY, 'state':STATE, 'limit':LIMIT,'q':QUERY}
BASE_URL='https://api.greatschools.org/search/schools'
# Send HTTP GET request to server and attempt to receive a response
# auth=basicAuthCredential
try:
response = requests.get(BASE_URL,PARAMS)
except requests.exceptions.RequestException as e:
print (e)
# If the HTTP GET request can be served
if response.status_code == 200:
rt = response.text
print(rt)
print('\n\n.....download complete........')
logging.debug('Length of response text (%s)' % (len(rt)))
soup = bs4.BeautifulSoup(rt, 'html.parser')
#print(soup.prettify())
test = soup.school
print('\n\n School search-----------------------\n\n')
#print(test)
print('\n\n-----School Find All----')
matchSchool = soup.find_all("school")
print(matchSchool)
print('\n' + str(len(matchSchool)) + '\n')
print(type(matchSchool))
for m in range(len(matchSchool)):
print(matchSchool[m].phone.string)
print('\n\n-----------------------\n\n')
#print('index of contents')
#print(soup.school.contents[1:4])
##schoolData = soup.school
##
##for sch in schoolData.children:
## print(sch)
logging.debug('End of program')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment