Graded Exercise 3

from urllib.request import Request, urlopen
import ssl
from bs4 import BeautifulSoup
import pandas as pd

url = 'https://www.who.int/emergencies/diseases/novel-coronavirus-2019/media-resources/news' 


#################################################
#################################################
### 
 
headers={'User-Agent': 'Mozilla/5.0 (Macinstosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36(KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
req = Request(url, headers=headers)
context = ssl._create_unverified_context()

uClient= urlopen(req, context=context)
html = uClient.read() # html is stored in variable html
uClient.close()
 
#################################################
#################################################
 
soup = BeautifulSoup(html, 'html.parser')
alltext = soup.getText()

maindiv = soup.find('div',class_='sf_colsIn col-md-10')

dataset = [ ]
Title = [ ]
Url = [ ]

for blogpost in soup.find_all('div',class_='list-view--item highlight-widget--content'):  
 title = blogpost.find('p').getText()
 Title.append(title)
 print(Title)

for link in soup.find_all('a'): 
 URL = link['href'] 
 Url.append(URL)

 dataset.append({
 'Title':title,
   'Url':URL
  })

# At the end, when all information is gathered...		
dataset = pd.DataFrame(dataset) # Converting list of dictionaries into dataframe
dataset.to_csv('who-news.csv',sep=';',index=False) # Writing dataframe into CSV file