Get Brazilian economic information from UOL's website

get_uol_finance.py

from os import system
from requests import get
from bs4 import BeautifulSoup

headers = {'Accept': 'text/html, application/xhtml+xml, application/xml',
            'Accept-Encoding': 'gzip, deflate, sdch',
            'Accept-Language': 'en-US, en',
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)'
}

url = 'https://economia.uol.com.br/cotacoes/indices-economicos/'
r = get(url, headers=headers, allow_redirects=True)
soup = BeautifulSoup(r.content, "lxml")
sections = soup.find_all('section', {"class": "mod-cotacoes mod composto"})

with open('tables.html', 'w') as f:
    for section in sections:
        tables = section.find_all('table')
        for table in tables:
            f.write(str(table))
            f.write('<br><br>')

system("bash -c 'xdg-open tables.html'")

parse_uol_finance.py

from bs4 import BeautifulSoup

with open('tables.html') as f:
    soup = BeautifulSoup(f.read(), "lxml")

tables = soup.find_all('table')

table = tables[0]
content = []
for tr in table.find_all('tr'):
    for td in tr.find_all('td'):
        content.append(td.text.strip())

results = {}

results['commodities'] = [{'produto': content[5],
            'unidade': content[6],
            'compra': content[7],
            'venda': content[8],
            'variacao': content[9]},
            {'produto': content[10],
            'unidade': content[11],
            'compra': content[12],
            'venda': content[13],
            'variacao': content[14]},
            {'produto': content[15],
            'unidade': content[16],
            'compra': content[17],
            'venda': content[18],
            'variacao': content[19]}]

table = tables[1]
content = []
for tr in table.find_all('tr'):
    for td in tr.find_all('td'):
        content.append(td.text.strip())

results['moedas'] = [{'produto': content[5],
            'compra': content[6],
            'venda': content[7],
            'variacao': content[8]},
            {'produto': content[9],
            'compra': content[10],
            'venda': content[11],
            'variacao': content[12]},
            {'produto': content[13],
            'compra': content[14],
            'venda': content[15],
            'variacao': content[16]}]

table = tables[2]
content = []
for tr in table.find_all('tr'):
    for td in tr.find_all('td'):
        content.append(td.text.strip())

results['bolsas'] = [{'produto': content[3],
            'variacao': content[4]},
            {'produto': content[5],
            'variacao': content[6]},
            {'produto': content[7],
            'variacao': content[8]},
            {'produto': content[9],
            'variacao': content[10]}]

table = tables[3]
content = []
for tr in table.find_all('tr'):
    for td in tr.find_all('td'):
        content.append(td.text.strip())

results['indices'] = [{'indice': content[3],
            'valor': content[4],
            'atualizacao': content[5]},
            {'indice': content[6],
            'valor': content[7],
            'atualizacao': content[8]},
            {'indice': content[9],
            'valor': content[10],
            'atualizacao': content[11]},
            {'indice': content[12],
            'valor': content[13],
            'atualizacao': content[14]},
            {'indice': content[15],
            'valor': content[16],
            'atualizacao': content[17]},
            {'indice': content[18],
            'valor': content[19],
            'atualizacao': content[20]}]

table = tables[4]
content = []
for tr in table.find_all('tr'):
    for td in tr.find_all('td'):
        content.append(td.text.strip())

results['inflacao'] = [{'indice': content[3],
            'mes': content[4],
            'valor': content[5]},
            {'indice': content[6],
            'mes': content[7],
            'valor': content[8]},
            {'indice': content[9],
            'mes': content[10],
            'valor': content[11]},
            {'indice': content[12],
            'mes': content[13],
            'valor': content[14]}]

for key in results:
    print('=' * 72)
    print(key)
    for subkey in results[key]:
        print(subkey)