scrape_euribor.py

import pandas as pd
import requests
from lxml import html, etree
from datetime import datetime as date

from fire import Fire

base_url = 'http://www.euribor-rates.eu/en/euribor-rates-by-year/{}/'

def shorten_label(label):
    label = label.replace(
        ' ', ''
    ).replace(
        'weeks',
        'w'
    ).replace(
        'week',
        'w'
    ).replace(
        'months',
        'm'
    ).replace(
        'month',
        'm'
    )
    return label


def shorten_labels(labels):
    return list(map(lambda x: shorten_label(x), labels))


def get_history_data(min_year=1999, max_year=date.now().year):
    years = [str(year) for year in range(min_year, max_year + 1)]

    tables = []
    for year in years:
        print(year)

        page = requests.get(base_url.format(year))
        tree = html.fromstring(page.text)
        table = tree.xpath('//table')[0]
        table = pd.read_html(etree.tostring(table))[0]

        table.columns = ['Date'] + shorten_labels(table.columns[1:])
        table["Date"] = pd.to_datetime(table["Date"], dayfirst=False, yearfirst=False)
        for col in table.columns[1:]:
            table[col] = table[col].str.replace("%", "").astype(float)

        tables.append(table)

    table = pd.concat(tables, axis=0)
    table = table.sort_values("Date").reset_index(drop=True)
    table.to_csv(f"euribor-{min_year}-{max_year}.csv", index=False)



if __name__ == '__main__':
    Fire(get_history_data)