Create notebook with energy time series data at 15 minutes interval. QuestDB demo

CreateEnergy15minsDataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3193bdd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install pandas questdb\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "df = pd.read_csv('https://data.open-power-system-data.org/time_series/2020-10-06/time_series_15min_singleindex.csv')\n",
    "\n",
    "\n",
    "#upper bound is exclusive, so only 2018 and 2019 here\n",
    "df = df.loc[df[\"utc_timestamp\"].between(\"2018-01-01\", \"2020-01-01\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "653e1852",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_rows = []\n",
    "country_codes = ['AT', 'BE', 'DE', 'HU', 'LU', 'NL']\n",
    "common_columns = ['utc_timestamp']\n",
    "base_column_names = ['load_actual_entsoe_transparency', 'load_forecast_entsoe_transparency']\n",
    "\n",
    "for country_code in country_codes:\n",
    "    local_column_names = [country_code + '_' + sub for sub in base_column_names]\n",
    "    country_columns = common_columns + local_column_names \n",
    "    country_df = df.filter(country_columns)\n",
    "    country_df.dropna(inplace=True)\n",
    "    country_df.columns = common_columns + base_column_names\n",
    "    country_df['country_code'] = country_code\n",
    "    country_rows.append(country_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a00540c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_countries = pd.concat(country_rows).rename(columns={'utc_timestamp': 'timestamp', 'load_actual_entsoe_transparency': 'load_actual', 'load_forecast_entsoe_transparency': 'load_forecast'} )\n",
    "all_countries.timestamp = pd.to_datetime(all_countries.timestamp, format=\"%Y-%m-%dT%H:%M:%SZ\")\n",
    "all_countries.to_parquet('energy_15_mins.parquet.gzip', compression='gzip')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}