DeepTCN end-to-end demo in PyTorch

DeepTCN.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom tqdm import trange, tqdm\n\nfrom io import BytesIO\nfrom urllib.request import urlopen\nfrom zipfile import ZipFile\n\nfrom pandas import read_csv\nfrom scipy import stats\n\nwindow_size = 192\nstride_size = 24\ntarget_window_size = 24\nnum_covariates = 3\ntrain_start = '2011-01-01 00:00:00'\ntrain_end = '2014-08-31 23:00:00'\ntest_start = '2014-08-25 00:00:00' #need additional 7 days as given info\ntest_end = '2014-09-07 23:00:00'\n\ndef prep_data(data, covariates, data_start, train = True):\n    time_len = data.shape[0]\n    input_size = window_size-stride_size\n    windows_per_series = np.full((num_series), (time_len-input_size-target_window_size) // stride_size)\n    if train: windows_per_series -= (data_start+stride_size-1) // stride_size\n    total_windows = np.sum(windows_per_series)\n    x_input = np.zeros((total_windows, window_size, 1 + num_covariates), dtype='float32')\n    label = np.zeros((total_windows, target_window_size, 1 + num_covariates), dtype='float32')\n    v_input = np.zeros((total_windows, 2), dtype='float32')\n    count = 0\n    for series in trange(num_series): # for each time series\n        for i in range(windows_per_series[series]):\n            if train:\n                window_start = stride_size*i+data_start[series]\n            else:\n                window_start = stride_size*i\n            window_end = window_start+window_size\n            target_window_end = window_end+target_window_size\n            x_input[count, :, 0] = data[window_start:window_end, series]\n            x_input[count, :, 1:1+num_covariates] = covariates[window_start:window_end, :]\n            label[count, :, 0] = data[window_end:target_window_end, series]\n            label[count,:, 1:1+num_covariates] = covariates[window_end:target_window_end, :]\n            nonzero_sum = (x_input[count, 1:input_size, 0]!=0).sum()\n            if nonzero_sum == 0:\n                v_input[count, 0] = 0\n            else:\n                v_input[count, 0] = np.true_divide(x_input[count, :input_size, 0].sum(),nonzero_sum)+1\n                x_input[count, :, 0] = x_input[count, :, 0]/v_input[count, 0]\n                label[count, :, 0] = label[count, :, 0]/v_input[count, 0]\n            count += 1\n    return x_input, v_input, label\n\ndef gen_covariates(times, num_covariates):\n    covariates = np.zeros((times.shape[0], num_covariates))\n    for i, input_time in enumerate(times):\n        covariates[i, 0] = input_time.weekday()\n        covariates[i, 1] = input_time.hour\n        covariates[i, 2] = input_time.month\n    return covariates[:, :num_covariates]","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:36:30.472715Z","iopub.execute_input":"2023-01-20T07:36:30.475173Z","iopub.status.idle":"2023-01-20T07:36:31.392844Z","shell.execute_reply.started":"2023-01-20T07:36:30.475075Z","shell.execute_reply":"2023-01-20T07:36:31.389170Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"name = 'LD2011_2014.txt'\nsave_name = 'elect'\nsave_path = os.path.join('data', save_name)\n\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ncsv_path = os.path.join(save_path, name)\nif not os.path.exists(csv_path):\n    zipurl = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00321/LD2011_2014.txt.zip'\n    with urlopen(zipurl) as zipresp:\n        with ZipFile(BytesIO(zipresp.read())) as zfile:\n            zfile.extractall(save_path)\n\ndata_frame = pd.read_csv(csv_path, sep=\";\", index_col=0, parse_dates=True, decimal=',')\ndata_frame = data_frame.resample('1H',label = 'left',closed = 'right').sum()[train_start:test_end]\ndata_frame.fillna(0, inplace=True) # (32304, 370)\n# generate covariates (has both train and test limits)\ncovariates = gen_covariates(data_frame[train_start:test_end].index, num_covariates) # (32304, 3)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:36:31.394787Z","iopub.execute_input":"2023-01-20T07:36:31.399909Z","iopub.status.idle":"2023-01-20T07:36:58.601299Z","shell.execute_reply.started":"2023-01-20T07:36:31.399872Z","shell.execute_reply":"2023-01-20T07:36:58.600364Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"cov_dims = pd.DataFrame(covariates).nunique().tolist()\ntrain_data = data_frame[train_start:train_end]\ntest_data = data_frame[test_start:test_end]","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:36:58.602815Z","iopub.execute_input":"2023-01-20T07:36:58.603204Z","iopub.status.idle":"2023-01-20T07:36:58.614576Z","shell.execute_reply.started":"2023-01-20T07:36:58.603154Z","shell.execute_reply":"2023-01-20T07:36:58.613783Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\nscaler.fit(train_data)\ntrain_target_df = pd.DataFrame(scaler.transform(train_data), index=train_data.index, columns=train_data.columns)\ntest_target_df = pd.DataFrame(scaler.transform(test_data), index=test_data.index, columns=test_data.columns)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:36:58.617414Z","iopub.execute_input":"2023-01-20T07:36:58.617767Z","iopub.status.idle":"2023-01-20T07:36:59.242899Z","shell.execute_reply.started":"2023-01-20T07:36:58.617732Z","shell.execute_reply":"2023-01-20T07:36:59.241916Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"train_data = train_target_df.values\ntest_data = test_target_df.values\ndata_start = (train_data!=0).argmax(axis=0) #find first nonzero value in each time series\ntotal_time = data_frame.shape[0] #32304\nnum_series = data_frame.shape[1] #370","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:36:59.244597Z","iopub.execute_input":"2023-01-20T07:36:59.244971Z","iopub.status.idle":"2023-01-20T07:36:59.260886Z","shell.execute_reply.started":"2023-01-20T07:36:59.244935Z","shell.execute_reply":"2023-01-20T07:36:59.260046Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"X_train, v_train, y_train = prep_data(train_data, covariates, data_start)\nX_test, v_test, y_test = prep_data(test_data, covariates, data_start, train=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:36:59.263138Z","iopub.execute_input":"2023-01-20T07:36:59.264046Z","iopub.status.idle":"2023-01-20T07:37:08.831191Z","shell.execute_reply.started":"2023-01-20T07:36:59.264009Z","shell.execute_reply":"2023-01-20T07:37:08.830205Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stderr","text":"100%|██████████| 370/370 [00:09<00:00, 38.92it/s]\n100%|██████████| 370/370 [00:00<00:00, 8252.25it/s]\n","output_type":"stream"}]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import RandomSampler\n\nclass TrainDataset(Dataset):\n    def __init__(self, data, label):\n        self.data = data\n        self.label = label\n        self.train_len = self.data.shape[0]\n\n    def __len__(self):\n        return self.train_len\n\n    def __getitem__(self, index):\n        # return time series sequence, current covariates, label sequence, future covariates\n        return (self.data[index,:,0], self.data[index,:,1:1+num_covariates], self.label[index,:,0], self.label[index,:,1:1+num_covariates])\n\nclass TestDataset(Dataset):\n    def __init__(self, data, v, label):\n        self.data = data\n        self.v = v\n        self.label = label\n        self.test_len = self.data.shape[0]\n\n    def __len__(self):\n        return self.test_len\n\n    def __getitem__(self, index):\n        # return time series sequence, current covariates, normalizing stats, label sequence, future covariates\n        return (self.data[index,:,0], self.data[index,:,1:1+num_covariates], self.v[index], self.label[index,:,0], self.label[index,:,1:1+num_covariates])","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:08.832589Z","iopub.execute_input":"2023-01-20T07:37:08.833299Z","iopub.status.idle":"2023-01-20T07:37:10.555274Z","shell.execute_reply.started":"2023-01-20T07:37:08.833260Z","shell.execute_reply":"2023-01-20T07:37:10.554114Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"train_batch_size = 8\n\ntrain_set = TrainDataset(X_train, y_train)\ntest_set = TestDataset(X_test, v_test, y_test)\ntrain_loader = DataLoader(train_set, batch_size=train_batch_size, drop_last=True)\ntest_loader = DataLoader(test_set, batch_size=len(test_set), sampler=RandomSampler(test_set))","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:10.556982Z","iopub.execute_input":"2023-01-20T07:37:10.557600Z","iopub.status.idle":"2023-01-20T07:37:10.564558Z","shell.execute_reply.started":"2023-01-20T07:37:10.557564Z","shell.execute_reply":"2023-01-20T07:37:10.563405Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:10.566888Z","iopub.execute_input":"2023-01-20T07:37:10.567284Z","iopub.status.idle":"2023-01-20T07:37:10.578186Z","shell.execute_reply.started":"2023-01-20T07:37:10.567252Z","shell.execute_reply":"2023-01-20T07:37:10.577071Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"class ResidualBlock(nn.Module):\n    def __init__(self, input_dim, d, stride=1, num_filters=35, p=0.2, k=2, weight_norm=True):\n        super(ResidualBlock, self).__init__()\n        self.k, self.d, self.dropout_fn = k, d, nn.Dropout(p)\n        \n        self.conv1 = nn.Conv1d(input_dim, num_filters, kernel_size=k, dilation=d)\n        self.conv2 = nn.Conv1d(num_filters, num_filters, kernel_size=k, dilation=d)\n        if weight_norm:\n            self.conv1, self.conv2 = nn.utils.weight_norm(self.conv1), nn.utils.weight_norm(self.conv2)\n        \n        self.downsample = nn.Conv1d(input_dim, num_filters, 1) if input_dim != num_filters else None\n    \n    def forward(self, x):\n        out = self.dropout_fn(F.relu(self.conv1(x.float())))\n        out = self.dropout_fn(F.relu(self.conv2(out)))\n        \n        residual = x if self.downsample is None else self.downsample(x)\n        return F.relu(out + residual[:,:,-out.shape[2]:])\n\nclass FutureResidual(nn.Module):\n    def __init__(self, in_features):\n        super(FutureResidual, self).__init__()\n        self.net = nn.Sequential(nn.Linear(in_features=in_features, out_features=in_features),\n#                                  nn.BatchNorm1d(in_features),\n                                 nn.ReLU(),\n                                 nn.Linear(in_features=in_features, out_features=in_features),)\n#                                  nn.BatchNorm1d(in_features),)\n        \n    def forward(self, lag_x, x):\n        out = self.net(x.squeeze())\n        return F.relu(torch.cat((lag_x, out), dim=2))\n\nclass DeepTCN(nn.Module):\n    def __init__(self, cov_dims=cov_dims, num_class=num_series, embedding_dim=20, dilations=[1,2,4,8,16,24,32], p=0.25, device=torch.device('cuda')):\n        super(DeepTCN, self).__init__()\n        self.input_dim, self.cov_dims, self.embeddings, self.device = 1+(len(cov_dims)*embedding_dim), cov_dims, [], device\n        for cov in cov_dims:\n            self.embeddings.append(nn.Embedding(num_class, embedding_dim, device=device))\n        \n        self.encoder = nn.ModuleList()\n        for d in dilations:\n            self.encoder.append(ResidualBlock(input_dim=self.input_dim, num_filters=self.input_dim, d=d))\n        self.decoder = FutureResidual(in_features=self.input_dim-1)\n        self.mlp = nn.Sequential(nn.Linear(1158, 8), nn.BatchNorm1d(8), nn.SiLU(), nn.Dropout(p), nn.Linear(8,1), nn.ReLU())\n    \n    def forward(self, x, current_cov, next_cov):\n        current_cov_embeddings, next_cov_embeddings = [], []\n        for cov_idx, cov_dim in enumerate(self.cov_dims):\n            current_cov_embeddings.append(self.embeddings[cov_idx](current_cov[:,:,cov_idx].to(self.device).long()))\n            next_cov_embeddings.append(self.embeddings[cov_idx](next_cov[:,:,cov_idx].to(self.device).long()))\n        embed_concat = torch.cat(current_cov_embeddings, dim=2).to(self.device)\n        next_cov_concat = torch.cat(next_cov_embeddings, dim=2).to(self.device)\n        \n        encoder_input = torch.cat((x.unsqueeze(2), embed_concat), dim=2)\n        encoder_input = encoder_input.permute(0, 2, 1)\n        \n        for layer in self.encoder:\n            encoder_input = layer(encoder_input)\n        encoder_output = encoder_input.permute(0, 2, 1)\n        encoder_output = torch.reshape(encoder_output, (encoder_output.shape[0], 1, -1))\n        encoder_output = torch.repeat_interleave(encoder_output, next_cov_concat.shape[1], dim=1)\n\n        decoder_output = self.decoder(lag_x=encoder_output, x=next_cov_concat)\n        t, n = decoder_output.size(0), decoder_output.size(1)\n        decoder_output = decoder_output.view(t * n, -1)\n        output = self.mlp(decoder_output.float())\n        output = output.view(t, n, -1)\n        \n        return output.squeeze()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:10.583049Z","iopub.execute_input":"2023-01-20T07:37:10.583375Z","iopub.status.idle":"2023-01-20T07:37:10.614295Z","shell.execute_reply.started":"2023-01-20T07:37:10.583345Z","shell.execute_reply":"2023-01-20T07:37:10.612993Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\nfrom tqdm import trange, tqdm\n\ndef train(model, device=torch.device('cuda'), num_epochs = 1, learning_rate = 1e-3):\n    train_len = len(train_loader)\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    loss_summary = np.zeros((train_len * num_epochs))\n    loss_fn = F.mse_loss\n    \n    for epoch in range(num_epochs):\n        model.train()\n        loss_epoch = np.zeros(len(train_loader))\n\n        pbar = tqdm(train_loader)\n        for (ts_data_batch, current_covs_batch, labels_batch, next_covs_batch) in pbar:\n            optimizer.zero_grad()\n            \n            loss = torch.zeros(1, device=device, dtype=torch.float32)\n            out = model(ts_data_batch.to(device), current_covs_batch.to(device), next_covs_batch.to(device))\n            loss = loss_fn(out.float(), labels_batch.squeeze().to(device).float())\n            \n            pbar.set_description(f\"Loss:{loss.item()}\")\n            loss.backward()\n            optimizer.step()\n        \n        loss_summary[epoch * train_len:(epoch + 1) * train_len] = loss.cpu().detach()\n        \n    return loss_summary, optimizer\n\ndef evaluate(model, optimizer, device=torch.device('cuda')):\n    results = []\n\n    with torch.no_grad():\n        model.eval()\n        loss_epoch = np.zeros(len(train_loader))\n\n        pbar = tqdm(test_loader)\n        for (ts_data_batch, current_covs_batch, v_batch, labels_batch, next_covs_batch) in pbar:\n            optimizer.zero_grad()\n\n            out = model(ts_data_batch.to(device), current_covs_batch.to(device), next_covs_batch.to(device))\n            results.append(out.squeeze(0).cpu())\n\n    predictions = torch.cat(results)\n    criterion = nn.MSELoss()\n    test_rmse = torch.sqrt(criterion(predictions, labels_batch)).item()\n    return test_rmse","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:10.616272Z","iopub.execute_input":"2023-01-20T07:37:10.616988Z","iopub.status.idle":"2023-01-20T07:37:10.633893Z","shell.execute_reply.started":"2023-01-20T07:37:10.616948Z","shell.execute_reply":"2023-01-20T07:37:10.632378Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"model = DeepTCN(device=torch.device('cuda')).cuda()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:10.635832Z","iopub.execute_input":"2023-01-20T07:37:10.636487Z","iopub.status.idle":"2023-01-20T07:37:13.419875Z","shell.execute_reply.started":"2023-01-20T07:37:10.636401Z","shell.execute_reply":"2023-01-20T07:37:13.418903Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"loss, optimizer = train(model, num_epochs=2)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T07:37:13.421294Z","iopub.execute_input":"2023-01-20T07:37:13.421649Z","iopub.status.idle":"2023-01-20T08:03:21.250009Z","shell.execute_reply.started":"2023-01-20T07:37:13.421615Z","shell.execute_reply":"2023-01-20T08:03:21.248503Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stderr","text":"Loss:0.00854562409222126: 100%|██████████| 48591/48591 [13:06<00:00, 61.82it/s]   \nLoss:0.009649071842432022: 100%|██████████| 48591/48591 [13:01<00:00, 62.16it/s]  \n","output_type":"stream"}]},{"cell_type":"code","source":"evaluate(model, optimizer, device=torch.device('cuda'))","metadata":{"execution":{"iopub.status.busy":"2023-01-20T08:03:21.251489Z","iopub.execute_input":"2023-01-20T08:03:21.251925Z","iopub.status.idle":"2023-01-20T08:03:21.379050Z","shell.execute_reply.started":"2023-01-20T08:03:21.251888Z","shell.execute_reply":"2023-01-20T08:03:21.377886Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stderr","text":"100%|██████████| 1/1 [00:00<00:00, 12.43it/s]\n","output_type":"stream"},{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"0.15075178444385529"},"metadata":{}}]}]}