Hyperparameter comparisons (with successive halving, hyperband, stop on plateau and passive random sampling)

_Hyperband-example.ipynb

autoencoder.py

import skorch.utils
from skorch import NeuralNetRegressor
import torch.nn as nn
import torch
import skorch


def _initialize(method, layer, gain=1):
    weight = layer.weight.data
    # _before = weight.data.clone()
    kwargs = {'gain': gain} if 'xavier' in str(method) else {}
    method(weight.data, **kwargs)
    # assert torch.all(weight.data != _before)


class Autoencoder(nn.Module):
    def __init__(self, activation='ReLU', init='xavier_uniform_',
                 **kwargs):
        super().__init__()

        self.activation = activation
        self.init = init
        self._iters = 0

        init_method = getattr(torch.nn.init, init)
        act_layer = getattr(nn, activation)
        act_kwargs = {'inplace': True} if self.activation != 'PReLU' else {}

        gain = 1
        if self.activation in ['LeakyReLU', 'ReLU']:
            name = 'leaky_relu' if self.activation == 'LeakyReLU' else 'relu'
            gain = torch.nn.init.calculate_gain(name)

        inter_dim = 28 * 28 // 4
        latent_dim = inter_dim // 4
        layers = [
            nn.Linear(28 * 28, inter_dim),
            act_layer(**act_kwargs),
            nn.Linear(inter_dim, latent_dim),
            act_layer(**act_kwargs)
        ]
        for layer in layers:
            if hasattr(layer, 'weight') and layer.weight.data.dim() > 1:
                _initialize(init_method, layer, gain=gain)
        self.encoder = nn.Sequential(*layers)
        layers = [
            nn.Linear(latent_dim, inter_dim),
            act_layer(**act_kwargs),
            nn.Linear(inter_dim, 28 * 28),
            nn.Sigmoid()
        ]
        layers = [
            nn.Linear(latent_dim, 28 * 28),
            nn.Sigmoid()
        ]
        for layer in layers:
            if hasattr(layer, 'weight') and layer.weight.data.dim() > 1:
                _initialize(init_method, layer, gain=gain)
        self.decoder = nn.Sequential(*layers)

    def forward(self, x):
        self._iters += 1
        shape = x.size()
        x = x.view(x.shape[0], -1)
        x = self.encoder(x)
        x = self.decoder(x)
        return x.view(shape)
    
class NegLossScore(NeuralNetRegressor):
    steps = 0
    def partial_fit(self, *args, **kwargs):
        
        super().partial_fit(*args, **kwargs)
        self.steps += 1
        
    def score(self, X, y):
        X = skorch.utils.to_tensor(X, device=self.device)
        y = skorch.utils.to_tensor(y, device=self.device)
        
        self.initialize_criterion()
        y_hat = self.predict(X)
        y_hat = skorch.utils.to_tensor(y_hat, device=self.device)
        loss = super().get_loss(y_hat, y, X=X, training=False).item()
        print(f'steps = {self.steps}, loss = {loss}')
        return -1 * loss
    
    def initialize(self, *args, **kwargs):
        super().initialize(*args, **kwargs)
        self.callbacks_ = []
        

noisy_mnist.py

from keras.datasets import mnist
import numpy as np
import skimage.util
import random

import skimage.filters
import skimage
import scipy.signal


def noise_img(x):
    noises = [
        {"mode": "s&p", "amount": np.random.uniform(0.1, 0.1)},
        {"mode": "gaussian", "var": np.random.uniform(0.10, 0.15)},
    ]
    # noise = random.choice(noises)
    noise = noises[1]
    return skimage.util.random_noise(x, **noise)


def train_formatting(img):
    img = img.reshape(28, 28).astype("float32")
    return img.flat[:]


def blur_img(img):
    assert img.ndim == 1
    n = int(np.sqrt(img.shape[0]))
    img = img.reshape(n, n)
    h = np.zeros((n, n))
    angle = np.random.uniform(-5, 5)
    w = random.choice(range(1, 3))
    h[n // 2, n // 2 - w : n // 2 + w] = 1
    h = skimage.transform.rotate(h, angle)
    h /= h.sum()
    y = scipy.signal.convolve(img, h, mode="same")
    return y.flat[:]


def dataset(n=None):
    (x_train, _), (x_test, _) = mnist.load_data()
    x = np.concatenate((x_train, x_test))
    if n:
        x = x[:n]
    else:
        n = int(70e3)
    x = x.astype("float32") / 255.
    x = np.reshape(x, (len(x), 28 * 28))

    y = np.apply_along_axis(train_formatting, 1, x)

    clean = y.copy()

    noisy = y.copy()
    # order = [noise_img, blur_img]
    # order = [blur_img]
    order = [noise_img]

    random.shuffle(order)

    for fn in order:
        noisy = np.apply_along_axis(fn, 1, noisy)

    noisy = noisy.astype("float32")
    clean = clean.astype("float32")
#     noisy = noisy.reshape(-1, 1, 28, 28).astype("float32")
#     clean = clean.reshape(-1, 1, 28, 28).astype("float32")
    return noisy, clean