dropout_test.lua

require 'nn'

local N = 1000
local FEAT = 64
local S = 2

function nn.SpatialConvolution:reset()
   self.weight:uniform(0, 1)
   self.bias:zero()
end

local function dropout_conv_conv_model()
   local seq = nn.Sequential()
   seq.name = "dropout_conv_conv"
   seq:add(nn.Dropout())
   seq:add(nn.SpatialConvolution(FEAT, FEAT, S, S, S, S, 0, 0))
   seq:add(nn.ReLU())
   seq:add(nn.SpatialConvolution(FEAT, 1, S, S, 1, 1, 0, 0))
   seq:add(nn.ReLU())
   return seq
end
local function dropout_avg_conv_model()
   local seq = nn.Sequential()
   seq.name = "dropout_avg_conv"
   seq:add(nn.Dropout())
   seq:add(nn.SpatialAveragePooling(S, S, S, S))
   seq:add(nn.SpatialConvolution(FEAT, 1, S, S, 1, 1, 0, 0))
   seq:add(nn.ReLU())
   return seq
end
local function dropout_max_conv_model() -- this is bad example
   local seq = nn.Sequential()
   seq.name = "dropout_max_conv"
   seq:add(nn.Dropout())
   seq:add(nn.SpatialMaxPooling(S, S, S, S))
   seq:add(nn.SpatialConvolution(FEAT, 1, S, S, 1, 1, 0, 0))
   seq:add(nn.ReLU())
   return seq
end
local function dropout_conv_max_conv_model()
   local seq = nn.Sequential()
   seq.name = "dropout_conv_max_conv"
   seq:add(nn.Dropout())
   seq:add(nn.SpatialConvolution(FEAT, FEAT, S, S, 1, 1, 0, 0))
   seq:add(nn.ReLU())
   seq:add(nn.SpatialMaxPooling(S, S, S, S))
   seq:add(nn.SpatialConvolution(FEAT, 1, 1, 1, 1, 1, 0, 0))
   seq:add(nn.ReLU())
   return seq
end
local function run_test()
   local models = {dropout_conv_conv_model(), dropout_avg_conv_model(), dropout_max_conv_model(), dropout_conv_max_conv_model()}
   local inputs = torch.Tensor(N, FEAT, S * 2, S * 2):uniform(0, 1)
   local res = {}

   for i = 1, #models do
      local output
      models[i]:training()
      output = models[i]:forward(inputs)
      res[models[i].name] = {train = output:mean()}
      --print(output:size())
      models[i]:evaluate()
      output = models[i]:forward(inputs)
      res[models[i].name]["predict"] = output:mean()
      res[models[i].name]["train/predict"] = res[models[i].name]["train"] / res[models[i].name]["predict"]
   end
   print(res)
end
torch.setdefaulttensortype("torch.FloatTensor")
run_test()

{
  dropout_avg_conv : 
    {
      train/predict : 0.99867722777529
      train : 64.987333992004
      predict : 65.07341129303
    }
  dropout_conv_max_conv : 
    {
      train/predict : 1.0416783267109
      train : 2198.5091051025
      predict : 2110.5451162109
    }
  dropout_max_conv : 
    {
      train/predict : 1.5319914154708
      train : 156.03075260925
      predict : 101.84832044983
    }
  dropout_conv_conv : 
    {
      train/predict : 0.99925533672022
      train : 8856.2585688477
      predict : 8862.8584140625
    }
}