PoC of Xavier initializer for mxnet

tryXavier.R

# Using regerssion as example to test my xavier initilizer

# Data
data(BostonHousing, package="mlbench")
train.ind = seq(1, 506, 3)
train.x = data.matrix(BostonHousing[train.ind, -14])
train.y = BostonHousing[train.ind, 14]
test.x = data.matrix(BostonHousing[-train.ind, -14])
test.y = BostonHousing[-train.ind, 14]

# Computing Symbols
data <- mx.symbol.Variable("data")
fc1 <- mx.symbol.FullyConnected(data, num_hidden=1)
lro <- mx.symbol.LinearRegressionOutput(fc1)

# Training model
mx.set.seed(0)
model <-
  mx.model.FeedForward.create(lro, X=train.x, y=train.y,
                              ctx=mx.cpu(), num.round=50, array.batch.size=20,
                              learning.rate=2e-6, momentum=0.9,
                              eval.metric=mx.metric.rmse)


# Prediction using test data
preds <- predict(model, test.x)
# See diff
diff <- sqrt(mean((preds-test.y)^2))

# Training model with Xavier
mx.set.seed(0)
modelX <-
  mx.model.FeedForward.create(lro, X=train.x, y=train.y,
                              ctx=mx.cpu(), num.round=50, array.batch.size=20,
                              learning.rate=2e-6, momentum=0.9,
                              eval.metric=mx.metric.rmse,
                              initializer = mx.init.Xavier(
                                rnd_type = "gaussian",
                                factor_type = "in",
                                magnitude = 2))
predsX <- predict(modelX, test.x)
diffX <- sqrt(mean((predsX - test.y)^2))

demo/basic_model.R. Feeding initializer = mx.init.Xavier("uniform", "in", 1) yields 0.9616, compared with default .9573