XGboost + LR

require(mlbench)
require(caret)
require(MLmetrics)
require(plyr)
require(ade4)

data(Satellite)
summary(Satellite)
dim(Satellite)
my_data <- Satellite
my_labels <- c('cc','non_cc')
my_data$classes <- ifelse(my_data$classes == 'cotton crop',1,0)
my_data$classes <- factor(
  my_data$classes,
  levels = 1:0,
  labels = my_labels
)
summary(my_data$classes)

set.seed(1234)

ind <- createDataPartition(my_data$classes, p = 0.3, list = FALSE)
train_data <- my_data[ind,]
test_data <- my_data[-ind,]
test_data_label <- ifelse(test_data$classes == my_labels[1], 1, 0)


my_fit <- trainControl(
  method = "cv",
  classProbs = T 
)
xgb_grid <- expand.grid(
  nrounds = c(50),
  eta = c(0.05, 0.1) ,
  gamma = 0,
  max_depth = c(1,2,3),              
  colsample_bytree =c(0.5,1),    
  min_child_weight = 1,
  subsample = c(0.7)
)

#xgboost
xgb_model <- train(
  classes ~ ., 
  data = train_data, 
  method = 'xgbTree',
  trControl = my_fit,
  tuneGrid = xgb_grid,
  verbose = F,
  #metric = 'Prec',
  metric = 'Accuracy',
  nthread = 4
)

xgb_pred <- predict(xgb_model, test_data, type = 'raw')
xgb_pred_prob <- predict(xgb_model, test_data, type = 'prob') 
confusionMatrix(xgb_pred, test_data$classes) 

#lr
lr_grid <- expand.grid(
  alpha = c(0,0.3,1),
  lambda = c(0.01,0.1,0.3)
)
lr_model <- train(
  classes~.,
  data = train_data,
  method = 'glmnet',
  trControl = my_fit,
  tuneGrid = lr_grid,
  #metric = 'Prec',
  metric = 'Accuracy',
  standardize = T
)
lr_pred <- predict(lr_model, test_data, type = 'raw')
lr_pred_prob <- predict(lr_model, test_data, type = 'prob')
confusionMatrix(lr_pred, test_data$classes) 

#combine
my_data_encode <- xgb.create.features(
  xgb_model$finalModel,
  as.matrix(my_data[,1:(ncol(my_data)-1)])
)
my_data_encode <- as.data.frame(as.matrix(my_data_encode))
names(my_data_encode) <- paste("f",1:ncol(my_data_encode),sep='')
my_data_encode$classes <- my_data$classes
train_data_encode <- my_data_encode[ind,]
test_data_encode <- my_data_encode[-ind,]

lr_model_combine <- train(
  classes~.,
  data = train_data_encode,
  method = 'glmnet',
  trControl = my_fit,
  tuneGrid = lr_grid,
  #metric = 'Prec',
  metric = 'Accuracy',
  standardize = T
)
lr_pred_combine <- predict(lr_model_combine, test_data_encode, type = 'raw')
lr_pred_prob_combine <- predict(lr_model_combine, test_data_encode, type = 'prob')
confusionMatrix(lr_pred_combine, test_data$classes)