Custom fold indices using lightgdm

fold_lgbm.py

import numpy as np
from sklearn.datasets import load_digits
from sklearn.metrics import roc_auc_score
import lightgbm as lgbm


if __name__ == '__main__':
    np.random.seed(4242)
    d = load_digits()


    xs = d['data']
    ys = d['target']
    indices = np.where((ys == 1) | (ys == 0))

    ys = ys[indices]
    xs = xs[indices]

    dset = lgbm.Dataset(xs, ys)

    model = None
    params = {
        'boosting_type': 'gbdt',
        'objective': 'binary',
        'learning_rate': 0.05,
        'num_leaves': 35,
        'metric': 'auc',
        'is_unbalance': True,
        'seed': 1024,
        'verbosity': -1
    }

    # create some folds
    niter = 100
    outer_folds = []
    aucv = []

    k = 5
    ss = len(xs)//k
    for i in np.arange(0, len(xs)-ss, ss):
        outer_folds.append((list(range(i, i+ss-10)), list(range(i+ss-10, i+ss))))

    # run a kfold cross validation using loop
    for i, (outer_trainset, outer_testset) in enumerate(outer_folds):
        trainset = lgbm.Dataset(xs[outer_trainset], ys[outer_trainset])
        model = lgbm.train(params, trainset, num_boost_round=niter)


        ydelta = model.predict(xs[outer_testset])
        aucv.append(roc_auc_score(ys[outer_testset], ydelta))

    # run a kfold cross validation using lightgbm cv
    r = lgbm.cv(params, folds=outer_folds, nfold=k, train_set=dset, shuffle=False, num_boost_round=niter)

    auc = r['auc-mean'][niter-1]
    stdev = r['auc-stdv'][niter-1]

    print('lightgbm cv auc: {:.4f} (+-{:.4f})'.format(auc, stdev))
    print('manual cv auc: {:.4f} (+-{:.4f})'.format(np.mean(aucv), np.std(aucv)))

lightgbm cv auc: 1.0000 (+-0.0000)
manual cv auc: 0.9792 (+-0.0361)