Recursive Feature Elimination

Recursive Feature Elimination

# Recursive Feature Elimination
from sklearn.feature_selection import RFE
select = RFE(RandomForestClassifier(n_estimators=100, random_state=42),
             n_features_to_select=15)
select.fit(X_train_full, y_train_full.values.ravel())
X_train_rfe = select.transform(X_train_full)
X_test_rfe = select.transform(X_test_full)
mask = select.get_support()
print(mask)
plt.matshow(mask.reshape(1, -1), cmap='gray_r')
plt.xlabel("Technical Indexes")

# GradientBoost Classifier
print('--------------------------Without Recursive Feature Elimination-------------------------------------')
pipe_gb = Pipeline([('scl', StandardScaler()), ('est', GradientBoostingClassifier(random_state=39))])
pipe_gb.fit(X_train_full, y_train_full.values.ravel())
print('Train Accuracy: {:.3f}'.format(accuracy_score(y_train_full.values.ravel(), pipe_gb.predict(X_train_full))))
print('Test Accuracy: {:.3f}'.format(accuracy_score(y_test_full.values.ravel(), pipe_gb.predict(X_test_full))))
print('Train F1 Score: {:.3f}'.format(f1_score(y_train_full.values.ravel(), pipe_gb.predict(X_train_full), average='micro')))
print('Test F1 Score: {:.3f}'.format(f1_score(y_test_full.values.ravel(), pipe_gb.predict(X_test_full), average='micro')))

# GradientBoost Classifier with Recursive Feature Elimination
print('----------------------------With Recursive Feature Elimination--------------------------------------')
pipe_gb_rfe = Pipeline([('scl', StandardScaler()), ('est', GradientBoostingClassifier(random_state=39))])
pipe_gb_rfe.fit(X_train_rfe, y_train_full.values.ravel())
print('Train Accuracy: {:.3f}'.format(accuracy_score(y_train_full.values.ravel(), pipe_gb_rfe.predict(X_train_rfe))))
print('Test Accuracy: {:.3f}'.format(accuracy_score(y_test_full.values.ravel(), pipe_gb_rfe.predict(X_test_rfe))))
print('Train F1 Score: {:.3f}'.format(f1_score(y_train_full.values.ravel(), pipe_gb_rfe.predict(X_train_rfe), average='micro')))
print('Test F1 Score: {:.3f}'.format(f1_score(y_test_full.values.ravel(), pipe_gb_rfe.predict(X_test_rfe), average='micro')))