Tpot Optimized Classification

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
%matplotlib inline
from imblearn.over_sampling import SMOTE
from tabulate import tabulate
import itertools
from copy import copy
from sklearn.externals import joblib

import tpot
from tpot import TPOTClassifier
from tpot.operator_utils import TPOTOperatorClassFactory
from tpot.config.classifier import classifier_config_dict
from tpot.builtins import StackingEstimator, CombineDFs

from tpot import TPOTClassifier, TPOTRegressor
from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code, get_by_name
from tpot.operator_utils import TPOTOperatorClassFactory
from tpot.config.classifier import classifier_config_dict

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Normalizer
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import FunctionTransformer
from sklearn.model_selection import ShuffleSplit
from sklearn.kernel_approximation import Nystroem
from sklearn.model_selection import train_test_split, learning_curve
from sklearn.pipeline import make_pipeline, make_union
from sklearn import metrics
from sklearn.metrics import roc_auc_score, roc_curve, auc, classification_report
from sklearn.metrics import confusion_matrix, accuracy_score, mean_squared_error
from sklearn.metrics import f1_score, precision_score, recall_score

def LoadData():
    global feature_names, response_name, n_features

    model_full = pd.read_csv("H:/ML/BlogExperiments/model-13-1.csv")

    response_name = ['Altitude']
    feature_names = ['BoxRatio', 'Thrust', 'Velocity', 'OnBalRun', 'vwapGain']
    n_features = len(feature_names)
    mask = feature_names + response_name

    model = model_full[mask]
    print('Model dataset:\n', model.head(5))
    # print('\nDescription of model dataset:\n', model[feature_names].describe(include='all'))

    # Correlation_plot(model)
    
    X = model[feature_names].values
    y = model[response_name].values.ravel()
    
    sm = SMOTE(random_state=12)
    X_resampled, y_resampled = sm.fit_sample(X, y)
    
    X_train, X_test, y_train, y_test = train_test_split(X_resampled,
                                                        y_resampled,
                                                        test_size = 0.3,
                                                        random_state = 0)
    print('Size of resampled data:')
    print(' train shape... ', X_train.shape, y_train.shape)
    print(' test shape.... ', X_test.shape, y_test.shape)

    return X_train, y_train, X_test, y_test, X_resampled, y_resampled

def ROC_Curve():
    plt.figure()
    plt.plot([0, 1], [0, 1], 'k--')
    plt.plot(false_positive_rate, true_positive_rate, color='darkorange', label='Random Forest')
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('ROC curve (area = %0.7f)' % auc)
    plt.legend(loc='best')
    plt.show()
    #path = "C:/Users/Charles/Desktop/ROC Curve.png"
    #plt.savefig(path)
    #plt.cla()
    #plt.close('all')

def Plot_predictor_importance(best_model, feature_names):
    feature_importance = best_model.feature_importances_
    feature_importance = 100.0 * (feature_importance / feature_importance.max())
    sorted_idx = np.argsort(feature_importance)
    y_pos = np.arange(sorted_idx.shape[0]) + .5
    fig, ax = plt.subplots()
    ax.barh(y_pos,
            feature_importance[sorted_idx],
            align='center',
            color='green',
            ecolor='black',
            height=0.5)
    ax.set_yticks(y_pos)
    ax.set_yticklabels(feature_names)
    ax.invert_yaxis()
    ax.set_xlabel('Relative Importance')
    ax.set_title('Predictor Importance')
    plt.show()

def Print_model_metrics(exported_pipeline, X_train, X_test, cm_train, cm_test, auc_train, auc_test):
    true_negative_train  = cm_train[0, 0]
    true_positive_train  = cm_train[1, 1]
    false_negative_train = cm_train[1, 0]
    false_positive_train = cm_train[0, 1]
    total_train = true_negative_train + true_positive_train + false_negative_train + false_positive_train
    accuracy_train = (true_positive_train + true_negative_train)/total_train
    precision_train = (true_positive_train)/(true_positive_train + false_positive_train)
    recall_train = (true_positive_train)/(true_positive_train + false_negative_train)
    misclassification_rate_train = (false_positive_train + false_negative_train)/total_train
    F1_train = (2*true_positive_train)/(2*true_positive_train + false_positive_train + false_negative_train)
    
    true_negative_test  = cm_test[0, 0]
    true_positive_test  = cm_test[1, 1]
    false_negative_test = cm_test[1, 0]
    false_positive_test = cm_test[0, 1]
    total_test = true_negative_test + true_positive_test + false_negative_test + false_positive_test
    accuracy_test = (true_positive_test + true_negative_test)/total_test
    precision_test = (true_positive_test)/(true_positive_test + false_positive_test)
    recall_test = (true_positive_test)/(true_positive_test + false_negative_test)
    misclassification_rate_test = (false_positive_test + false_negative_test)/total_test
    F1_test = (2*true_positive_test)/(2*true_positive_test + false_positive_test + false_negative_test)
    
    y_predict_train = exported_pipeline.predict(X_train)
    y_predict_test  = exported_pipeline.predict(X_test)
    mse_train       = metrics.mean_squared_error(y_train, y_predict_train)
    mse_test        = metrics.mean_squared_error(y_test, y_predict_test)
    logloss_train   = metrics.log_loss(y_train, y_predict_train)
    logloss_test    = metrics.log_loss(y_test, y_predict_test)
    accuracy_train  = metrics.accuracy_score(y_train, y_predict_train)
    accuracy_test   = metrics.accuracy_score(y_test, y_predict_test)
    precision_test  = precision_score(y_test, y_predict_test, average='binary')
    recall_test     = recall_score(y_test, y_predict_test, average='binary')
    F1_train        = metrics.f1_score(y_train, y_predict_train)
    F1_test         = metrics.f1_score(y_test, y_predict_test)
    r2_train        = metrics.r2_score(y_train, y_predict_train)
    r2_test         = metrics.r2_score(y_test, y_predict_test)
    auc_train       = metrics.roc_auc_score(y_train, y_predict_train)
    auc_test        = metrics.roc_auc_score(y_test, y_predict_test)
    
    header = ["Metric", "Train", "Test"]
    table = [["accuracy",               accuracy_train,               accuracy_test],
             ["precision",              precision_train,              precision_test],
             ["recall",                 recall_train,                 recall_test],
             ["misclassification rate", misclassification_rate_train, misclassification_rate_test],
             ["F1",                     F1_train,                     F1_test],
             ["r2",                     r2_train,                     r2_test],
             ["AUC",                    auc_train,                    auc_test],
             ["mse",                    mse_train,                    mse_test],
             ["logloss",                logloss_train,                logloss_test]
            ]

    print(tabulate(table, header, tablefmt="fancy_grid"))

def Plot_confusion_matrix(cm, classes, title, normalize=False):
    cmap = plt.cm.Blues
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=0)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

X_train, y_train, X_test, y_test, X, y = LoadData()

exported_pipeline = make_pipeline(
    make_union(
        FunctionTransformer(copy),
        Nystroem(gamma=0.45,
                 kernel="linear",
                 n_components=9)
    ),
    RandomForestClassifier(bootstrap=False,
                           criterion="gini",
                           max_features=0.25,
                           min_samples_leaf=1,
                           min_samples_split=2,
                           n_estimators=100)
)

exported_pipeline.fit(X_train, y_train)
print('\nScore: \n', exported_pipeline.score(X_test, y_test))

best_model = exported_pipeline._final_estimator
print("best model:\n", best_model)

joblib.dump(exported_pipeline, 'C:/sm/trained_models/tpot_model.pkl')

y_predicted_train = exported_pipeline.predict(X_train)
y_predicted_test  = exported_pipeline.predict(X_test)

# Since our target is (0,1), the classifier produces a probability matrix of (N,2).
# The first column refers to the probability that our data belong to class 0 (failure to reach altitude), 
# and the second column refers to the probability that the data belong to class 1 (it's a bottle rocket).
# Therefore, let's take the second column to compute the 'auc' metric.
y_probabilities_test = exported_pipeline.predict_proba(X_test)
y_probabilities_success = y_probabilities_test[:, 1]

false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_probabilities_success)
auc = roc_auc_score(y_train, y_predicted_train)

# reshape so we can append the columns
y_test_ = y_test.reshape(y_test.shape[0], 1)
y_predicted_test_ = y_predicted_test.reshape(y_predicted_test.shape[0], 1)

predicted_name = ['predicted']
probability_names= ['prob 1', 'prob 2']
headers = feature_names + response_name + predicted_name + probability_names
df = pd.DataFrame(columns=headers)
df = pd.DataFrame(X_test, columns=feature_names)
df[response_name] = pd.DataFrame(y_test_).astype(int)
df[predicted_name] = pd.DataFrame(y_predicted_test_).astype(int)
df[probability_names] = pd.DataFrame(y_probabilities_test, columns=probability_names)
print('\ntest_dataset:\n', df.head(), end='')
df.to_csv("test_dataset.csv")

import lime
import lime.lime_tabular

train, test, labels_train, labels_test = train_test_split(X, y, train_size=0.80, test_size=0.20)
explainer = lime.lime_tabular.LimeTabularExplainer(train,
                                                   feature_names=feature_names,
                                                   class_names=response_name,
                                                   discretize_continuous=True)
i = np.random.randint(0, test.shape[0])
exp = explainer.explain_instance(test[i], exported_pipeline.predict_proba, num_features=2, top_labels=1)
exp.show_in_notebook(show_table=True, show_all=False)

i = np.random.randint(0, test.shape[0])
exp = explainer.explain_instance(test[i], exported_pipeline.predict_proba, num_features=2, top_labels=1)
exp.show_in_notebook(show_table=True, show_all=False)

i = np.random.randint(0, test.shape[0])
exp = explainer.explain_instance(test[i], exported_pipeline.predict_proba, num_features=2, top_labels=1)
exp.show_in_notebook(show_table=True, show_all=False)

ROC_Curve()

print('\nFeature Importances:', best_model.feature_importances_)
Plot_predictor_importance(best_model, feature_names)

cm_train = confusion_matrix(y_train, y_predicted_train)
auc_train = roc_auc_score(y_train, y_predicted_train)
    
cm_test = confusion_matrix(y_test, y_predicted_test)
auc_test = roc_auc_score(y_test, y_predicted_test)

class_names = [0, 1]
plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
Plot_confusion_matrix(cm_train, classes=class_names, title='Confusion matrix (Train)')
    
plt.subplot(1, 2, 2)
Plot_confusion_matrix(cm_test, classes=class_names, title='Confusion matrix (Test)')
plt.show()
    
c_report = classification_report(y_test, y_predicted_test)
print('\nClassification report:\n', c_report)

ntotal = len(y_test)
correct = y_test == y_predicted_test
numCorrect = sum(correct)
percent = round( (100.0*numCorrect)/ntotal, 6)
print("\nCorrect classifications on test data: {0:d}/{1:d} {2:8.3f}%".format(numCorrect,
                                                                              ntotal,
                                                                              percent))
prediction_score = 100.0*exported_pipeline.score(X_test, y_test)
assert (round(percent,3) == round(prediction_score, 3)), "prediction scores do not agree"

Print_model_metrics(exported_pipeline, X_train, X_test, cm_train, cm_test, auc_train, auc_test)