H2o Classification Deep Neural Network

from sklearn import metrics
import numpy as np
from matplotlib import pyplot as plt
%matplotlib inline
import h2o
import seaborn as sns
import time
from hyperopt import fmin, tpe, hp, Trials
import sys

def printf(format, *args):
    sys.stdout.write(format % args)

def Get_Model_Data():
    model = h2o.import_file(path="../../model-8-1.csv")

    mask = list(['BoxRatio','Thrust', 'Velocity', 'OnBalRun', 'vwapGain', 'Altitude'])
    response_column = 'Altitude'

    df_new = model[mask].as_data_frame()
    plt.ioff()
    red_green = ["#ff0000", "#00ff00"]
    sns.set_palette(red_green)
    np.seterr(divide='ignore', invalid='ignore')
    g = sns.pairplot(df_new,
                     diag_kind='kde',
                     hue=response_column,
                     markers=["o", "D"],
                     size=1.5,
                     aspect=1,
                     plot_kws={"s": 10})
    g.fig.subplots_adjust(right=0.9)
    plt.show()

    # Split the data into Train/Validation/Test with Train having 70% and test and validation 15% each
    train_full, valid_full, test_full = model.split_frame(ratios=[.7, .15])

    feature_columns = ['BoxRatio','Thrust', 'Velocity', 'OnBalRun', 'vwapGain']

    train_ = train_full[mask].as_data_frame(use_pandas=True, header=True)
    valid_ = valid_full[mask].as_data_frame(use_pandas=True, header=True)
    test_  = test_full[mask].as_data_frame(use_pandas=True, header=True)
    print('train_: \n', train_.head(5))
    print('valid_: \n', valid_.head(5))
    print('test_: \n',  test_.head(5))

    x_train = train_full[feature_columns]
    y_train = train_full[response_column].asfactor()
    x_valid = valid_full[feature_columns]
    y_valid = valid_full[response_column].asfactor()
    x_test  = test_full[feature_columns]
    y_test  = test_full[response_column].asfactor()

    train = train_full[mask]
    train[response_column] = y_train
    valid = valid_full[mask]
    valid[response_column] = y_valid
    test  = test_full[mask]
    test[response_column] = y_test

    return x_train, y_train, x_valid, y_valid, x_test, y_test, train, valid, test

def ROC_Curve(model, df):
    performance = model.model_performance(df)
    auc = performance.auc()
    false_positive_rate = performance.fprs
    true_positive_rate = performance.tprs

    plt.style.use('ggplot')
    plt.figure()
    plt.plot(false_positive_rate, true_positive_rate, 'k--')
    plt.plot(false_positive_rate,
             true_positive_rate,
             color='darkorange',
             lw = 2,
             label='ROC curve (area = %0.2f)' % auc)
    plt.plot([0,1], [0,1], color = 'navy', lw = 2, linestyle = '--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('ROC curve')
    plt.legend(loc='best')
    plt.show()

def Plot_predictor_importance(model):
    fig, ax = plt.subplots()
    variables = model._model_json['output']['variable_importances']['variable']
    y_pos = np.arange(len(variables))
    scaled_importance = model._model_json['output']['variable_importances']['scaled_importance']
    ax.barh(y_pos, scaled_importance, align='center', color='green', ecolor='black', height=0.5)
    ax.set_yticks(y_pos)
    ax.set_yticklabels(variables)
    ax.invert_yaxis()
    ax.set_xlabel('Scaled Importance')
    ax.set_title('Variable Importance')
    plt.show()

def Print_results(cm, auc, heading):
    print('Confusion matrix:\n', cm)
    true_negative  = cm[0,0]
    true_positive  = cm[1,1]
    false_negative = cm[1,0]
    false_positive = cm[0,1]
    total = true_negative + true_positive + false_negative + false_positive
    misclassification_rate = (false_positive + false_negative)/total
    accuracy  = (true_positive + true_negative)/total
    precision = (true_positive)/(true_positive + false_positive)
    recall    = (true_positive)/(true_positive + false_negative)
    F1        = (2*true_positive)/(2*true_positive + false_positive + false_negative)
    print('')
    print(heading)
    print('misclassification_rate...%7.4f' % misclassification_rate)
    print('accuracy.................%7.4f' % accuracy)
    print('precision................%7.4f' % precision)
    print('recall...................%7.4f' % recall)
    print('F1.......................%7.4f' % F1)
    print('auc......................%7.4f' % auc)

    print('\nComputed from confusion matrix:')
    predicted = best_nn.predict(x_test).as_data_frame(use_pandas=True)
    test_actual = test.as_data_frame(use_pandas=True)['Altitude']
    test_predicted = predicted['predict']
    ntotal = len(test_actual)
    correct = test_actual == predicted['predict']
    numCorrect = sum(correct)
    percent = round(numCorrect/ntotal*100, 6)
    printf("Correct classifications on all data: %d/%d (%f)\n", numCorrect, ntotal, percent)
    accuracy  = metrics.accuracy_score(test_actual, test_predicted)
    precision = metrics.precision_score(test_actual, test_predicted)
    recall    = metrics.recall_score(test_actual, test_predicted)
    F1        = metrics.f1_score(test_actual, test_predicted)
    print('accuracy.................', accuracy)
    print('precision................', precision)
    print('recall...................', recall)
    print('F1.......................', F1)

def Print_Metrics(saved_nn):
    print('\nModel performance on validate and test data set:')
    performance_valid = saved_nn.model_performance(valid)
    # accuracy, precision and F1 produce two numbers, which are the threshold and the value respectively.
    # we index them to extract just the value.
    mse             = performance_valid.mse()
    logloss_valid   = performance_valid.logloss()
    accuracy_valid  = performance_valid.accuracy()[0][1]
    precision_valid = performance_valid.precision()[0][1]
    F1_valid        = performance_valid.F1()[0][1]
    r2_valid        = performance_valid.r2()
    auc_valid       = performance_valid.auc()

    predictions = saved_nn.predict(x_test)
    accuracy = (predictions['predict'] == y_test).as_data_frame(use_pandas=True).mean()
    print('Percent correct predictions on test set (accuracy): ', accuracy[0])

    performance_test = saved_nn.model_performance(test)
    mse            = performance_test.mse()
    logloss_test   = performance_test.logloss()
    accuracy_test  = performance_test.accuracy()[0][1]
    precision_test = performance_test.precision()[0][1]
    F1_test        = performance_test.F1()[0][1]
    auc_test       = performance_test.auc()
    r2_test        = performance_test.r2()

    header = ["Metric", "Validate", "Test"]
    table  = [
               ["logloss",   logloss_valid,   logloss_test],
               ["accuracy",  accuracy_valid,  accuracy_test],
               ["precision", precision_valid, precision_test],
               ["F1",        F1_valid,        F1_test],
               ["r2",        r2_valid,        r2_test],
               ["AUC",       auc_valid,       auc_test]
             ]
    h2o.display.H2ODisplay(table, header)

def Train(x_train, y_train,
          x_valid, y_valid,
          hidden,
          activation,
          dropout):

    nn = h2o.estimators.deeplearning.H2ODeepLearningEstimator(
            training_frame = x_train,
            validation_frame = x_valid,
            response_column = 'Altitude',
            activation = activation,
            distribution = 'auto',
            epochs = 100,
            epsilon = 0.00001,
            force_load_balance = True,
            hidden = hidden,
            loss = 'automatic',
            nfolds = 5,
            overwrite_with_best_model = True,
            # stopping_metric="misclassification",
            stopping_metric="auc",
            standardize = True,
            stopping_rounds = 5,
            max_w2 = 10.0,
            l1 = 0.00001,
            balance_classes = True,
            variable_importances = True
            )

    predictor_labels = list(set(x_train.names)-{"label"})
    response_label = y_train.names[0]
    nn.train(x = predictor_labels,
             y = response_label,
             training_frame = train,
             validation_frame = valid
            )

    mse = nn.mse()
    auc = nn.auc()
    return nn, mse, auc

class switch(object):
    def __init__(self, value):
        self.value = value
        self.fall = False

    def __iter__(self):
        """Return the match method once, then stop"""
        yield self.match
        raise StopIteration

    def match(self, *args):
        """Indicate whether or not to enter a case suite"""
        if self.fall or not args:
            return True
        elif self.value in args: # changed for v1.5, see below
            self.fall = True
            return True
        else:
            return False

def objective(params):
    # print(' ')
    for name, value in params.items():
        for case in switch(name):
            if case('hidden'):
                hid = list(value)
                # print('hidden: ', hid);
                break;
            if case('activation'):
                act = value
                # print('activation: ', act);
                break;

    nn, mse, auc = Train(x_train, y_train,
                    x_valid, y_valid,
                    hidden = hid,
                    activation = act,
                    dropout = 0.2)

    global best_nn, best_mse, best_auc, loop, max_loop
    loop = loop + 1
    if auc > best_auc:
        best_nn = nn
        best_mse = mse
        best_auc = auc
        print(loop, 'hid:',hid,', act:',act,', best auc:', best_auc)
    else:
        if loop%50 == 0:
            print(loop,', auc:', auc)

    return (1.0 - auc)

#h2o.remove_all()
localH2O = h2o.init(ip = "localhost",
                    port = 54321,
                    max_mem_size="24G",
                    nthreads = 3)
h2o.no_progress()

x_train, y_train, x_valid, y_valid, x_test, y_test, train, valid, test = Get_Model_Data()

hidden_list = list([[40],[80],[100],[200],[300],[400],[500],
                   [80,40],[100,50],[200,100],[300,150],[400,200], [500,250],
                   [80,70,60,50],
                   [100,80,60,40],
                   [200,150,100,50],
                   [300,250,200,150],
                   [80,70,60,50,40],
                   [100,80,60,40,20],
                   [200,180,160,140,120],
                   [300,250,200,150,100],
                   [400,350,300,250,200],
                   [500,400,300,200,100],
                   [80,70,60,50,40,30],
                   [100,90,80,70,60,50],
                   [200,180,160,150,140,130],
                   [300,250,200,150,100,50],
                   [400,350,300,250,200,150],
                   [500,400,300,200,100,50]])

activation_list = ['tanh', 'tanh_with_dropout', 'rectifier', 'rectifier_with_dropout', 'maxout']
best_mse = 100000.0
best_auc = 0.0
loop = 0
max_loop = 200
start_time = int(time.time())

space = {
    'hidden': hp.choice('hidden', hidden_list),
    'activation': hp.choice('activation', activation_list)
}

trials = Trials()

best = fmin(objective, space, algo=tpe.suggest, max_evals=max_loop)

best_hidden = hidden_list[best['hidden']]
best_activation = activation_list[best['activation']]

print('\nbest model:',
      '\n hidden........ ', best_hidden,
      '\n activation.... ', best_activation,
      '\n best mse...... ', best_mse,
      '\n type.......... ', best_nn.type)

print('\nModel performance:')
performance = best_nn.model_performance(test_data = test)
print('\nPerformance: ', performance)

ROC_Curve(best_nn, train)

Plot_predictor_importance(best_nn)

test_actual = test.as_data_frame(use_pandas=True)['Altitude']
predicted = best_nn.predict(x_test).as_data_frame(use_pandas=True)
test_predicted = predicted['predict']
cm = metrics.confusion_matrix(test_actual, test_predicted)
performance_train = best_nn.model_performance(train)
auc = best_nn.auc()

Print_results(cm, auc, 'Model performance on training data')

model_path = h2o.save_model(model=best_nn, path="C:/sm/BottleRockets/trained_models/h2o_nn_classify", force=True)

# load the model
saved_nn = h2o.load_model(model_path)

print('\nMetrics summary:\n', saved_nn.cross_validation_metrics_summary())

Print_Metrics(saved_nn)

end_time = int(time.time())
d = divmod(end_time - start_time,86400)  # days
h = divmod(d[1],3600)  # hours
m = divmod(h[1],60)  # minutes
s = m[1]  # seconds
print('%d days, %d hours, %d minutes, %d seconds' % (d[0],h[0],m[0],s))

h2o.cluster().shutdown()