Visualize The Bottle Rocket Dataset

import itertools
import sys

import collections as count
from collections import Counter
import eli5
import lightgbm as lgb
import matplotlib
# import matplotlib.gridspec as gridspec
from matplotlib import gridspec, pyplot as plt
import numpy as np
from numpy import where
import os
import pandas as pd
import pickle
import platform
import seaborn
import seaborn as sns
import shap
import lime
import warnings
from eli5.sklearn import PermutationImportance
from lime.lime_tabular import LimeTabularExplainer
from pylab import rcParams
from scipy import stats
import sklearn
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.model_selection import train_test_split

%matplotlib inline
shap.initjs()
sns.set(style='whitegrid', palette='muted', font_scale=1.1)

print('Operating system version....', platform.platform())
print("Python version is........... %s.%s.%s" % sys.version_info[:3])
print('scikit-learn version is.....', sklearn.__version__)
print('pandas version is...........', pd.__version__)
print('numpy version is............', np.__version__)
print('seaborn version is..........', seaborn.__version__)
print('matplotlib version is.......', matplotlib.__version__)
print('shap version is.............', shap.__version__)

choices = 'Choose: \n'\
    ' 1-rocket-train-classify.csv\n'\
    ' 2-rocket-test-classify.csv\n'\
    ' 3-rocket-train-classify-smote.csv\n'
path = ''
response = input(choices)
if response == '1':
    path = "rocket-train-classify.csv"
if response == '2':
    # print('2 was chosen')
    path = "rocket-test-classify.csv"
if response == '3':
    # print('2 was chosen')
    path = "rocket-train-classify-smote.csv"

if os.path.isfile(path):
    print('found o.k.')
else:
    print('File not found.')

np.random.seed(1)

try:
    file_path = "H:/HedgeTools/Datasets/" + path
    df_orig = pd.read_csv(file_path)
except FileNotFoundError:
    print('file not found. The path is: ', file_path)
    # raise SystemExit("Stop right there!")
    raise ValueError("Re-run the notebook")

response_name = ['Altitude']
feature_names = ['BoxRatio', 'Thrust', 'Acceleration', 'Velocity', 'OnBalRun', 'vwapGain']
mask = feature_names + response_name
df = df_orig[mask]

print('df:\n', df.head(5))
pd.set_option('display.expand_frame_repr', False)
print('\nDescription of df dataset:\n', df[feature_names].describe(include='all'))

X = df[feature_names].values
y = df[response_name].values.ravel()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

print('Shape of datasets:')
print(' train shape... ', X_train.shape, y_train.shape)
print(' test shape.... ', X_test.shape, y_test.shape)

print("Are there missing values in the dataset?")
df.isna().sum()

warnings.filterwarnings('ignore')
plt.figure(figsize=(10, 4))

df_br = df[['BoxRatio']]
v = df_br.BoxRatio.values
plt.subplot(1, 2, 1)
sns.scatterplot(data=v)
plt.subplot(1, 2, 2)
sns.distplot(v, axlabel='BoxRatio')
df1 = pd.DataFrame(data=v, columns=['BoxRatio'])
plt.show()

plt.figure(figsize=(10, 4))
df_th = df[['Thrust']]
v = df_th.Thrust.values
plt.subplot(1, 2, 1)
sns.scatterplot(data=v)
plt.subplot(1, 2, 2)
sns.distplot(v, axlabel='Thrust')
df2 = pd.DataFrame(data=v, columns=['Thrust'])
df3 = pd.concat([df1, df2], axis=1).reindex(df1.index)
plt.show()

plt.figure(figsize=(10,4))
df_ac = df[['Acceleration']]
v = df_ac.Acceleration.values
plt.subplot(1, 2, 1)
sns.scatterplot(data=v)
plt.subplot(1, 2, 2)
sns.distplot(v, axlabel='Acceleration')
df4 = pd.DataFrame(data=v, columns=['Acceleration'])
df5 = pd.concat([df3, df4], axis=1).reindex(df1.index)
plt.show() 

plt.figure(figsize=(10, 4))
df_ve = df[['Velocity']]
v = df_ve.Velocity.values
plt.subplot(1, 2, 1)
sns.scatterplot(data=v)
plt.subplot(1, 2, 2)
sns.distplot(v, axlabel='Velocity')
df6 = pd.DataFrame(data=v, columns=['Velocity'])
df7 = pd.concat([df5, df6], axis=1).reindex(df1.index)
plt.show()

plt.figure(figsize=(10, 4))
df_on = df[['OnBalRun']]
v = df_on.OnBalRun.values
plt.subplot(1, 2, 1)
sns.scatterplot(data=v)
plt.subplot(1, 2, 2)
sns.distplot(v, axlabel='OnBalRun')
df8 = pd.DataFrame(data=v, columns=['OnBalRun'])
df9 = pd.concat([df7, df8], axis=1).reindex(df1.index)
plt.show()

plt.figure(figsize=(10, 4))
df_vw = df[['vwapGain']]
v = df_vw.vwapGain.values
plt.subplot(1, 2, 1)
sns.scatterplot(data=v)
plt.subplot(1, 2, 2)
sns.distplot(v, axlabel='vwapGain')
df10 = pd.DataFrame(data=v, columns=['vwapGain'])
df11 = pd.concat([df9, df10], axis=1).reindex(df1.index)
plt.show()

plt.tight_layout()

df12 = df_orig[['Altitude']]
df_scaled = pd.concat([df11, df12], axis=1).reindex(df1.index)

#print('df_scaled:\n', df_scaled.head(5))
#pd.set_option('display.expand_frame_repr', False)
#print('\nDescription of df_scaled dataset:\n', df_scaled[feature_names].describe(include='all'))

X = df_scaled[feature_names].values
y = df_scaled[response_name].values.ravel()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

count = pd.value_counts(df['Altitude'], sort=True)
print('\ngoal failed: ', count[0], ' goal met: ', count[1])

rcParams['figure.figsize'] = 5, 5
datas = [{'label': '0',   'color': 'r', 'height': count[0]},
         {'label': '1', 'color': 'g', 'height': count[1]}]

plt.figure(figsize=(8, 5))
plt.subplot(121)
i = 0
for data in datas:
    plt.bar(i, data['height'], align='center', color=data['color'], width=0.4)
    i += 1

labels = [data['label'] for data in datas]
pos = [i for i in range(len(datas))]
font_size = 14

plt.xticks(pos, labels, size=font_size)
plt.yticks(size=font_size)
plt.ylabel('Frequency', size=font_size)
plt.title('Profit/Loss Distribution', size=font_size)
plt.rc('legend', **{'fontsize': font_size})
plt.legend(labels)

plt.subplot(122)
counter = Counter(y)
for label, _ in counter.items():
    row_ix = where(y == label)[0]
    if label == 0:
        c = 'red'
    else:
        c = 'green'
    plt.scatter(X[row_ix, 0], X[row_ix, 1], label=str(label), color=c, s=5)
plt.legend()
plt.title('Summary of class distribution')

plt.tight_layout()
plt.show()

# Create correlation matrix
corr_matrix = df_scaled.corr().abs()
d = df_scaled.drop(['Altitude'], axis=1)

df_corr = pd.DataFrame([[(i, j), d.corr().loc[i, j]]
                       for i, j in list(itertools.combinations(d.corr(), 2))],
                       columns=['pairs', 'corr'])
print(df_corr.sort_values(by='corr', ascending=False))

# Select upper triangle of correlation matrix
upper = corr_matrix.where(
    np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))

# Find index of feature columns with correlation greater than 0.95
to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
if not to_drop:
    print('Features to drop: None.\n')
else:
    print('Features to drop: \n', to_drop)

warnings.filterwarnings('ignore')
warnings.simplefilter('ignore')
sns.set(font_scale=1.2)


class SeabornFig2Grid():

    def __init__(self, seaborngrid, fig,  subplot_spec):
        self.fig = fig
        self.sg = seaborngrid
        self.subplot = subplot_spec
        if isinstance(self.sg, sns.axisgrid.FacetGrid) or \
                isinstance(self.sg, sns.axisgrid.PairGrid):
            self._movegrid()
        elif isinstance(self.sg, sns.axisgrid.JointGrid):
            self._movejointgrid()
        self._finalize()

    def _movegrid(self):
        """ Move PairGrid or Facetgrid """
        self._resize()
        n = self.sg.axes.shape[0]
        m = self.sg.axes.shape[1]
        self.subgrid = gridspec.GridSpecFromSubplotSpec(
            n, m, subplot_spec=self.subplot)
        for i in range(n):
            for j in range(m):
                self._moveaxes(self.sg.axes[i, j], self.subgrid[i, j])

    def _movejointgrid(self):
        """ Move Jointgrid """
        h = self.sg.ax_joint.get_position().height
        h2 = self.sg.ax_marg_x.get_position().height
        r = int(np.round(h/h2))
        self._resize()
        self.subgrid = gridspec.GridSpecFromSubplotSpec(
            r+1, r+1, subplot_spec=self.subplot)

        self._moveaxes(self.sg.ax_joint, self.subgrid[1:, :-1])
        self._moveaxes(self.sg.ax_marg_x, self.subgrid[0, :-1])
        self._moveaxes(self.sg.ax_marg_y, self.subgrid[1:, -1])

    def _moveaxes(self, ax, gs):
        ax.remove()
        ax.figure = self.fig
        self.fig.axes.append(ax)
        self.fig.add_axes(ax)
        ax._subplotspec = gs
        ax.set_position(gs.get_position(self.fig))
        ax.set_subplotspec(gs)

    def _finalize(self):
        plt.close(self.sg.fig)
        self.fig.canvas.mpl_connect("resize_event", self._resize)
        self.fig.canvas.draw()

    def _resize(self, evt=None):
        self.sg.fig.set_size_inches(self.fig.get_size_inches())

g0 = sns.jointplot("Acceleration", "Velocity",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g0.annotate(stats.pearsonr)

g1 = sns.jointplot("OnBalRun", "vwapGain",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g1.annotate(stats.pearsonr)

g2 = sns.jointplot("BoxRatio", "Thrust",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g2.annotate(stats.pearsonr)

g3 = sns.jointplot("Acceleration", "OnBalRun",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g3.annotate(stats.pearsonr)

g4 = sns.jointplot("Velocity", "vwapGain",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g4.annotate(stats.pearsonr)

g5 = sns.jointplot("Acceleration", "vwapGain",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g5.annotate(stats.pearsonr)

g6 = sns.jointplot("Velocity", "OnBalRun",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g6.annotate(stats.pearsonr)

g7 = sns.jointplot("BoxRatio", "vwapGain",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g7.annotate(stats.pearsonr)


g8 = sns.jointplot("BoxRatio", "Velocity",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g8.annotate(stats.pearsonr)

g9 = sns.jointplot("Thrust", "vwapGain",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g9.annotate(stats.pearsonr)

g10 = sns.jointplot("BoxRatio", "Acceleration",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g10.annotate(stats.pearsonr)

g11 = sns.jointplot("BoxRatio", "OnBalRun",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g11.annotate(stats.pearsonr)

g12 = sns.jointplot("Thrust", "OnBalRun",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g12.annotate(stats.pearsonr)

g13 = sns.jointplot("Thrust", "Acceleration",
                   data=df,
                   fit_reg=True,
                   kind='reg',
                   height=7,
                   ratio=3,
                   color="b",
                   scatter_kws={"s": 5})
g13.annotate(stats.pearsonr)

fig = plt.figure(figsize=(11, 20))
gs = gridspec.GridSpec(7, 2)

mg0 = SeabornFig2Grid(g0, fig, gs[0])
mg1 = SeabornFig2Grid(g1, fig, gs[1])

mg2 = SeabornFig2Grid(g2, fig, gs[2])
mg3 = SeabornFig2Grid(g3, fig, gs[3])

mg4 = SeabornFig2Grid(g4, fig, gs[4])
mg5 = SeabornFig2Grid(g5, fig, gs[5])

mg6 = SeabornFig2Grid(g6, fig, gs[6])
mg7 = SeabornFig2Grid(g7, fig, gs[7])

mg8 = SeabornFig2Grid(g8, fig, gs[8])
mg9 = SeabornFig2Grid(g9, fig, gs[9])

mg10 = SeabornFig2Grid(g10, fig, gs[10])
mg11 = SeabornFig2Grid(g11, fig, gs[11])

mg12 = SeabornFig2Grid(g12, fig, gs[12])
mg13 = SeabornFig2Grid(g13, fig, gs[13])

gs.tight_layout(fig)

plt.show()

d_train = lgb.Dataset(X_train, label=y_train, feature_name=feature_names)
d_test = lgb.Dataset(X_test, label=y_test, feature_name=feature_names)

params = {
    "max_bin": 512,
    "learning_rate": 0.05,
    "boosting_type": "gbdt",
    "objective": "binary",
    "metric": "binary_logloss",
    "num_leaves": 10,
    "verbose": -1,
    "min_data": 100,
    "boost_from_average": True
}

model = lgb.train(params,
                  d_train,
                  10000,
                  valid_sets=[d_test],
                  early_stopping_rounds=50,
                  verbose_eval=1000)

feature_importances = model.feature_importance()
std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0)
indices = np.argsort(feature_importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for k in range(X_train.shape[1]):
    j = indices[k]
    print("%d: %s (%f)" % (k + 1, feature_names[j], feature_importances[j]))

feature_importance = 100.0 * (feature_importances / feature_importances.max())
sorted_idx = np.argsort(feature_importance)
y_pos = np.arange(sorted_idx.shape[0]) + .5
fig, ax = plt.subplots()
fig.set_size_inches(8, 5)
ax.barh(y_pos,
        feature_importance[sorted_idx],
        align='center',
        color='green',
        ecolor='black',
        height=0.5)
ax.set_yticks(y_pos)
ax.set_yticklabels(feature_names)
ax.invert_yaxis()
ax.set_xlabel('Relative Importance')
ax.set_title('Feature Importance')
plt.show()

perm = PermutationImportance(forest, n_iter=2).fit(X_train, y_train)
eli5.show_weights(perm, feature_names=feature_names)

explainer = LimeTabularExplainer(X_train, mode="classification", feature_names=feature_names)
for index in range(5):
    X_observation = X_train[index, :]
    print('X_observation:', X_observation)
    explanation = explainer.explain_instance(X_observation, forest.predict_proba)
    explanation.show_in_notebook(show_table=False, show_all=True)
    print(explanation.score)

explainer = shap.TreeExplainer(forest)
df_shap = pd.DataFrame(data=X, columns=feature_names)
shap_values = explainer.shap_values(df_shap)

explainer_save = explainer
shap_values_save = shap_values
import pickle
pickle.dump(explainer, open("explainer.pkl", "wb")) 
pickle.dump(shap_values, open("shap_values.pkl", "wb"))

explainer = pickle.load(open("explainer.pkl", "rb"))
shap_values = pickle.load(open("shap_values.pkl", "rb"))

df_shap = pd.DataFrame(data=X_test, columns=feature_names)
shap.summary_plot(shap_values, df_shap, plot_type="bar")

explainer2 = shap.TreeExplainer(model)
X2 = df_orig[feature_names]
shap_values2 = explainer2.shap_values(X2)

shap.force_plot(explainer2.expected_value[1], shap_values2[1][0,:], X2.iloc[1])

shap.force_plot(explainer2.expected_value[1], shap_values2[1][:2,:], X2.iloc[:2,:])

shap.force_plot(explainer2.expected_value[1], shap_values2[1][:100,:], X2.iloc[:100,:])