C#_automl_classification

#r "nuget:Microsoft.ML"
#r "nuget:Microsoft.ML.AutoML"

using System;
using System.Diagnostics;
using System.Linq;
using System.IO;
using System.Linq;

using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.AutoML;
using Microsoft.ML.Trainers.FastTree;

using XPlot.Plotly;

public class BinaryExperimentProgressHandler:IProgress<RunDetail<BinaryClassificationMetrics>> {
  private int _iterationIndex;
  private double best_f1;

  public void Report(RunDetail<BinaryClassificationMetrics> iterationResult) {
    if (_iterationIndex++ == 0) {
      Console.WriteLine("     Trainer                               Accuracy    AUC       AUPRC   F1-score  Duration");
    }
    var trainerName = iterationResult.TrainerName;
    var accuracy = iterationResult.ValidationMetrics.Accuracy;
    var auc      = iterationResult.ValidationMetrics.AreaUnderRocCurve;
    var aupr     = iterationResult.ValidationMetrics.AreaUnderPrecisionRecallCurve;
    var f1       = iterationResult.ValidationMetrics.F1Score;
    var runtimeInSeconds = iterationResult.RuntimeInSeconds;
    if (f1 > best_f1) {
      best_f1 = f1;
      Console.WriteLine("{0, 4} {1, -35} {2, 9:F4} {3, 9:F4} {4, 9:F4} {5, 9:F4} {6, 9:F4}",
                        _iterationIndex, trainerName, accuracy, auc, aupr, f1, runtimeInSeconds);
    } else {
      Console.Write("{0, 4}\r", _iterationIndex);
    }
  }
}

private static void PrintMetrics(BinaryClassificationMetrics metrics) {
  Console.WriteLine("  Accuracy........................ {0:f6}", metrics.Accuracy);
  Console.WriteLine("  AreaUnderPrecisionRecallCurve... {0:f6}", metrics.AreaUnderPrecisionRecallCurve);
  Console.WriteLine("  AreaUnderRocCurve............... {0:f6}", metrics.AreaUnderRocCurve);
  Console.WriteLine("  F1Score......................... {0:f6}", metrics.F1Score);
  Console.WriteLine("  NegativePrecision............... {0:f6}", metrics.NegativePrecision);
  Console.WriteLine("  NegativeRecall.................. {0:f6}", metrics.NegativeRecall);
  Console.WriteLine("  PositivePrecision............... {0:f6}", metrics.PositivePrecision);
  Console.WriteLine("  PositiveRecall.................. {0:f6}", metrics.PositiveRecall);

  Console.WriteLine("\nConfusion Matrix:\n{0}", metrics.ConfusionMatrix.GetFormattedConfusionTable());
}

public class ModelInput     {
  [ColumnName("BoxRatio"), LoadColumn(0)]
  public float BoxRatio { get; set; }
  [ColumnName("Thrust"), LoadColumn(1)]
  public float Thrust { get; set; }
  [ColumnName("Acceleration"), LoadColumn(2)]
  public float Acceleration { get; set; }
  [ColumnName("Velocity"), LoadColumn(3)]
  public float Velocity { get; set; }
  [ColumnName("OnBalRun"), LoadColumn(4)]
  public float OnBalRun { get; set; }
  [ColumnName("vwapGain"), LoadColumn(5)]
  public float VwapGain { get; set; }
  [ColumnName("Altitude"), LoadColumn(6)]
  public bool Altitude { get; set; }
}

public class ModelOutput     {
  [ColumnName("PredictedLabel")]
  public bool Prediction { get; set; }
  public float Probability { get; set; }
  public float Score { get; set; }
}

var sw = Stopwatch.StartNew();
var mlContext = new MLContext(seed: 1);

Console.WriteLine("\nStep 1: Load the Bottle Rocket dataset.");
var trainDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
  path: @"H:\HedgeTools\Datasets\rocket-train-classify-smote.csv",
  hasHeader: true,
  separatorChar: ',');

var validDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
  path: @"H:\HedgeTools\Datasets\rocket-valid-classify.csv",
  hasHeader: true,
  separatorChar: ',');

var optimizingMetrics = new BinaryClassificationMetric[4];
optimizingMetrics[0] = BinaryClassificationMetric.Accuracy;
optimizingMetrics[1] = BinaryClassificationMetric.F1Score;
optimizingMetrics[2] = BinaryClassificationMetric.AreaUnderRocCurve;
optimizingMetrics[3] = BinaryClassificationMetric.AreaUnderPrecisionRecallCurve;

var trainers = new BinaryClassificationTrainer[2];
trainers[0] = BinaryClassificationTrainer.LightGbm;
trainers[1] = BinaryClassificationTrainer.FastTree;

var bestAccuracy = 0.0;
var bestOptimizingMetric = "";
var bestTrainer = "";
var loop = 0;
ExperimentResult<BinaryClassificationMetrics> bestExperimentResult = null;

foreach (var trainer in trainers) {
  foreach (var optimizingMetric in optimizingMetrics) {
    var settings = new BinaryExperimentSettings {
      MaxExperimentTimeInSeconds = 1 * 30 * 60,
      OptimizingMetric = optimizingMetric,
      CacheDirectory = null
    };
    settings.Trainers.Clear();
    settings.Trainers.Add(trainer);

    Console.WriteLine("\n" + (++loop) + ": Running AutoML binary classification experiment using: " +
                      trainer.ToString() + ", " + optimizingMetric.ToString()
                     );
    var experimentResult =
        mlContext.Auto().CreateBinaryClassificationExperiment(settings)
                        .Execute(trainData: trainDataView,
                                 labelColumnName: "Altitude",
                                 progressHandler: new BinaryExperimentProgressHandler());

    var bestRun = experimentResult.BestRun;

    Console.WriteLine("Total models produced.... {0}", experimentResult.RunDetails.Count());
            var validDataViewWithBestScore = bestRun.Model.Transform(validDataView);
            var validMetrics = mlContext.BinaryClassification.EvaluateNonCalibrated(data: validDataViewWithBestScore,
                                                                                    labelColumnName: "Altitude");
            Console.WriteLine("\nMetrics using validation dataset:");
            PrintMetrics(validMetrics);

    var crossValidationResults =
        mlContext.BinaryClassification.CrossValidateNonCalibrated(validDataView,
                                                                  bestRun.Estimator,
                                                                  numberOfFolds: 6,
                                                                  labelColumnName:
                                                                  "Altitude");
    var metricsInMultipleFolds = crossValidationResults.Select(r => r.Metrics);
    var AccuracyValues = metricsInMultipleFolds.Select(m => m.Accuracy);
    var accuracyValues = AccuracyValues as double[] ?? AccuracyValues.ToArray();
    var AccuracyAverage = accuracyValues.Average();
    double average = accuracyValues.Average();
    double sumOfSquaresOfDifferences = accuracyValues.Select(val => (val - average) * (val - average)).Sum();
    double AccuraciesStdDeviation = Math.Sqrt(sumOfSquaresOfDifferences / (accuracyValues.Length - 1));
    double confidenceInterval95 = 1.96 * AccuraciesStdDeviation / Math.Sqrt((accuracyValues.Length - 1));
    var AccuraciesConfidenceInterval95 = confidenceInterval95;

    if (AccuracyAverage > bestAccuracy) {
      bestAccuracy = AccuracyAverage;
      bestTrainer = bestRun.TrainerName;
      bestOptimizingMetric = optimizingMetric.ToString();
      bestExperimentResult = experimentResult;
      Console.WriteLine("\nBest model's trainer............ {0}", bestTrainer);
      Console.WriteLine("Best model's optimizingMetric... {0}", bestOptimizingMetric);
      Console.WriteLine("Cross Validation Metric: ");
      Console.WriteLine("  Average Accuracy: {0:f4}, Standard deviation: {1:f4}, Confidence Interval 95%: {2:f4}",
        AccuracyAverage, AccuraciesStdDeviation, AccuraciesConfidenceInterval95);

      var mlModel = bestRun.Model;
      mlContext.Model.Save(mlModel, trainDataView.Schema, "./MLModel.zip");
      Console.WriteLine("Done. The model is saved.");
    }
  }
}

var threshold = 0.5;
var modelPath = "./MLModel.zip";
var mlContext = new MLContext();
if (!File.Exists(modelPath)) {
  Console.WriteLine("Cannot find MLModel.zip");
  return;
}
DataViewSchema modelSchema;
var mlModel = mlContext.Model.Load(modelPath, out modelSchema);

var scatters = bestExperimentResult.RunDetails.Where(d => d.ValidationMetrics != null).GroupBy(
    r => r.TrainerName,
    (name, details) => new Graph.Scatter() {
        name = name,
        x = details.Select(r => r.RuntimeInSeconds),
        y = details.Select(r => r.ValidationMetrics.Accuracy),
        mode = "markers",
        marker = new Graph.Marker() { size = 5 }
    }
);

var chart1 = Chart.Plot(scatters);
chart1.WithXTitle("Training Time (in seconds)");
chart1.WithYTitle("Accuracy");
chart1.Width = 500;
chart1.Height = 500;
display(chart1)

var scatters = bestExperimentResult.RunDetails.Where(d => d.ValidationMetrics != null).GroupBy(
    r => r.TrainerName,
    (name, details) => new Graph.Scatter() {
        name = name,
        x = details.Select(r => r.RuntimeInSeconds),
        y = details.Select(r => r.ValidationMetrics.F1Score),
        mode = "markers",
        marker = new Graph.Marker() { size = 5 }
    }
);

var chart2 = Chart.Plot(scatters);
chart2.WithXTitle("Training Time (in seconds)");
chart2.WithYTitle("F1Score");
chart2.Width = 500;
chart2.Height = 500;
display(chart2);

var validDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
  path: @"H:\HedgeTools\Datasets\rocket-valid-classify.csv",
  hasHeader: true,
  separatorChar: ',');

var bestRun = bestExperimentResult.BestRun;
var validDataViewWithBestScore = bestRun.Model.Transform(validDataView);
var validMetrics = mlContext.BinaryClassification.EvaluateNonCalibrated(data: validDataViewWithBestScore,
                                                                        labelColumnName: "Altitude");
Console.WriteLine("\nMetrics of best model on validation data:");
PrintMetrics(validMetrics);

var crossValidationResults = mlContext.BinaryClassification.CrossValidateNonCalibrated(validDataView,
                                                                                       bestRun.Estimator,
                                                                                       numberOfFolds: 6,
                                                                                       labelColumnName: "Altitude");
var metricsInMultipleFolds = crossValidationResults.Select(r => r.Metrics);
var AccuracyValues = metricsInMultipleFolds.Select(m => m.Accuracy);
var accuracyValues = AccuracyValues as double[] ?? AccuracyValues.ToArray();
var AccuracyAverage = accuracyValues.Average();
double average = accuracyValues.Average();
double sumOfSquaresOfDifferences = accuracyValues.Select(val => (val - average) * (val - average)).Sum();
double AccuraciesStdDeviation = Math.Sqrt(sumOfSquaresOfDifferences / (accuracyValues.Length - 1));
double confidenceInterval95 = 1.96 * AccuraciesStdDeviation / Math.Sqrt((accuracyValues.Length - 1));
var AccuraciesConfidenceInterval95 = confidenceInterval95;
Console.WriteLine("Cross Validation Metrics:");
Console.WriteLine("  Average Accuracy: {0:f4}, Standard deviation: {1:f4}, Confidence Interval 95%: {2:f4}",
                  AccuracyAverage, AccuraciesStdDeviation, AccuraciesConfidenceInterval95);

var testDataPath = @"H:\HedgeTools\Datasets\rocket-test-classify.csv";
var testDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
  path: testDataPath,
  hasHeader: true,
  separatorChar: ',');

var testDataViewWithBestScore = bestRun.Model.Transform(testDataView);
var testMetrics = mlContext.BinaryClassification.EvaluateNonCalibrated(data: testDataViewWithBestScore,
                                                                       labelColumnName: "Altitude");
Console.WriteLine("\nMetrics using the test data:");
PrintMetrics(testMetrics);

public void Predict(string predictors, double threshold, out int predicted, out double probability) {
  try {

    var predEngine = mlContext.Model.CreatePredictionEngine<ModelInput, ModelOutput>(mlModel);
    var modelInput = new ModelInput();
    var fields = predictors.Split(',');

    modelInput.BoxRatio     = Convert.ToSingle(fields[0]);
    modelInput.Thrust       = Convert.ToSingle(fields[1]);
    modelInput.Acceleration = Convert.ToSingle(fields[2]);
    modelInput.Velocity     = Convert.ToSingle(fields[3]);
    modelInput.OnBalRun     = Convert.ToSingle(fields[4]);
    modelInput.VwapGain     = Convert.ToSingle(fields[5]);

    var prediction = predEngine.Predict(modelInput);
    predicted = prediction.Prediction ? 1 : 0;
    probability = Math.Round(prediction.Probability, 8);

    if (predicted == 1 && probability < threshold) {
      // Console.WriteLine("Corrected prediction 1 to 0 because probability {0:f8} < {1:f2}",
      //                   probability, threshold);
      predicted = 0;
    }

    if (predicted == 0 && probability >= threshold) {
      Console.WriteLine("We should never come here.");
      predicted = 1;
    }

    return;
  } catch (EndOfStreamException ex) {
    Console.WriteLine("Predict error: " + ex.Message);
  } catch (Exception ex) {
    Console.WriteLine("Predict error: " + ex.Message);
  }
  predicted = 0;
  probability = 0.0;
}

Console.WriteLine("Using dataset:" + testDataPath);
var tableRows = new List<string>();

try {
  var reader = new StreamReader(File.OpenRead(testDataPath));
  reader.ReadLine(); // Ignore the header
  while (!reader.EndOfStream) {
    var line = reader.ReadLine();
    tableRows.Add(line);
  }
  reader.Close();
} catch (Exception ex) {
  Console.WriteLine("Oh drat! We have an error: " + ex.Message);
  return;
}

int false_positive, true_negative, false_negative;
var true_positive = false_positive = true_negative = false_negative = 0;
double goal_met = 0.0, goal_failed = 0.0, probability;
var max_corrected = 0;
var nrows = tableRows.Count;
sw = Stopwatch.StartNew();
var best_threshold = 0.0;
var max_accuracy = 0.0;
Console.WriteLine("\nComputing Metrics using the test dataset, nrows: " + nrows);

for (var threshold = 0.50; threshold < 0.96; threshold += 0.02) {
  var true_positive = 0;
  var false_positive = 0;
  var false_negative = 0;
  var true_negative = 0;
  double goal_met = 0.0, goal_failed = 0.0;
  var corrected = 0;

  foreach (var predictors in tableRows) {
    int altitude, predicted;
    double probability;
    try {
      var fields = predictors.Split(',');
      altitude = Convert.ToInt32(fields[6]);
      // Drop the "Altitude" field, leaving only the "features'.
      var features = string.Join(",", fields.Take(fields.Length - 1));

      // An Altitude of 1 means that the goal wa reached. This is a "ground truth" value.
      // The probability here is simply the probability that a given prediction is True (i.e. 1).
      // If the probability is less than the threshold then it is more likely to be False than True,
      // thus we change the predicion to False (i.e. 0).

      Predict(features, threshold, out predicted, out probability);
    } catch (Exception) {
      Console.WriteLine("dataset is not in the proper format. Did you delete the index column?");
      return;
    }

    if (altitude == 1) {
      goal_met++;
    } else {
      goal_failed++;
    }

    if (predicted == 1 && altitude == 1) {
      // We have found a True Positive. The model predicted a gain and we got it.
      true_positive += 1;
    }

    if (predicted == 1 && altitude == 0) {
      // We have found a False Positive. The model predicted a gain, but we got a loss.
      false_positive += 1;
    }

    if (predicted == 0 && altitude == 1) {
      // We have found a False Negative. The model predicted a loss, but we got a gain.
      false_negative += 1;
    }

    if (predicted == 0 && altitude == 0) {
      // We have found a True Negative. The model predicted a loss, and we got it.
      true_negative += 1;
    }
  }

  var total = Convert.ToDouble(true_positive + false_positive + true_negative + false_negative);
  var accuracy = 100.0 * (true_positive + true_negative) / total;
  accuracy = Math.Round(accuracy, 4);

  Console.WriteLine("accurancy: {0:f4}, max_accuracy: {1:f4}, threshold: {2:f2}",
                    accuracy, max_accuracy, threshold);
  if (accuracy > max_accuracy) {
    max_accuracy = accuracy;

    double recall, f1score;
    var precision = recall = f1score = 0.0;

    var denom = Convert.ToDouble(true_positive + false_negative);
    if (denom > 0.0) recall = true_positive / denom;

    denom = Convert.ToDouble(true_positive + false_positive);
    if (denom > 0.0) precision = true_positive / denom;

    if (precision + recall > 0.0) f1score = 2.0 * (precision * recall) / (precision + recall);
    precision = Math.Round(precision, 4);
    recall = Math.Round(recall, 4);
    f1score = Math.Round(f1score, 4);
    best_threshold = threshold;

    Console.WriteLine("Metrics:");
    Console.WriteLine("  threshold................. {0:f2}", threshold);
    Console.WriteLine("  No. of True Positive...... {0}/{1}", true_positive, goal_met);
    Console.WriteLine("  No. of False Positive..... {0}", false_positive);
    Console.WriteLine("  No. of False Negative..... {0}", false_negative);
    Console.WriteLine("  No. of True Negative...... {0}/{1}", true_negative, goal_failed);
    Console.WriteLine("  Accuracy.................. {0:f4}%", accuracy);
    Console.WriteLine("  Precision................. {0:f4}" + precision);
    Console.WriteLine("  Recall.................... {0:f4}" + recall);
    Console.WriteLine("  F1 Score.................. {0:f4}" + f1score);
  }
}
Console.WriteLine("Done. The best threshold is {0:f2}", best_threshold);