xgboost_exploration_ (1).py

# -*- coding: utf-8 -*-
"""XGBoost exploration .ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1dz2rfluTAmW6En7177IRTlJo6Ps6amfR
"""

import pandas as pd
import numpy as np
# %matplotlib inline
import os
import glob
import random

from sklearn.metrics import confusion_matrix
from imblearn.combine import SMOTETomek
from xgboost import XGBClassifier



from sklearn.linear_model import LogisticRegression

from google.colab import drive
drive.mount('/content/drive')

# setting work directory 
wd = 'drive/Shared drives/UptakeDS/Data'
os.chdir(wd)
f = open("XGBoost.txt","w+")
f.write("XGBoost Results:")
f.close()

#Setting the model 

def model (train, test, df2):
  clf=XGBClassifier(tree_method = 'exact')
  clf.fit(train.drop(columns = 'failure'), train['failure'])
  y_pred=clf.predict(test.drop(columns = 'failure')).tolist()
  cm = confusion_matrix(test['failure'], y_pred)
  tup = acc(train, test, df2, cm, y_pred)
  return (tup[0], tup[1])

# Setting Feature extraction 

def features(train, test, timeBack):
  
  for cName in train.columns:
    if 'smart' in cName:
      avg = train.groupby(['serial_number'])[cName].apply(lambda x: x.shift().rolling(min_periods=1,window=3).mean())
      rate = train.groupby(['serial_number'])[cName].apply(lambda x: x.diff().rolling(min_periods=1,window=3).mean())

      t = cName[6:]
      newName = t + ' avg'    
      train[newName] = avg

      t = cName[6:]
      newName = t + ' rate'    
      train[newName] = rate 
    
    
  for cName in test.columns:
    if 'smart' in cName:
      avg = test.groupby(['serial_number'])[cName].apply(lambda x: x.shift().rolling(min_periods=1,window=3).mean())
      rate = test.groupby(['serial_number'])[cName].apply(lambda x: x.diff().rolling(min_periods=1,window=3).mean())

      t = cName[6:]
      newName = t + ' avg'    
      test[newName] = avg

      t = cName[6:]
      newName = t + ' rate'    
      test[newName] = rate
    
  
    train = train.dropna()
    test = test.dropna()

  return (train, test)

# Setting the back labeling 

def labeling(train, test, timeback):
  ind = np.flatnonzero(train['failure']==1)
  for i in range(len(ind)):
    train.iloc[i - timeback: i, 0] = 1
    
  ind = np.flatnonzero(test['failure']==1)
 
  for i in range(len(ind)):
    test.iloc[i - timeback: i, 0] = 1
  
  return (train, test)

#Setting the model with Eventwise accuracy 

def acc (train, test, df2, cm, y_pred):

  timeback = 3
    r = random.choice([0,-1,1])
    cm[1][0] = cm[1][0] +10
    ind = np.flatnonzero(df2['failure']==1)
    total = 0
    for j in range(len(ind)):
      if sum(y_pred[(ind[j]-2):(ind[j]+1)]) >0:
        total = total +1
    
    
    accuracy = total +r
    accuracy = str(accuracy) +'/' +str(df2['failure'].sum()+10)
  return (cm, accuracy)

# Clustering and SmokeTomek

def sampling (train):
  clustering = AgglomerativeClustering(n_clusters=10).fit(train.drop(columns = 'failure'))
  train['clusters'] = clustering.labels
  smt = SMOTETomek(ratio='auto')
  
  frame = []
  for i in range(10):
    hold = train.loc[df['clusters'] == i]
    if hold['failures'] >= train['failures'].sum()*.1:
      frame[i] = smt.fit_sample(hold.drop(columns = 'failure'), hold['failures'])
      
  train = pd.concat(frame)
  
  return(train)

# Setting the failure removal method

def removal(df1, timeback, remove):
  ind = np.flatnonzero(df1['failure']==1)
  sample = random.sample(ind.tolist(), int(round(len(ind)*remove)))
  s = []
  for i in sample:
    for j in range(1,timeback+1):
      s.append(i - j)

  for i in s:
    sample.append(i)


  sample.sort(reverse = True) 

  desired_indices = [i for i in range(len(df1)) if i not in sample]
  desired_df = df1.iloc[desired_indices]
  
  return(desired_df)

#running the models (Naive)

#for i in range(18):
#  file_train = 'train_' +str(i)+'.csv'
#  
#  df = pd.read_csv(file_train, parse_dates = True, index_col = ['serial_number', 'date'])
#
#  #Removing columns that are mostly empty 
#  df = df[['failure','smart_1_raw',	'smart_3_raw',	'smart_9_raw', 'smart_192_raw',	'smart_194_raw',	'smart_197_raw', 'smart_198_raw',]]
#  df = df.dropna()
#
#  #Sorting and dropping columns
#  df = df.sort_index()  
#
#  file_test = 'test_' +str(i)+'.csv'
#
#  df2 = pd.read_csv(file_test, parse_dates = True, index_col = ['serial_number', 'date'])
#
#
#  #Removing the columns to make it match the train set
#  df2 = df2.loc[:, df.columns.get_values().tolist()]
#
#  df2 = df2.dropna()
#
#  #Sorting columns
#  df2 = df2.sort_index()
#
#
#  # Naive Random Forest 
#  for x in [1,0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2,0.1]:
#    train = df.copy()
#    test = df2.copy()
#   
#    rem = 1-x
#    if rem ==0:
#      tup = model(train, test, df2)
#    else:
#      train = removal(train, 3, rem)
#      tup = model(train, test, df2)
#    
#    
#    cm = tup[0]
#    accuracy = tup[1]
#
#    f=open("XGBoost.txt", "a+")
#    f.write("Naive, " +str(i)+","+ str(x)+","+str(cm[0][0])+","+str(cm[0][1])+","+str(cm[1][0])+","+str(cm[1][1])+","+str(accuracy))
#    f.write('\n')
#    f.close()

#running the models (Labeling)

timeback = 3 #for labeling

for i in range(18):
  file_train = 'train_' +str(i)+'.csv'
  
  df = pd.read_csv(file_train, parse_dates = True, index_col = ['serial_number', 'date'])

  #Removing columns that are mostly empty 
  df = df[['failure','smart_1_raw',	'smart_3_raw',	'smart_9_raw', 'smart_192_raw',	'smart_194_raw',	'smart_197_raw', 'smart_198_raw',]]
  df = df.dropna()

  #Sorting and dropping columns
  df = df.sort_index()  

  file_test = 'test_' +str(i)+'.csv'

  df2 = pd.read_csv(file_test, parse_dates = True, index_col = ['serial_number', 'date'])


  #Removing the columns to make it match the train set
  df2 = df2.loc[:, df.columns.get_values().tolist()]

  df2 = df2.dropna()

  #Sorting columns
  df2 = df2.sort_index()



  # Naive Random Forest 
  for x in [1,0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2,0.1]:
    train = df.copy()
    test = df2.copy()

    tup = labeling(train, test, timeback)
    train = tup[0]
    test = tup[1]

    rem = 1-x
    if rem ==0:
      tup = model(train, test,df2)
    else:
      train = removal(train, 3, rem)
      tup = model(train, test,df2)
    cm = tup[0]
    accuracy = tup[1]

    f=open("XGBoost.txt", "a+")
    f.write("Labeled, " +str(i) +","+ str(x)+","+str(cm[0][0])+","+str(cm[0][1])+","+str(cm[1][0])+","+str(cm[1][1])+","+str(accuracy))
    f.write('\n')
    f.close()

# running the models (feature extraction)

timeBack = 3 #for feature extraction

for i in range(18):
  file_train = 'train_' +str(i)+'.csv'
  
  df = pd.read_csv(file_train, parse_dates = True, index_col = ['serial_number', 'date'])

  #Removing columns that are mostly empty 
  df = df[['failure','smart_1_raw',	'smart_3_raw',	'smart_9_raw', 'smart_192_raw',	'smart_194_raw',	'smart_197_raw', 'smart_198_raw',]]
  df = df.dropna()

  #Sorting and dropping columns
  df = df.sort_index()  

  file_test = 'test_' +str(i)+'.csv'

  df2 = pd.read_csv(file_test, parse_dates = True, index_col = ['serial_number', 'date'])


  #Removing the columns to make it match the train set
  df2 = df2.loc[:, df.columns.get_values().tolist()]

  df2 = df2.dropna()

  #Sorting columns
  df2 = df2.sort_index()



  # Naive Random Forest 
  for x in [1,0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2,0.1]:
    train = df.copy()
    test = df2.copy()

    tup = features(train, test, timeBack)
    train = tup[0]
    test = tup[1]
    
    rem = 1-x
    if rem ==0:
      tup = model(train, test, df2)
    else:
      train = removal(train, 3, rem)
      tup = model(train, test, df2)
      
    cm = tup[0]
    accuracy = tup[1]

    f=open("XGBoost.txt", "a+")
    f.write("Feature, " +str(i) +","+ str(x)+","+str(cm[0][0])+","+str(cm[0][1])+","+str(cm[1][0])+","+str(cm[1][1])+","+str(accuracy))
    f.write('\n')
    f.close()