-
Notifications
You must be signed in to change notification settings - Fork 0
/
01_preprocessing.py
122 lines (99 loc) · 6.15 KB
/
01_preprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
## CICIDS2017 csv files are required for the operation of the program.
## These files must be located under the "CSVs" folder in the same directory as the program.
## The purpose of this program is to clear the csv files containing CICIDS2017 data from errors.
## the faults observed are:
## 1- 288602 of the entries in the file "Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv" are empty / meaningless.
## (e.g. ",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,")
##
## 2- In the original csv files, while describing the Web Attack types such as Brute Force, XSS, Sql Injection, the character used is not recognized
## by the Python-Pandas library and leads to the error.
## this character ("–", Unicode code:8211) has been changed with another character ("-", Unicode code:45) to correct the error.
##
## After the error correction, all the csv files were made into a single file (all_date.csv) to make it easier to process.
import pandas as pd
import os
from sklearn import preprocessing
import time
seconds = time.time()
print("This process may take 5 to 10 minutes, depending on the performance of your computer.\n\n\n")
number="0123456789"
# CSV files names:
csv_files=["Monday-WorkingHours.pcap_ISCX",
"Tuesday-WorkingHours.pcap_ISCX",
"Wednesday-workingHours.pcap_ISCX",
"Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX",
"Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX",
"Friday-WorkingHours-Morning.pcap_ISCX",
"Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX",
"Friday-WorkingHours-Afternoon-DDos.pcap_ISCX",]
# Headers of column
main_labels=["Flow ID","Source IP","Source Port","Destination IP","Destination Port","Protocol","Timestamp","Flow Duration","Total Fwd Packets",
"Total Backward Packets","Total Length of Fwd Packets","Total Length of Bwd Packets","Fwd Packet Length Max","Fwd Packet Length Min",
"Fwd Packet Length Mean","Fwd Packet Length Std","Bwd Packet Length Max","Bwd Packet Length Min","Bwd Packet Length Mean","Bwd Packet Length Std",
"Flow Bytes/s","Flow Packets/s","Flow IAT Mean","Flow IAT Std","Flow IAT Max","Flow IAT Min","Fwd IAT Total","Fwd IAT Mean","Fwd IAT Std","Fwd IAT Max",
"Fwd IAT Min","Bwd IAT Total","Bwd IAT Mean","Bwd IAT Std","Bwd IAT Max","Bwd IAT Min","Fwd PSH Flags","Bwd PSH Flags","Fwd URG Flags","Bwd URG Flags",
"Fwd Header Length","Bwd Header Length","Fwd Packets/s","Bwd Packets/s","Min Packet Length","Max Packet Length","Packet Length Mean","Packet Length Std",
"Packet Length Variance","FIN Flag Count","SYN Flag Count","RST Flag Count","PSH Flag Count","ACK Flag Count","URG Flag Count","CWE Flag Count",
"ECE Flag Count","Down/Up Ratio","Average Packet Size","Avg Fwd Segment Size","Avg Bwd Segment Size","faulty-Fwd Header Length","Fwd Avg Bytes/Bulk",
"Fwd Avg Packets/Bulk","Fwd Avg Bulk Rate","Bwd Avg Bytes/Bulk","Bwd Avg Packets/Bulk","Bwd Avg Bulk Rate","Subflow Fwd Packets","Subflow Fwd Bytes",
"Subflow Bwd Packets","Subflow Bwd Bytes","Init_Win_bytes_forward","Init_Win_bytes_backward","act_data_pkt_fwd",
"min_seg_size_forward","Active Mean","Active Std","Active Max","Active Min","Idle Mean","Idle Std","Idle Max","Idle Min","Label","External IP"]
main_labels2=main_labels
main_labels=( ",".join( i for i in main_labels ) )
main_labels=main_labels+"\n"
flag=True
for i in range(len(csv_files)):
ths = open(str(i)+".csv", "w")
ths.write(main_labels)
with open("./CSVs/"+csv_files[i]+".csv", "r") as file:
while True:
try:
line=file.readline()
if line[0] in number:# this line eliminates the headers of CSV files and incomplete streams .
if " – " in str(line): ## if there is "–" character ("–", Unicode code:8211) in the flow , it will be chanced with "-" character ( Unicode code:45).
line=(str(line).replace(" – "," - "))
ths.write(str(line))
else:
continue
except:
break
ths.close()
df=pd.read_csv(str(i)+".csv",low_memory=False)
df=df.fillna(0)
string_features=["Flow Bytes/s","Flow Packets/s"]
for ii in string_features: #Some data in the "Flow Bytes / s" and "Flow Packets / s" columns are not numeric. Fixing this bug in this loop
df[ii]=df[ii].replace('Infinity', -1)
df[ii]=df[ii].replace('NaN', 0)
number_or_not=[]
for iii in df[ii]:
try:
k=int(float(iii))
number_or_not.append(int(k))
except:
number_or_not.append(iii)
df[ii]=number_or_not
string_features=[]
for j in main_labels2: # In this section, non-numeric (string and / or categorical) properties (columns) are detected.
if df[j].dtype=="object":
string_features.append(j)
try:
string_features.remove('Label')#The "Label" property was removed from the list. Because it has to remain "categorical" for using with different machine learning approach.
except:
print("error!")
labelencoder_X = preprocessing.LabelEncoder()
for ii in string_features: ## In this loop, non-numeric (string and/or categorical) properties converted to numeric features.
try:
df[ii]=labelencoder_X.fit_transform(df[ii])
except:
df[ii]=df[ii].replace('Infinity', -1)
df=df.drop(main_labels2[61], axis=1) ## Column 61 is deleted because it is unnecessary, column 41 ("Fwd Header Length" feature) had be mistakenly rewritten.
##All CSV files are merged into a single file.
if flag:
df.to_csv('all_data.csv' ,index = False)
flag=False
else:
df.to_csv('all_data.csv' ,index = False,header=False,mode="a")
os.remove(str(i)+".csv")
print("The pre-processing phase of the ",csv_files[i]," file is completed.\n")
print("mission accomplished!")
print("Total operation time: = ",time.time()- seconds ,"seconds")