youtube_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "id": "b1b2afab-c727-4571-9c75-50d17ae8aa7f",
   "metadata": {},
   "source": [
    "PREPROCESS YOUTUBE DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9c2d6407-a817-403b-be5f-77c9ba82ecd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c8d4cfc7-8331-43cf-bd84-96982991ad31",
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR = 'data/YouTube-Spam-Collection-v1'\n",
    "FILE_NAMES = ['Youtube01-Psy.csv', 'Youtube02-KatyPerry.csv', 'Youtube03-LMFAO.csv', 'Youtube04-Eminem.csv','Youtube04-Eminem.csv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fd0e39d9-66b6-4044-a7c0-2d3de4b55c44",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = []\n",
    "\n",
    "for f in FILE_NAMES:\n",
    "    path = DIR+'/'+f\n",
    "    df = pd.read_csv(path)\n",
    "    new_df = pd.DataFrame(columns=['text','label'])\n",
    "    new_df['text'] = df['CONTENT']\n",
    "    new_df['label'] = df['CLASS']\n",
    "    dfs.append(new_df)\n",
    "\n",
    "df_train = pd.concat(dfs[:4])\n",
    "df_val_test = dfs[4]\n",
    "\n",
    "df_val, df_test = train_test_split(df_val_test,\n",
    "                                     test_size=250, \n",
    "                                     random_state=123, \n",
    "                                     stratify=df_val_test.label)\n",
    "\n",
    "df_dev = df_train.sample(100, random_state=123)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fce1560d-fd9f-4c2a-8eb3-3917b9324ca8",
   "metadata": {},
   "outputs": [],
   "source": [
    "LABELED_PATH = DIR+'/'+'L_data.csv'\n",
    "UNLABELED_PATH = DIR+'/'+'U_data.csv'\n",
    "VAL_PATH = DIR +'/' +'V_data.csv'\n",
    "TEST_PATH = DIR+ '/'+'test_data.csv'\n",
    "\n",
    "df_train.to_csv(UNLABELED_PATH)\n",
    "df_dev.to_csv(LABELED_PATH)\n",
    "df_val.to_csv(VAL_PATH)\n",
    "df_test.to_csv(TEST_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "55925bac-4527-4a7d-835e-cd1b0e8b444f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Huh, anyway check out this you[tube] channel: ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hey guys check out my new channel and our firs...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just for test I have to say murdev.com</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>me shaking my sexy ass on my channel enjoy ^_^ ﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>watch?v=vtaRGgvGtWQ   Check this out .﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>SUBSCRIBE TO MY CHANNEL X PLEASE!. SPARE</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>Check out my videos guy! :) Hope you guys had ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <td>3 yrs ago I had a health scare but thankfully ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>Rihanna looks so beautiful with red hair ;)﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>857.482.940 views AWESOME !!!!!!!!!!!!!!!!!!!!...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1586 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  label\n",
       "0    Huh, anyway check out this you[tube] channel: ...      1\n",
       "1    Hey guys check out my new channel and our firs...      1\n",
       "2               just for test I have to say murdev.com      1\n",
       "3     me shaking my sexy ass on my channel enjoy ^_^ ﻿      1\n",
       "4              watch?v=vtaRGgvGtWQ   Check this out .﻿      1\n",
       "..                                                 ...    ...\n",
       "443           SUBSCRIBE TO MY CHANNEL X PLEASE!. SPARE      1\n",
       "444  Check out my videos guy! :) Hope you guys had ...      1\n",
       "445  3 yrs ago I had a health scare but thankfully ...      1\n",
       "446       Rihanna looks so beautiful with red hair ;)﻿      0\n",
       "447  857.482.940 views AWESOME !!!!!!!!!!!!!!!!!!!!...      0\n",
       "\n",
       "[1586 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "01b4856f-ca85-481c-a3d1-68547e695fc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>Nice﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>Almost a billion﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>http://vimeo.com/106865403﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>I love this song so much﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>Something to dance to, even if your sad JUST ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>Search \"Chubbz Dinero - Ready Or Not \" Thanks ﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Check out this video on YouTube:﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Check out our Channel for nice Beats!!﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>Like this comment, guys i just started up a ne...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>This has had over 2 billion views.  Holy shit.﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  label\n",
       "84                                               Nice﻿      0\n",
       "332                                  Almost a billion﻿      0\n",
       "276                        http://vimeo.com/106865403﻿      1\n",
       "10                           I love this song so much﻿      0\n",
       "343   Something to dance to, even if your sad JUST ...      0\n",
       "..                                                 ...    ...\n",
       "156    Search \"Chubbz Dinero - Ready Or Not \" Thanks ﻿      1\n",
       "237                  Check out this video on YouTube:﻿      1\n",
       "17             Check out our Channel for nice Beats!!﻿      1\n",
       "406  Like this comment, guys i just started up a ne...      1\n",
       "120    This has had over 2 billion views.  Holy shit.﻿      0\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9a8af52b-36b4-4c36-a9d1-77c2ad551aab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>im M.E.S an aspiring young rapper with high ho...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Eminem and Rihanna sing the song very well.﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>Lol thats the guy from animal planet and lost....</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>watch youtube video &amp;quot;EMINEM -YTMA artist ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>You guys should check out this EXTRAORDINARY w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>Share Eminem&amp;#39;s Artist of the Year video so...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>Rihanna looks so beautiful with red hair ;)﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>Hey guys I&amp;#39;m 87 cypher im 11 years old and...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>Lil m !!!!! Check hi out!!!!! Does live the wa...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>Anybody else here in 2015?﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  label\n",
       "327  im M.E.S an aspiring young rapper with high ho...      1\n",
       "54        Eminem and Rihanna sing the song very well.﻿      0\n",
       "299  Lol thats the guy from animal planet and lost....      0\n",
       "156  watch youtube video &quot;EMINEM -YTMA artist ...      1\n",
       "181  You guys should check out this EXTRAORDINARY w...      1\n",
       "..                                                 ...    ...\n",
       "142  Share Eminem&#39;s Artist of the Year video so...      1\n",
       "446       Rihanna looks so beautiful with red hair ;)﻿      0\n",
       "312  Hey guys I&#39;m 87 cypher im 11 years old and...      1\n",
       "382  Lil m !!!!! Check hi out!!!!! Does live the wa...      1\n",
       "341                        Anybody else here in 2015?﻿      0\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "07590549-d564-4d35-a0fa-ff3aabb25cf5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>awesome﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Do you need more instagram followers or photo ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>eminem is a ginius stop!﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>414</th>\n",
       "      <td>Not bad﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>Tell us the title so i can like and subscribe ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>I know that maybe no one will read this but PL...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>2015 and more....﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>Fire..﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>Okay trust me I&amp;#39;m doing a favor. You NEED ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>best song ever (y)﻿</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>198 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  label\n",
       "392                                           awesome﻿      0\n",
       "114  Do you need more instagram followers or photo ...      1\n",
       "97                           eminem is a ginius stop!﻿      0\n",
       "414                                           Not bad﻿      0\n",
       "160  Tell us the title so i can like and subscribe ...      1\n",
       "..                                                 ...    ...\n",
       "205  I know that maybe no one will read this but PL...      1\n",
       "198                                 2015 and more....﻿      0\n",
       "353                                            Fire..﻿      0\n",
       "335  Okay trust me I&#39;m doing a favor. You NEED ...      1\n",
       "199                                best song ever (y)﻿      0\n",
       "\n",
       "[198 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "ce8f7c00-f9d1-482c-b94e-0b5886b1cf93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(ngram_range=(1, 2))"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(ngram_range=(1, 2))\n",
    "\n",
    "all_text = np.append(np.append(np.append(np.array(df_train['text']), np.array(df_dev['text'])),np.array(df_val['text'])),np.array(df_test['text']))\n",
    "vectorizer.fit(all_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "e4f29d67-6fb9-4b90-be4f-57bf9c529ed1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1586, 16634)\n"
     ]
    }
   ],
   "source": [
    "U_feats = vectorizer.transform(np.array(df_train['text']))\n",
    "U_feats_label = np.array(df_train['label'])\n",
    "print(U_feats.shape)\n",
    "\n",
    "with open(DIR+'/'+'U_feats.npy', 'wb') as f:\n",
    "    np.save(f, U_feats.todense())\n",
    "    np.save(f, U_feats_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "ec618546-fb3b-4eb3-a468-626fa994bbde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 16634)\n"
     ]
    }
   ],
   "source": [
    "L_feats = vectorizer.transform(np.array(df_dev['text']))\n",
    "L_feats_label = np.array(df_dev['label'])\n",
    "print(L_feats.shape)\n",
    "\n",
    "with open(DIR+'/'+'L_feats.npy', 'wb') as f:\n",
    "    np.save(f, L_feats.todense())\n",
    "    np.save(f, L_feats_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "26c1cac3-014c-41d3-8824-af5d0e3a2678",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(198, 16634)\n"
     ]
    }
   ],
   "source": [
    "V_feats = vectorizer.transform(np.array(df_val['text']))\n",
    "V_feats_label = np.array(df_val['label'])\n",
    "print(V_feats.shape)\n",
    "\n",
    "with open(DIR+'/'+'V_feats.npy', 'wb') as f:\n",
    "    np.save(f, V_feats.todense())\n",
    "    np.save(f, V_feats_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "05ee3e4b-ddbd-4bb1-bec6-daedd0982980",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(250, 16634)\n"
     ]
    }
   ],
   "source": [
    "test_feats = vectorizer.transform(np.array(df_test['text']))\n",
    "test_feats_label = np.array(df_test['label'])\n",
    "print(test_feats.shape)\n",
    "\n",
    "with open(DIR+'/'+'test_feats.npy', 'wb') as f:\n",
    "    np.save(f, test_feats.todense())\n",
    "    np.save(f, test_feats_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "66338fde-1b81-48e0-bb03-1b2fd7793bfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_prcoessed_files(feats_file, rule_labels_file, processed_file):\n",
    "    with open(feats_file,'rb') as f:\n",
    "        feats = np.load(f,allow_pickle=True)\n",
    "        feats_label = np.load(f,allow_pickle=True)\n",
    "    with open(rule_labels_file,'rb') as f:\n",
    "        rule_labels = np.load(f,allow_pickle=True)\n",
    "\n",
    "\n",
    "    f = open(processed_file,'wb')\n",
    "\n",
    "#     x : feature representation of instances\n",
    "#     shape : [num_instances, num_features]\n",
    "    np.save(f, feats)\n",
    "    \n",
    "#     l : Class Labels assigned by rules\n",
    "#     shape : [num_instances, num_rules]\n",
    "    np.save(f, rule_labels)\n",
    "    \n",
    "#     m : Rule coverage mask\n",
    "#     A binary matrix of shape [num_instances, num_rules]\n",
    "    m = rule_labels>=0 + 0\n",
    "    np.save(f, m)\n",
    "    \n",
    "#     L : Instance labels\n",
    "#     shape : [num_instances, ]\n",
    "    np.save(f, feats_label)\n",
    "    \n",
    "#     r : A binary matrix of shape [num_instances, num_rules]\n",
    "#     r[i][j]=1 if jth rule was associated with ith instance\n",
    "    r = np.zeros(rule_labels.shape)\n",
    "    np.save(f, r)\n",
    "    \n",
    "    \n",
    "    print(\"feats : \", feats.shape)\n",
    "    print(\"rule_labels : \",rule_labels.shape)\n",
    "    print(\"m : \",m.shape)\n",
    "    print(\"feats_label : \",feats_label.shape)\n",
    "    print(\"r : \", r.shape)\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "92e0f4f3-6cb6-4952-8ac1-e72040022c84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feats :  (100, 16634)\n",
      "rule_labels :  (100, 9)\n",
      "m :  (100, 9)\n",
      "feats_label :  (100,)\n",
      "r :  (100, 9)\n"
     ]
    }
   ],
   "source": [
    "feats_file = 'data/YouTube-Spam-Collection-v1/L_feats.npy'\n",
    "rule_labels_file = 'data/YouTube-Spam-Collection-v1/L_L.npy'\n",
    "preprocessed_file = 'data/YouTube-Spam-Collection-v1/L_preprocess.npy'\n",
    "create_prcoessed_files(feats_file, rule_labels_file,preprocessed_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "1eaf01b1-e762-4db8-83e8-ab041b8e71d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feats :  (1586, 16634)\n",
      "rule_labels :  (1586, 9)\n",
      "m :  (1586, 9)\n",
      "feats_label :  (1586,)\n",
      "r :  (1586, 9)\n"
     ]
    }
   ],
   "source": [
    "feats_file = 'data/YouTube-Spam-Collection-v1/U_feats.npy'\n",
    "rule_labels_file = 'data/YouTube-Spam-Collection-v1/U_L.npy'\n",
    "preprocessed_file = 'data/YouTube-Spam-Collection-v1/U_preprocess.npy'\n",
    "create_prcoessed_files(feats_file, rule_labels_file,preprocessed_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "3018eb24-caef-4392-a02d-16ce350a023e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feats :  (198, 16634)\n",
      "rule_labels :  (198, 9)\n",
      "m :  (198, 9)\n",
      "feats_label :  (198,)\n",
      "r :  (198, 9)\n"
     ]
    }
   ],
   "source": [
    "feats_file = 'data/YouTube-Spam-Collection-v1/V_feats.npy'\n",
    "rule_labels_file = 'data/YouTube-Spam-Collection-v1/V_L.npy'\n",
    "preprocessed_file = 'data/YouTube-Spam-Collection-v1/V_preprocess.npy'\n",
    "create_prcoessed_files(feats_file, rule_labels_file,preprocessed_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "276db3de-42d3-4bbd-8d9b-e041605d84ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feats :  (250, 16634)\n",
      "rule_labels :  (250, 9)\n",
      "m :  (250, 9)\n",
      "feats_label :  (250,)\n",
      "r :  (250, 9)\n"
     ]
    }
   ],
   "source": [
    "feats_file = 'data/YouTube-Spam-Collection-v1/test_feats.npy'\n",
    "rule_labels_file = 'data/YouTube-Spam-Collection-v1/test_L.npy'\n",
    "preprocessed_file = 'data/YouTube-Spam-Collection-v1/test_preprocess.npy'\n",
    "create_prcoessed_files(feats_file, rule_labels_file,preprocessed_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "470a244e-196f-41a1-831f-1dfbe0ec9f8d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce6c1519-52a4-4b73-a111-278ccdb2d285",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "aml",
   "language": "python",
   "name": "aml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}