01-RF-Explore-Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "from smart_open import smart_open\n",
    "%matplotlib inline\n",
    "import itertools\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data():\n",
    "    with smart_open('./data/dataset.jsonl.gz') as fh:\n",
    "        dataset = [\n",
    "            json.loads(line.decode('utf-8'))\n",
    "            for line in fh\n",
    "        ]\n",
    "    return pd.DataFrame(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(101781, 9)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71349"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.PRODUCT_ID.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BRAND</th>\n",
       "      <th>NAME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53060</th>\n",
       "      <td>Levis Ambiance</td>\n",
       "      <td>Levis muurverf 'Ambiance' satijn artisjok 5L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65554</th>\n",
       "      <td>None</td>\n",
       "      <td>Alabastine voorstrijk vlekken 2,5L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67703</th>\n",
       "      <td>Decor</td>\n",
       "      <td>Decor split 'carrara' 20 kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93913</th>\n",
       "      <td>None</td>\n",
       "      <td>Briloner verstelbare inbouwspot Attach dimbaar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50837</th>\n",
       "      <td>ATTEMA</td>\n",
       "      <td>ATTEMA Tuit enkel 19 mm voor inbouwdoos U40/U5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25173</th>\n",
       "      <td>Sencys</td>\n",
       "      <td>Sencys schietlood 3 m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8828</th>\n",
       "      <td>Duraline</td>\n",
       "      <td>Duraline wandkubus rond zwart 60x30cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27990</th>\n",
       "      <td>Hamstra</td>\n",
       "      <td>HAMSTRA Horprofiel wit 150 cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100612</th>\n",
       "      <td>Stanley</td>\n",
       "      <td>Stanley Interlock uitschuifmes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48143</th>\n",
       "      <td>Perfax</td>\n",
       "      <td>Perfax behangafsteker</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 BRAND                                               NAME\n",
       "53060   Levis Ambiance       Levis muurverf 'Ambiance' satijn artisjok 5L\n",
       "65554             None                 Alabastine voorstrijk vlekken 2,5L\n",
       "67703            Decor                        Decor split 'carrara' 20 kg\n",
       "93913             None  Briloner verstelbare inbouwspot Attach dimbaar...\n",
       "50837           ATTEMA  ATTEMA Tuit enkel 19 mm voor inbouwdoos U40/U5...\n",
       "25173           Sencys                              Sencys schietlood 3 m\n",
       "8828          Duraline              Duraline wandkubus rond zwart 60x30cm\n",
       "27990          Hamstra                      HAMSTRA Horprofiel wit 150 cm\n",
       "100612         Stanley                     Stanley Interlock uitschuifmes\n",
       "48143           Perfax                              Perfax behangafsteker"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['BRAND','NAME']].sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.SHOP.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BRAND</th>\n",
       "      <th>CONTENTS</th>\n",
       "      <th>DALTIX_ID</th>\n",
       "      <th>DESCRIPTION</th>\n",
       "      <th>DISPLAY_URL</th>\n",
       "      <th>NAME</th>\n",
       "      <th>PRODUCT_ID</th>\n",
       "      <th>SHOP</th>\n",
       "      <th>SPECIFICATIONS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kärcher</td>\n",
       "      <td>[]</td>\n",
       "      <td>43823f3f6826dcfd1f14b5898d742a1ead54937001f980...</td>\n",
       "      <td>De velgenreiniger 'Car &amp; Bike' van Kärcher is ...</td>\n",
       "      <td>https://www.plan-it.be/nl/tuin-buitenleven/aut...</td>\n",
       "      <td>Kärcher velgenreiniger 'Car &amp; Bike' 0,5 L</td>\n",
       "      <td>5305104</td>\n",
       "      <td>planit</td>\n",
       "      <td>{'Inhoud': '\"500.0 ml\"', 'Toepassing': '\"Reini...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Levi's</td>\n",
       "      <td>[]</td>\n",
       "      <td>2258f16fa7ff7aab35c4534ac645017637d072f2fee9cc...</td>\n",
       "      <td>De matte muurverf Originals potloodgrijs van L...</td>\n",
       "      <td>https://www.bol.com/nl/p/muurverf-originals-ma...</td>\n",
       "      <td>Muurverf Originals mat 2,5l potloodgrijs</td>\n",
       "      <td>9200000095428359</td>\n",
       "      <td>bol</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rapid</td>\n",
       "      <td>[]</td>\n",
       "      <td>69ec8d4f776200897422a4b8f93d3416a36781fe110d75...</td>\n",
       "      <td>Vlakdraad nieten. Oorspronkelijk ontworpen voo...</td>\n",
       "      <td>https://www.bol.com/nl/p/rapid-nieten-no-970-8...</td>\n",
       "      <td>Rapid Nieten - No 970 - 8 mm</td>\n",
       "      <td>9200000010411327</td>\n",
       "      <td>bol</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Differnz</td>\n",
       "      <td>[]</td>\n",
       "      <td>f03ef881a1d1934c62b9db6b4403a59ae34de96148f362...</td>\n",
       "      <td>Toiletmeubel Fabulous 40 is een van de meest v...</td>\n",
       "      <td>https://www.bol.com/nl/p/differnz-fabulous-fon...</td>\n",
       "      <td>Differnz Fabulous Fonteinmeubel - 40 cm - Wit</td>\n",
       "      <td>9200000051330319</td>\n",
       "      <td>bol</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pickup</td>\n",
       "      <td>[]</td>\n",
       "      <td>aaff0aa6db24814ad25d5cb410ded08361ab32bcb953e1...</td>\n",
       "      <td>Pickup pictogram is zelfklevend, weerbestendig...</td>\n",
       "      <td>https://www.plan-it.be/nl/verf-laminaat-decora...</td>\n",
       "      <td>Pickup pictogram P621 \"Heren\"</td>\n",
       "      <td>1409779</td>\n",
       "      <td>planit</td>\n",
       "      <td>{'Formaat': '\"10 x 10cm\"', 'Gebruik': '\"Voor b...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      BRAND CONTENTS                                          DALTIX_ID  \\\n",
       "0   Kärcher       []  43823f3f6826dcfd1f14b5898d742a1ead54937001f980...   \n",
       "1    Levi's       []  2258f16fa7ff7aab35c4534ac645017637d072f2fee9cc...   \n",
       "2     Rapid       []  69ec8d4f776200897422a4b8f93d3416a36781fe110d75...   \n",
       "3  Differnz       []  f03ef881a1d1934c62b9db6b4403a59ae34de96148f362...   \n",
       "4    Pickup       []  aaff0aa6db24814ad25d5cb410ded08361ab32bcb953e1...   \n",
       "\n",
       "                                         DESCRIPTION  \\\n",
       "0  De velgenreiniger 'Car & Bike' van Kärcher is ...   \n",
       "1  De matte muurverf Originals potloodgrijs van L...   \n",
       "2  Vlakdraad nieten. Oorspronkelijk ontworpen voo...   \n",
       "3  Toiletmeubel Fabulous 40 is een van de meest v...   \n",
       "4  Pickup pictogram is zelfklevend, weerbestendig...   \n",
       "\n",
       "                                         DISPLAY_URL  \\\n",
       "0  https://www.plan-it.be/nl/tuin-buitenleven/aut...   \n",
       "1  https://www.bol.com/nl/p/muurverf-originals-ma...   \n",
       "2  https://www.bol.com/nl/p/rapid-nieten-no-970-8...   \n",
       "3  https://www.bol.com/nl/p/differnz-fabulous-fon...   \n",
       "4  https://www.plan-it.be/nl/verf-laminaat-decora...   \n",
       "\n",
       "                                            NAME        PRODUCT_ID    SHOP  \\\n",
       "0      Kärcher velgenreiniger 'Car & Bike' 0,5 L           5305104  planit   \n",
       "1       Muurverf Originals mat 2,5l potloodgrijs  9200000095428359     bol   \n",
       "2                   Rapid Nieten - No 970 - 8 mm  9200000010411327     bol   \n",
       "3  Differnz Fabulous Fonteinmeubel - 40 cm - Wit  9200000051330319     bol   \n",
       "4                  Pickup pictogram P621 \"Heren\"           1409779  planit   \n",
       "\n",
       "                                      SPECIFICATIONS  \n",
       "0  {'Inhoud': '\"500.0 ml\"', 'Toepassing': '\"Reini...  \n",
       "1                                               None  \n",
       "2                                               None  \n",
       "3                                               None  \n",
       "4  {'Formaat': '\"10 x 10cm\"', 'Gebruik': '\"Voor b...  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches = [] \n",
    "for name, group in df.groupby('PRODUCT_ID'):\n",
    "    matches += list(itertools.combinations(list(group['DALTIX_ID']), 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index('DALTIX_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['BRAND'] = df['BRAND'].fillna(df['NAME'].str.split(' ').str[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pairs = pd.DataFrame(matches,columns=['daltix_id_1','daltix_id_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pairs['brand_id_1'] = df_pairs.daltix_id_1.apply(lambda x: df.loc[x,'BRAND'])\n",
    "df_pairs['brand_id_2'] = df_pairs.daltix_id_2.apply(lambda x: df.loc[x,'BRAND'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pairs['brand_id_1'] = df_pairs.brand_id_1.str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pairs['brand_id_2'] = df_pairs.brand_id_2.str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pairs['filter_brand'] = (df_pairs.brand_id_1 == df_pairs.brand_id_2) | pd.isnull(df_pairs.brand_id_1) | pd.isnull(df_pairs.brand_id_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_submission = df_pairs[df_pairs.filter_brand == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_submission[['daltix_id_1','daltix_id_2']].to_csv('firstsubmission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     27213\n",
       "False     3238\n",
       "Name: filter_brand, dtype: int64"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pairs['filter_brand'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101781"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5096960             3\n",
       "5021003             3\n",
       "5531371             3\n",
       "6027021             3\n",
       "5230263             3\n",
       "5608565             3\n",
       "5075309             3\n",
       "6336913             3\n",
       "5553352             3\n",
       "5236894             3\n",
       "5070057             3\n",
       "5553354             3\n",
       "5096967             3\n",
       "5139636             3\n",
       "5608507             3\n",
       "5075339             3\n",
       "5553353             3\n",
       "5096977             3\n",
       "5599178             3\n",
       "9200000010648613    2\n",
       "9200000058277974    2\n",
       "3443615             2\n",
       "9200000023061183    2\n",
       "1994786             2\n",
       "5227924             2\n",
       "5541281             2\n",
       "9200000050599785    2\n",
       "9200000010402397    2\n",
       "9200000033415786    2\n",
       "9200000086678074    2\n",
       "                   ..\n",
       "9200000021232782    1\n",
       "861219              1\n",
       "584208              1\n",
       "506368              1\n",
       "875165              1\n",
       "792402              1\n",
       "847285              1\n",
       "5633463             1\n",
       "5024796             1\n",
       "1111                1\n",
       "7117900             1\n",
       "5019532             1\n",
       "2101266             1\n",
       "5647823             1\n",
       "175015              1\n",
       "5018285             1\n",
       "5496472             1\n",
       "577775              1\n",
       "535218              1\n",
       "110432              1\n",
       "6129798             1\n",
       "9200000010415548    1\n",
       "6087277             1\n",
       "4065157             1\n",
       "5603101             1\n",
       "5433645             1\n",
       "565894              1\n",
       "7066433             1\n",
       "5590704             1\n",
       "5470553             1\n",
       "Name: PRODUCT_ID, Length: 71349, dtype: int64"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.PRODUCT_ID.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BRAND</th>\n",
       "      <th>PRODUCT_ID</th>\n",
       "      <th>SHOP</th>\n",
       "      <th>NAME</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DALTIX_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>61d9c04542ac6040979e35b3961d1fb7ff09d2a151e03e514587f90e13bf8180</th>\n",
       "      <td>KOMAR</td>\n",
       "      <td>5531371</td>\n",
       "      <td>hornbach</td>\n",
       "      <td>KOMAR Fotobehang papier Rose Garden 368x254 cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7ba0d49068ace47f2adbcc3749f5634251c7a412078a429b250322490b0d4975</th>\n",
       "      <td>Aurlane</td>\n",
       "      <td>5531371</td>\n",
       "      <td>planit</td>\n",
       "      <td>Aurlane douchecabine 'Pepper Mint' 85 x 85 cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f17d207b54c7decb6af1e774a6d903bd4b6e899e884343052fb1f26874bc50bf</th>\n",
       "      <td>Douchecabine</td>\n",
       "      <td>5531371</td>\n",
       "      <td>praxis</td>\n",
       "      <td>Douchecabine Pepper Mint 230x85x85cm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           BRAND PRODUCT_ID  \\\n",
       "DALTIX_ID                                                                     \n",
       "61d9c04542ac6040979e35b3961d1fb7ff09d2a151e03e5...         KOMAR    5531371   \n",
       "7ba0d49068ace47f2adbcc3749f5634251c7a412078a429...       Aurlane    5531371   \n",
       "f17d207b54c7decb6af1e774a6d903bd4b6e899e8843430...  Douchecabine    5531371   \n",
       "\n",
       "                                                        SHOP  \\\n",
       "DALTIX_ID                                                      \n",
       "61d9c04542ac6040979e35b3961d1fb7ff09d2a151e03e5...  hornbach   \n",
       "7ba0d49068ace47f2adbcc3749f5634251c7a412078a429...    planit   \n",
       "f17d207b54c7decb6af1e774a6d903bd4b6e899e8843430...    praxis   \n",
       "\n",
       "                                                                                              NAME  \n",
       "DALTIX_ID                                                                                           \n",
       "61d9c04542ac6040979e35b3961d1fb7ff09d2a151e03e5...  KOMAR Fotobehang papier Rose Garden 368x254 cm  \n",
       "7ba0d49068ace47f2adbcc3749f5634251c7a412078a429...   Aurlane douchecabine 'Pepper Mint' 85 x 85 cm  \n",
       "f17d207b54c7decb6af1e774a6d903bd4b6e899e8843430...            Douchecabine Pepper Mint 230x85x85cm  "
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['BRAND','PRODUCT_ID','SHOP','NAME']][df.PRODUCT_ID == \"5531371\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME</th>\n",
       "      <th>BRAND</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>39282</th>\n",
       "      <td>HG Schoendeo 250ml</td>\n",
       "      <td>HG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52404</th>\n",
       "      <td>FERM AGM1087 Haakse slijper 850W - 125mm</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39009</th>\n",
       "      <td>GAH Alberts platprofiel staal warmgewalst 1 m ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73743</th>\n",
       "      <td>Sencys universele schroeven 'Torx' staal gegal...</td>\n",
       "      <td>Sencys</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28543</th>\n",
       "      <td>Pickup Plakletter J 90mm zwart mat</td>\n",
       "      <td>Pickup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65483</th>\n",
       "      <td>Domasol Manuele zonneluifel F10 350x300 cm ora...</td>\n",
       "      <td>Domasol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26151</th>\n",
       "      <td>3M SandBlaster schuurpapier op rol korrel 80 g...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1954</th>\n",
       "      <td>Busters veiligheidsbril 'Harrier I' kunststof</td>\n",
       "      <td>Busters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6868</th>\n",
       "      <td>Sencys afdekkap voor schroeven Pozi PZD 2 50 s...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98104</th>\n",
       "      <td>Tiger Melbourne Zeepdispenser - RVS Geborsteld</td>\n",
       "      <td>Tiger</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    NAME    BRAND\n",
       "39282                                 HG Schoendeo 250ml       HG\n",
       "52404           FERM AGM1087 Haakse slijper 850W - 125mm     None\n",
       "39009  GAH Alberts platprofiel staal warmgewalst 1 m ...     None\n",
       "73743  Sencys universele schroeven 'Torx' staal gegal...   Sencys\n",
       "28543                 Pickup Plakletter J 90mm zwart mat   Pickup\n",
       "65483  Domasol Manuele zonneluifel F10 350x300 cm ora...  Domasol\n",
       "26151  3M SandBlaster schuurpapier op rol korrel 80 g...     None\n",
       "1954       Busters veiligheidsbril 'Harrier I' kunststof  Busters\n",
       "6868   Sencys afdekkap voor schroeven Pozi PZD 2 50 s...     None\n",
       "98104     Tiger Melbourne Zeepdispenser - RVS Geborsteld    Tiger"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['NAME','BRAND']].sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['NAME'] = df.NAME.str.lower()\n",
    "df['BRAND'] = df.BRAND.str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['first_word_name']  = df.NAME.str.split(' ').str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2 = df[~pd.isnull(df.BRAND)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81.9750164365549"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(df_2.BRAND == df_2.first_word_name)/len(df_2) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BRAND              25731\n",
       "CONTENTS               0\n",
       "DALTIX_ID              0\n",
       "DESCRIPTION         3659\n",
       "DISPLAY_URL            0\n",
       "NAME                   0\n",
       "PRODUCT_ID             0\n",
       "SHOP                   0\n",
       "SPECIFICATIONS     35060\n",
       "first_word_name        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76050"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.SHOP.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}