decision_trees.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import datasets\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps=np.finfo(float).eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "iris = load_iris(as_frame=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    50\n",
       "1    50\n",
       "2    50\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.584962500721156"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def find_entropy_whole(data):\n",
    "    overall_entropy = 0\n",
    "    target = data.target\n",
    "    values = target.unique()\n",
    "    for value in values:\n",
    "        p=target.value_counts()[value]/len(target)\n",
    "        overall_entropy+=-p*np.log2(p)\n",
    "    return overall_entropy\n",
    "\n",
    "find_entropy_whole(iris)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.5, 3. , 3.2, 3.1, 3.6, 3.9, 3.4, 2.9, 3.7, 4. , 4.4, 3.8, 3.3,\n",
       "       4.1, 4.2, 2.3, 2.8, 2.4, 2.7, 2. , 2.2, 2.5, 2.6])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.data['sepal width (cm)'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_entropy_attribute(data,attribute):\n",
    "    attribute_entropy = 0\n",
    "    target=data.target\n",
    "    val_tar = target.unique()\n",
    "    val_attr = data.data[attribute].unique()\n",
    "    for val_a in val_attr:\n",
    "        overall_entropy = 0\n",
    "        for val_t in val_tar:\n",
    "            num = len(data.data[attribute][data.data[attribute]==val_a][target==val_t])\n",
    "            den=len(data.data[attribute][data.data[attribute]==val_a])\n",
    "            p=num/(den+eps)\n",
    "            overall_entropy+=-p*np.log2(p+eps)\n",
    "        p2=den/len(data.data)\n",
    "        attribute_entropy+=-p2*overall_entropy\n",
    "    return abs(attribute_entropy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sepal length (cm) 0.7080248798300978\n",
      "sepal width (cm) 1.0683196250406573\n",
      "petal length (cm) 0.1386459770753558\n",
      "petal width (cm) 0.14906466204571406\n"
     ]
    }
   ],
   "source": [
    "for i in iris.data.keys():\n",
    "    print(i,find_entropy_attribute(iris,i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'petal length (cm)'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def find_best_attr(data):\n",
    "    IG = []\n",
    "    all = data.data.keys()\n",
    "    for attr in all:\n",
    "        IG.append(find_entropy_whole(data)-find_entropy_attribute(data,attr))\n",
    "    index = np.argmax(IG)\n",
    "    best = all[index]\n",
    "    return best\n",
    "find_best_attr(iris)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# d=data.data\n",
    "# d['target']=data.target\n",
    "# def build_tree(d,tree=None):\n",
    "#     target = d.keys()\n",
    "#     node = find_best_attr(d)\n",
    "#     attValue = np.unique(d[node])\n",
    "#     if tree is None:\n",
    "#         tree = {}\n",
    "#         tree[node] = {}\n",
    "#     for value in attValue:\n",
    "#         subtable = d[d[node]==value].reset_index(drop=True)\n",
    "#         clVal,counts= np.unique(subtable['target'],return_counts=True)\n",
    "#         if len(counts)==1:\n",
    "#             tree[node][value]=clvalue[0]\n",
    "#         else:\n",
    "#             tree[node][value]=build_tree(subtable)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9df1b3d7a578ae7f9767aeeed95c1791e173583b81c588f90ca906e4d0abdec3"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}