clustering_exps_cifar10_PCA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "clustering-exps-cifar10-PCA.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KMVxeD4vXAZB",
        "outputId": "c5f92ec4-6596-4e0d-a00c-f14d3ce8c25d"
      },
      "source": [
        "#@title Don't forget to upload usps.h5\n",
        "\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "\n",
        "def purity_score(y_true, y_pred): # from https://stackoverflow.com/a/51672699/7947996; in [0,1]; 0-bad,1-good\n",
        "    # compute contingency matrix (also called confusion matrix)\n",
        "    contingency_matrix = metrics.cluster.contingency_matrix(y_true, y_pred)\n",
        "    # return purity\n",
        "    return np.sum(np.amax(contingency_matrix, axis=0)) / np.sum(contingency_matrix) \n",
        "\n",
        "from sklearn.metrics.cluster import adjusted_rand_score # in [0,1]; 0-bad,1-good\n",
        "from sklearn.metrics.cluster import normalized_mutual_info_score # in [0,1]; 0-bad,1-good\n",
        "\n",
        "!pip install coclust\n",
        "from coclust.evaluation.external import accuracy # in [0,1]; 0-bad,1-good\n",
        "\n",
        "def get_data_20news():\n",
        "  import tensorflow as tf\n",
        "  from sklearn.datasets import fetch_20newsgroups\n",
        "  from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "  _20news = fetch_20newsgroups(subset=\"all\")\n",
        "  data = _20news.data\n",
        "  target = _20news.target\n",
        "\n",
        "  vectorizer = TfidfVectorizer(max_features=2000)\n",
        "  data = vectorizer.fit_transform(data)\n",
        "  data = data.toarray()\n",
        "\n",
        "  return data, target\n",
        "\n",
        "\n",
        "def get_data_mnist():\n",
        "  import tensorflow as tf\n",
        "  mnist = tf.keras.datasets.mnist\n",
        "  (x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
        "\n",
        "  x_train = np.concatenate((x_train,x_test))\n",
        "  y_train = np.concatenate((y_train,y_test))\n",
        "\n",
        "  real_labels = y_train\n",
        "\n",
        "  # # indices = np.isin(y_train,range(number_of_dist))\n",
        "  # x_train = x_train[indices]\n",
        "  # y_train = y_train[indices]\n",
        "\n",
        "  samples = (x_train.reshape((x_train.shape[0],-1))/255.).astype(np.float32)\n",
        "  \n",
        "  return samples, real_labels\n",
        "\n",
        "def get_data_cifar10():\n",
        "  import tensorflow as tf\n",
        "  mnist = tf.keras.datasets.cifar10\n",
        "  (x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
        "\n",
        "  x_train = np.concatenate((x_train,x_test))\n",
        "  y_train = np.concatenate((y_train,y_test))\n",
        "  y_train = y_train.squeeze()\n",
        "\n",
        "  real_labels = y_train\n",
        "\n",
        "  # # indices = np.isin(y_train,range(number_of_dist))\n",
        "  # x_train = x_train[indices]\n",
        "  # y_train = y_train[indices]\n",
        "\n",
        "  samples = (x_train.reshape((x_train.shape[0],-1))/255.).astype(np.float32)\n",
        "  \n",
        "  return samples, real_labels\n",
        "\n",
        "def get_data_mnist5():\n",
        "  import tensorflow as tf\n",
        "  mnist = tf.keras.datasets.mnist\n",
        "  (x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
        "\n",
        "  x_train = np.concatenate((x_train,x_test))\n",
        "  y_train = np.concatenate((y_train,y_test))\n",
        "\n",
        "  indices = y_train < 5\n",
        "  x_train = x_train[indices]\n",
        "  y_train = y_train[indices]\n",
        "\n",
        "  real_labels = y_train\n",
        "  \n",
        "  samples = (x_train.reshape((x_train.shape[0],-1))/255.).astype(np.float32)\n",
        "  \n",
        "  return samples, real_labels\n",
        "\n",
        "def get_data_fmnist():\n",
        "  import tensorflow as tf\n",
        "  mnist = tf.keras.datasets.fashion_mnist\n",
        "  (x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
        "\n",
        "  x_train = np.concatenate((x_train,x_test))\n",
        "  y_train = np.concatenate((y_train,y_test))\n",
        "\n",
        "  real_labels = y_train\n",
        "\n",
        "  # # indices = np.isin(y_train,range(number_of_dist))\n",
        "  # x_train = x_train[indices]\n",
        "  # y_train = y_train[indices]\n",
        "\n",
        "  samples = (x_train.reshape((x_train.shape[0],-1))/255.).astype(np.float32)\n",
        "  \n",
        "  return samples, real_labels\n",
        "\n",
        "def get_data_usps():\n",
        "  import h5py\n",
        "  path = \"./usps.h5\"\n",
        "  with h5py.File(path, 'r') as hf:\n",
        "    train = hf.get('train')\n",
        "    X_tr = train.get('data')[:]\n",
        "    y_tr = train.get('target')[:]\n",
        "    test = hf.get('test')\n",
        "    X_te = test.get('data')[:]\n",
        "    y_te = test.get('target')[:]\n",
        "\n",
        "  samples = np.concatenate((X_tr,X_te))\n",
        "  real_labels = np.concatenate((y_tr,y_te))\n",
        "  return samples, real_labels\n",
        "\n",
        "original_data_name = \"cifar10\" # @param [\"mnist\", \"mnist5\", \"cifar10\", \"fmnist\", \"20news\", \"usps\"]\n",
        "\n",
        "if original_data_name == \"mnist\":\n",
        "    samples, real_labels = get_data_mnist()\n",
        "elif original_data_name == \"mnist5\":\n",
        "    samples, real_labels = get_data_mnist5()\n",
        "elif original_data_name == \"cifar10\":\n",
        "    samples, real_labels = get_data_cifar10()\n",
        "elif original_data_name == \"fmnist\":\n",
        "    samples, real_labels = get_data_fmnist()\n",
        "elif original_data_name == \"20news\":\n",
        "    samples, real_labels = get_data_20news()\n",
        "elif original_data_name == \"usps\":\n",
        "    samples, real_labels = get_data_usps()\n",
        "  \n",
        "k = len(np.unique(real_labels))\n",
        "n_init = 10\n",
        "dim_pca = 100\n",
        "\n",
        "if dim_pca is not None:\n",
        "    import numpy as np\n",
        "    from sklearn.decomposition import PCA\n",
        "    X = samples\n",
        "    pca = PCA(n_components=dim_pca)\n",
        "    samples = pca.fit_transform(X)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting coclust\n",
            "  Downloading https://files.pythonhosted.org/packages/5d/44/ad5a69c7187c2b7bcf2c45596e9052811a3be52f4fcaa6709937c5146ee2/coclust-0.2.1.tar.gz\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from coclust) (1.19.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from coclust) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from coclust) (0.22.2.post1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->coclust) (1.0.1)\n",
            "Building wheels for collected packages: coclust\n",
            "  Building wheel for coclust (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for coclust: filename=coclust-0.2.1-cp37-none-any.whl size=29871 sha256=ee54dd41657e88a4238afb42b14985759c1b2d3357acda37816b4c07587e2883\n",
            "  Stored in directory: /root/.cache/pip/wheels/cd/d7/68/df601d0b5f8b934cf890dc626c2271df381fb0c3e910b0a34e\n",
            "Successfully built coclust\n",
            "Installing collected packages: coclust\n",
            "Successfully installed coclust-0.2.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:22: FutureWarning: The linear_assignment_ module is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 11s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UM2n39n8x0fE"
      },
      "source": [
        "### Random"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ExRzemEgvWeb",
        "outputId": "06a88323-385d-4a25-843d-e88ebcdedaeb"
      },
      "source": [
        "predicted_random = np.random.randint(k,size=len(real_labels))\n",
        "\n",
        "print(purity_score(real_labels,predicted_random))\n",
        "print(adjusted_rand_score(real_labels,predicted_random))\n",
        "print(normalized_mutual_info_score(real_labels,predicted_random))\n",
        "print(accuracy(real_labels,predicted_random))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.10578333333333333\n",
            "2.0707135223154302e-05\n",
            "0.0003345272053564096\n",
            "0.1055\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vE1OLiiGx3Tl"
      },
      "source": [
        "### k-means"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fSvnYVhb930",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d46eb5b2-306f-44d8-c6bc-201195770e1a"
      },
      "source": [
        "from sklearn.cluster import KMeans\n",
        "import numpy as np\n",
        "X = samples\n",
        "kmeans = KMeans(n_clusters=k,n_init=n_init).fit(X)\n",
        "predicted_km = kmeans.predict(X)\n",
        "\n",
        "print(purity_score(real_labels,predicted_km))\n",
        "print(adjusted_rand_score(real_labels,predicted_km))\n",
        "print(normalized_mutual_info_score(real_labels,predicted_km))\n",
        "print(accuracy(real_labels,predicted_km))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.22128333333333333\n",
            "0.041738892707942295\n",
            "0.07927056034155963\n",
            "0.20603333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e9XY7cupOKay",
        "outputId": "872e7bcb-8f51-4d97-f27a-d05f20bcbda3"
      },
      "source": [
        "import sklearn.metrics\n",
        "matrix = sklearn.metrics.cluster.contingency_matrix(real_labels, predicted_km)\n",
        "print(matrix)\n",
        "print(matrix/matrix.sum(axis=1, keepdims=True))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 158  913 1177  492  274  651  250  587 1147  351]\n",
            " [ 471  587  180  489  889 1025  592  989  256  522]\n",
            " [ 417  217  631  367 1092  403  713  322  444 1394]\n",
            " [ 910  143  554  459  858  568  829  393  307  979]\n",
            " [1123  206  391  209 1242  312  721  333  139 1324]\n",
            " [1040  113  737  341  678 1012  665  245  196  973]\n",
            " [ 569   53  174  457 1440  431 1282  171  206 1217]\n",
            " [ 882  250  363  675  772  475  374  857  139 1213]\n",
            " [ 255 1894  616  210  191 1048  225 1077  281  203]\n",
            " [ 192  892  159  716  638  512  162 2056  200  473]]\n",
            "[[0.02633333 0.15216667 0.19616667 0.082      0.04566667 0.1085\n",
            "  0.04166667 0.09783333 0.19116667 0.0585    ]\n",
            " [0.0785     0.09783333 0.03       0.0815     0.14816667 0.17083333\n",
            "  0.09866667 0.16483333 0.04266667 0.087     ]\n",
            " [0.0695     0.03616667 0.10516667 0.06116667 0.182      0.06716667\n",
            "  0.11883333 0.05366667 0.074      0.23233333]\n",
            " [0.15166667 0.02383333 0.09233333 0.0765     0.143      0.09466667\n",
            "  0.13816667 0.0655     0.05116667 0.16316667]\n",
            " [0.18716667 0.03433333 0.06516667 0.03483333 0.207      0.052\n",
            "  0.12016667 0.0555     0.02316667 0.22066667]\n",
            " [0.17333333 0.01883333 0.12283333 0.05683333 0.113      0.16866667\n",
            "  0.11083333 0.04083333 0.03266667 0.16216667]\n",
            " [0.09483333 0.00883333 0.029      0.07616667 0.24       0.07183333\n",
            "  0.21366667 0.0285     0.03433333 0.20283333]\n",
            " [0.147      0.04166667 0.0605     0.1125     0.12866667 0.07916667\n",
            "  0.06233333 0.14283333 0.02316667 0.20216667]\n",
            " [0.0425     0.31566667 0.10266667 0.035      0.03183333 0.17466667\n",
            "  0.0375     0.1795     0.04683333 0.03383333]\n",
            " [0.032      0.14866667 0.0265     0.11933333 0.10633333 0.08533333\n",
            "  0.027      0.34266667 0.03333333 0.07883333]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "57lT0F8Mx4Fh"
      },
      "source": [
        "### GMM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xxjlKBYscsE8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9347b584-efba-4770-a97e-fde21c262017"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.mixture import GaussianMixture\n",
        "X = samples\n",
        "gm = GaussianMixture(n_components=k,n_init=n_init).fit(X)\n",
        "predicted_gmm = gm.predict(X)\n",
        "\n",
        "print(purity_score(real_labels,predicted_gmm))\n",
        "print(adjusted_rand_score(real_labels,predicted_gmm))\n",
        "print(normalized_mutual_info_score(real_labels,predicted_gmm))\n",
        "print(accuracy(real_labels,predicted_gmm))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.298\n",
            "0.09402174449013917\n",
            "0.1619164030939813\n",
            "0.28731666666666666\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UOjm81-zyy3q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34accab5-42cb-4129-9dbd-a434155383e5"
      },
      "source": [
        "import sklearn.metrics\n",
        "matrix = sklearn.metrics.cluster.contingency_matrix(real_labels, predicted_gmm)\n",
        "print(matrix)\n",
        "print(matrix/matrix.sum(axis=1, keepdims=True))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 228  835 1144  285  200 1040 1012  387  417  452]\n",
            " [ 397  299   39   82  105  978  459    6 2431 1204]\n",
            " [ 323  299  132 1314 1045  328  301 1409  355  494]\n",
            " [ 387   91    9 1855  921  349  169  312  521 1386]\n",
            " [ 592  323   86  607 1369  395  585 1351  364  328]\n",
            " [ 537   65    6 2385  659  230  100  385  369 1264]\n",
            " [ 148  149   27  542 2726  301   96  806  581  624]\n",
            " [2627   86    8  597  344  432  353  114  477  962]\n",
            " [ 138 1596  800  137  212 1077  873  107  725  335]\n",
            " [ 478  169   44  149  110  950  377    5 2245 1473]]\n",
            "[[0.038      0.13916667 0.19066667 0.0475     0.03333333 0.17333333\n",
            "  0.16866667 0.0645     0.0695     0.07533333]\n",
            " [0.06616667 0.04983333 0.0065     0.01366667 0.0175     0.163\n",
            "  0.0765     0.001      0.40516667 0.20066667]\n",
            " [0.05383333 0.04983333 0.022      0.219      0.17416667 0.05466667\n",
            "  0.05016667 0.23483333 0.05916667 0.08233333]\n",
            " [0.0645     0.01516667 0.0015     0.30916667 0.1535     0.05816667\n",
            "  0.02816667 0.052      0.08683333 0.231     ]\n",
            " [0.09866667 0.05383333 0.01433333 0.10116667 0.22816667 0.06583333\n",
            "  0.0975     0.22516667 0.06066667 0.05466667]\n",
            " [0.0895     0.01083333 0.001      0.3975     0.10983333 0.03833333\n",
            "  0.01666667 0.06416667 0.0615     0.21066667]\n",
            " [0.02466667 0.02483333 0.0045     0.09033333 0.45433333 0.05016667\n",
            "  0.016      0.13433333 0.09683333 0.104     ]\n",
            " [0.43783333 0.01433333 0.00133333 0.0995     0.05733333 0.072\n",
            "  0.05883333 0.019      0.0795     0.16033333]\n",
            " [0.023      0.266      0.13333333 0.02283333 0.03533333 0.1795\n",
            "  0.1455     0.01783333 0.12083333 0.05583333]\n",
            " [0.07966667 0.02816667 0.00733333 0.02483333 0.01833333 0.15833333\n",
            "  0.06283333 0.00083333 0.37416667 0.2455    ]]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}