Etude_sur_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bf0a0f4-7ada-406a-bdf0-6444b9ca86ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "from principal_DNN_MNIST import init_DNN, pretrain_DNN, retropropagation, test_DNN, entree_sortie_reseau\n",
    "from utils import read_mnist, save_object, plot_error_rates, download_data, save_dict_to_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be06a306-6782-44f2-ac77-0549a7e27cd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "download_data()\n",
    "X_train, X_test, y_train, y_test = read_mnist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f58b145-d9f1-448a-b689-7fc47bd88fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=128 \n",
    "n_iter_rbm=100\n",
    "learning_rate=0.01 \n",
    "n_iter_dbn=200\n",
    "verbose=True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "199dd02a-889a-4fee-ae32-3e481a155286",
   "metadata": {},
   "source": [
    "# Number of Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5a8157f-7e3d-4355-8b78-281f687a410d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "layer_configurations = [\n",
    "    (784, 200, 10),\n",
    "    (784, 200, 200, 10),\n",
    "    (784, 200, 200, 200, 10),\n",
    "    (784, 200, 200, 200, 200, 10),\n",
    "    (784, 200, 200, 200, 200, 200, 10)\n",
    "]\n",
    "# Store error rates\n",
    "error_rates_pretrained = {}\n",
    "error_rates_not_pretrained = {}\n",
    "\n",
    "# Store losses\n",
    "losses_finetune_not_pretrained_dict = {}\n",
    "losses_pretrain_dict = {}\n",
    "losses_finetune_pretrained_dict = {}\n",
    "\n",
    "base_dir = \"DNN_Models\"\n",
    "\n",
    "print(\"Starting experiments...\")\n",
    "\n",
    "if not os.path.exists(base_dir):\n",
    "    os.makedirs(base_dir)\n",
    "    print(f\"Created base directory {base_dir} for saving models.\")\n",
    "\n",
    "for index, neurons in enumerate(layer_configurations, start=1):\n",
    "    print(f\"\\nExperiment with Layer Configuration {index}: {neurons}\")\n",
    "    \n",
    "    config_dir = os.path.join(base_dir, '_'.join(map(str, neurons)))\n",
    "    if not os.path.exists(config_dir):\n",
    "        os.makedirs(config_dir)\n",
    "        print(f\"Created configuration directory: {config_dir}\")\n",
    "\n",
    "    error_rates_pretrained[neurons] = {'train': None, 'test': None}\n",
    "    error_rates_not_pretrained[neurons] = {'train': None, 'test': None}\n",
    "\n",
    "    print(\"Initializing DNNs...\")\n",
    "    dnn_pretrained = init_DNN(neurons)\n",
    "    dnn_not_pretrained = init_DNN(neurons)\n",
    "\n",
    "    print(\"Training non pretrained DNNs...\")\n",
    "    dnn_not_pretrained, losses_finetune_not_pretrained = retropropagation(dnn_not_pretrained, X_train, y_train, n_iter_dbn, learning_rate, batch_size)\n",
    "    losses_finetune_not_pretrained_dict[neurons] = losses_finetune_not_pretrained\n",
    "\n",
    "    print(\"Training completed. Evaluating fine-tuned models...\")\n",
    "    error_not_pretrained_train = test_DNN(dnn_not_pretrained, X_train, y_train, verbose=False)\n",
    "    print(f'Error not Pretrained for train: {error_not_pretrained_train}')\n",
    "    error_rates_not_pretrained[neurons]['train'] = error_not_pretrained_train\n",
    "    \n",
    "    error_not_pretrained_test = test_DNN(dnn_not_pretrained, X_test, y_test, verbose=False)\n",
    "    print(f'Error not Pretrained for test: {error_not_pretrained_test}')\n",
    "    error_rates_not_pretrained[neurons]['test'] = error_not_pretrained_test\n",
    "\n",
    "    print(\"Saving fine-tuned models...\")\n",
    "    save_object(dnn_not_pretrained, os.path.join(config_dir, \"dnn_not_pretrained_finetuned.pkl\"))\n",
    "    \n",
    "    \n",
    "    print(\"Pre-training one DNN...\")\n",
    "    dnn_pretrained, losses_pretrain = pretrain_DNN(\n",
    "        dnn_pretrained, \n",
    "        X_train, \n",
    "        epochs=n_iter_rbm, \n",
    "        learning_rate=learning_rate, \n",
    "        batch_size=batch_size, \n",
    "        verbose=verbose\n",
    "    )\n",
    "    losses_pretrain_dict[neurons] = losses_pretrain\n",
    "    print(\"Pre-training completed. Saving pre-trained model and losses...\")\n",
    "    save_object(dnn_pretrained, os.path.join(config_dir, \"dnn_pretrained.pkl\"))\n",
    "    save_object(losses_pretrain, os.path.join(config_dir, \"losses_pretrain.pkl\"))\n",
    "    \n",
    "    print(\"Training pretrained DNNs...\")\n",
    "    dnn_pretrained, losses_finetune_pretrained = retropropagation(dnn_pretrained, X_train, y_train, n_iter_dbn, learning_rate, batch_size)\n",
    "    losses_finetune_pretrained_dict[neurons] = losses_finetune_pretrained\n",
    "\n",
    "    print(\"Training completed. Evaluating fine-tuned models...\")\n",
    "    error_pretrained_train = test_DNN(dnn_pretrained, X_train, y_train, verbose=False)\n",
    "    print(f'Error Pretrained for train: {error_pretrained_train}')\n",
    "    error_rates_pretrained[neurons]['train'] = error_pretrained_train\n",
    "\n",
    "    error_pretrained_test = test_DNN(dnn_pretrained, X_test, y_test, verbose=False)\n",
    "    print(f'Error Pretrained for test: {error_pretrained_test}')\n",
    "    error_rates_pretrained[neurons]['test'] = error_pretrained_test\n",
    "\n",
    "    print(\"Saving fine-tuned models...\")\n",
    "    save_object(dnn_pretrained, os.path.join(config_dir, \"dnn_pretrained_finetuned.pkl\"))\n",
    "\n",
    "    # Convert the tuple keys to strings for JSON compatibility\n",
    "    error_rates_pretrained_json = {str(k): v for k, v in error_rates_pretrained.items()}\n",
    "    error_rates_not_pretrained_json = {str(k): v for k, v in error_rates_not_pretrained.items()}\n",
    "    \n",
    "    # Save the dictionaries as JSON in the specified base directory\n",
    "    save_dict_to_json(error_rates_pretrained_json, os.path.join(base_dir, \"error_rates_pretrained_n_layers.json\"))\n",
    "    save_dict_to_json(error_rates_not_pretrained_json, os.path.join(base_dir, \"error_rates_not_pretrained_n_layers.json\"))\n",
    "    \n",
    "    print(\"Both dictionaries were saved as JSON files in the DNN_Models directory.\")\n",
    "\n",
    "print(\"All experiments completed. Plotting results...\")\n",
    "\n",
    "plot_error_rates(\n",
    "    error_rates_pretrained, \n",
    "    error_rates_not_pretrained, \n",
    "    [1, 2, 3, 4, 5],\n",
    "    \"Number of Layers\", \n",
    "    \"experiment_layers.png\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33a87bf9-4b99-491f-ac7d-7f96486e06c7",
   "metadata": {},
   "source": [
    "# Number of neurons by layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3257c4e-2cf5-4fc7-b6ff-6a0f8acc595b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "layer_configurations = [\n",
    "    (784, 100, 100, 10),\n",
    "    (784, 200, 200, 10),\n",
    "    (784, 300, 300, 10),\n",
    "    (784, 400, 400, 10),\n",
    "    (784, 500, 500, 10),\n",
    "    (784, 600, 600, 10),\n",
    "    (784, 700, 700, 10),\n",
    "]\n",
    "\n",
    "# Store error rates\n",
    "error_rates_pretrained = {}\n",
    "error_rates_not_pretrained = {}\n",
    "\n",
    "# Store losses\n",
    "losses_finetune_not_pretrained_dict = {}\n",
    "losses_pretrain_dict = {}\n",
    "losses_finetune_pretrained_dict = {}\n",
    "\n",
    "base_dir = \"DNN_Models\"\n",
    "\n",
    "print(\"Starting experiments...\")\n",
    "\n",
    "if not os.path.exists(base_dir):\n",
    "    os.makedirs(base_dir)\n",
    "    print(f\"Created base directory {base_dir} for saving models.\")\n",
    "\n",
    "for index, neurons in enumerate(layer_configurations, start=1):\n",
    "    print(f\"\\nExperiment with Layer Configuration {index}: {neurons}\")\n",
    "    \n",
    "    config_dir = os.path.join(base_dir, '_'.join(map(str, neurons)))\n",
    "    if not os.path.exists(config_dir):\n",
    "        os.makedirs(config_dir)\n",
    "        print(f\"Created configuration directory: {config_dir}\")\n",
    "\n",
    "    error_rates_pretrained[neurons] = {'train': None, 'test': None}\n",
    "    error_rates_not_pretrained[neurons] = {'train': None, 'test': None}\n",
    "\n",
    "    print(\"Initializing DNNs...\")\n",
    "    dnn_pretrained = init_DNN(neurons)\n",
    "    dnn_not_pretrained = init_DNN(neurons)\n",
    "\n",
    "\n",
    "    print(\"Training non pretrained DNNs...\")\n",
    "    dnn_not_pretrained, losses_finetune_not_pretrained = retropropagation(dnn_not_pretrained, X_train, y_train, n_iter_dbn, learning_rate, batch_size)\n",
    "    losses_finetune_not_pretrained_dict[neurons] = losses_finetune_not_pretrained\n",
    "\n",
    "    print(\"Training completed. Evaluating fine-tuned models...\")\n",
    "    error_not_pretrained_train = test_DNN(dnn_not_pretrained, X_train, y_train, verbose=False)\n",
    "    print(f'Error not Pretrained for train: {error_not_pretrained_train}')\n",
    "    error_rates_not_pretrained[neurons]['train'] = error_not_pretrained_train\n",
    "    \n",
    "    error_not_pretrained_test = test_DNN(dnn_not_pretrained, X_test, y_test, verbose=False)\n",
    "    print(f'Error not Pretrained for test: {error_not_pretrained_test}')\n",
    "    error_rates_not_pretrained[neurons]['test'] = error_not_pretrained_test\n",
    "\n",
    "    print(\"Saving fine-tuned models...\")\n",
    "    save_object(dnn_not_pretrained, os.path.join(config_dir, \"dnn_not_pretrained_finetuned.pkl\"))\n",
    "    \n",
    "    \n",
    "    print(\"Pre-training one DNN...\")\n",
    "    dnn_pretrained, losses_pretrain = pretrain_DNN(\n",
    "        dnn_pretrained, \n",
    "        X_train, \n",
    "        epochs=n_iter_rbm, \n",
    "        learning_rate=learning_rate, \n",
    "        batch_size=batch_size, \n",
    "        verbose=verbose\n",
    "    )\n",
    "    losses_pretrain_dict[neurons] = losses_pretrain\n",
    "    print(\"Pre-training completed. Saving pre-trained model and losses...\")\n",
    "    save_object(dnn_pretrained, os.path.join(config_dir, \"dnn_pretrained.pkl\"))\n",
    "    save_object(losses_pretrain, os.path.join(config_dir, \"losses_pretrain.pkl\"))\n",
    "    \n",
    "    print(\"Training pretrained DNNs...\")\n",
    "    dnn_pretrained, losses_finetune_pretrained = retropropagation(dnn_pretrained, X_train, y_train, n_iter_dbn, learning_rate, batch_size)\n",
    "    losses_finetune_pretrained_dict[neurons] = losses_finetune_pretrained\n",
    "\n",
    "    print(\"Training completed. Evaluating fine-tuned models...\")\n",
    "    error_pretrained_train = test_DNN(dnn_pretrained, X_train, y_train, verbose=False)\n",
    "    print(f'Error Pretrained for train: {error_pretrained_train}')\n",
    "    error_rates_pretrained[neurons]['train'] = error_pretrained_train\n",
    "\n",
    "    error_pretrained_test = test_DNN(dnn_pretrained, X_test, y_test, verbose=False)\n",
    "    print(f'Error Pretrained for test: {error_pretrained_test}')\n",
    "    error_rates_pretrained[neurons]['test'] = error_pretrained_test\n",
    "\n",
    "    print(\"Saving fine-tuned models...\")\n",
    "    save_object(dnn_pretrained, os.path.join(config_dir, \"dnn_pretrained_finetuned.pkl\"))\n",
    "\n",
    "    # Convert the tuple keys to strings for JSON compatibility\n",
    "    error_rates_pretrained_json = {str(k): v for k, v in error_rates_pretrained.items()}\n",
    "    error_rates_not_pretrained_json = {str(k): v for k, v in error_rates_not_pretrained.items()}\n",
    "    \n",
    "    # Save the dictionaries as JSON in the specified base directory\n",
    "    save_dict_to_json(error_rates_pretrained_json, os.path.join(base_dir, \"error_rates_pretrained_n_neurons.json\"))\n",
    "    save_dict_to_json(error_rates_not_pretrained_json, os.path.join(base_dir, \"error_rates_not_pretrained_n_neurons.json\"))\n",
    "    \n",
    "    print(\"Both dictionaries were saved as JSON files in the DNN_Models directory.\")\n",
    "\n",
    "print(\"All experiments completed. Plotting results...\")\n",
    "\n",
    "plot_error_rates(\n",
    "    error_rates_pretrained, \n",
    "    error_rates_not_pretrained, \n",
    "    [100, 200, 300, 400, 500, 600, 700],\n",
    "    \"Number of neurons in hidden layers\", \n",
    "    \"experiment_n_neurons.png\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ce388de-c1d4-464e-87eb-f5e60b88e8f0",
   "metadata": {},
   "source": [
    "# Number of samples "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee0f45dd-7068-43cf-9370-3759a3c169c7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# The single layer configuration to use for this experiment\n",
    "layer_configuration = (784, 200, 200, 10)\n",
    "\n",
    "# The different sample sizes for the training set\n",
    "sample_sizes = [1000, 3000, 7000, 10000, 30000, 60000]\n",
    "\n",
    "# Store error rates\n",
    "error_rates_pretrained = {}\n",
    "error_rates_not_pretrained = {}\n",
    "\n",
    "# Store losses\n",
    "losses_finetune_not_pretrained_dict = {}\n",
    "losses_pretrain_dict = {}\n",
    "losses_finetune_pretrained_dict = {}\n",
    "\n",
    "base_dir = \"DNN_Models\"\n",
    "\n",
    "if not os.path.exists(base_dir):\n",
    "    os.makedirs(base_dir)\n",
    "    print(f\"Created base directory {base_dir} for saving models.\")\n",
    "\n",
    "print(\"Starting experiments...\")\n",
    "\n",
    "for sample_size in sample_sizes:\n",
    "    print(f\"\\nExperiment with Sample Size: {sample_size}\")\n",
    "\n",
    "    # Shuffle the indices\n",
    "    indices = np.random.permutation(len(X_train))\n",
    "    \n",
    "    # Select a subset of the training data and labels\n",
    "    X_train_sub = X_train[indices[:sample_size]]\n",
    "    y_train_sub = y_train[indices[:sample_size]]\n",
    "\n",
    "    print(\"Subset of training data shape:\", X_train_sub.shape, \"Subset of training labels shape:\", y_train_sub.shape)\n",
    "    \n",
    "    config_dir_name = '_'.join(map(str, layer_configuration)) + f\"_samples_{sample_size}\"\n",
    "    config_dir = os.path.join(base_dir, config_dir_name)\n",
    "    \n",
    "    if not os.path.exists(config_dir):\n",
    "        os.makedirs(config_dir)\n",
    "        print(f\"Created configuration directory: {config_dir}\")\n",
    "    \n",
    "    error_rates_pretrained[sample_size] = {'train': None, 'test': None}\n",
    "    error_rates_not_pretrained[sample_size] = {'train': None, 'test': None}\n",
    "\n",
    "    print(\"Initializing DNNs...\")\n",
    "    dnn_pretrained = init_DNN(layer_configuration)\n",
    "    dnn_not_pretrained = init_DNN(layer_configuration)\n",
    "\n",
    "\n",
    "    print(\"Training non pretrained DNNs...\")\n",
    "    dnn_not_pretrained, losses_finetune_not_pretrained = retropropagation(dnn_not_pretrained, X_train_sub, y_train_sub, n_iter_dbn, learning_rate, batch_size)\n",
    "    losses_finetune_not_pretrained_dict[sample_size] = losses_finetune_not_pretrained\n",
    "\n",
    "    print(\"Training completed. Evaluating fine-tuned models...\")\n",
    "    error_not_pretrained_train = test_DNN(dnn_not_pretrained, X_train_sub, y_train_sub, verbose=False)\n",
    "    print(f'Error not Pretrained for train: {error_not_pretrained_train}')\n",
    "    error_rates_not_pretrained[sample_size]['train'] = error_not_pretrained_train\n",
    "    \n",
    "    error_not_pretrained_test = test_DNN(dnn_not_pretrained, X_test, y_test, verbose=False)\n",
    "    print(f'Error not Pretrained for test: {error_not_pretrained_test}')\n",
    "    error_rates_not_pretrained[sample_size]['test'] = error_not_pretrained_test\n",
    "\n",
    "    print(\"Saving fine-tuned models...\")\n",
    "    save_object(dnn_not_pretrained, os.path.join(config_dir, \"dnn_not_pretrained_finetuned.pkl\"))\n",
    "    \n",
    "    \n",
    "    print(\"Pre-training one DNN...\")\n",
    "    dnn_pretrained, losses_pretrain = pretrain_DNN(\n",
    "        dnn_pretrained, \n",
    "        X_train_sub, \n",
    "        epochs=n_iter_rbm, \n",
    "        learning_rate=learning_rate, \n",
    "        batch_size=batch_size, \n",
    "        verbose=verbose\n",
    "    )\n",
    "    losses_pretrain_dict[sample_size] = losses_pretrain\n",
    "    print(\"Pre-training completed. Saving pre-trained model and losses...\")\n",
    "    save_object(dnn_pretrained, os.path.join(config_dir, \"dnn_pretrained.pkl\"))\n",
    "    save_object(losses_pretrain, os.path.join(config_dir, \"losses_pretrain.pkl\"))\n",
    "    \n",
    "    print(\"Training pretrained DNNs...\")\n",
    "    dnn_pretrained, losses_finetune_pretrained = retropropagation(dnn_pretrained, X_train_sub, y_train_sub, n_iter_dbn, learning_rate, batch_size)\n",
    "    losses_finetune_pretrained_dict[sample_size] = losses_finetune_pretrained\n",
    "\n",
    "    print(\"Training completed. Evaluating fine-tuned models...\")\n",
    "    error_pretrained_train = test_DNN(dnn_pretrained, X_train_sub, y_train_sub, verbose=False)\n",
    "    print(f'Error Pretrained for train: {error_pretrained_train}')\n",
    "    error_rates_pretrained[sample_size]['train'] = error_pretrained_train\n",
    "\n",
    "    error_pretrained_test = test_DNN(dnn_pretrained, X_test, y_test, verbose=False)\n",
    "    print(f'Error Pretrained for test: {error_pretrained_test}')\n",
    "    error_rates_pretrained[sample_size]['test'] = error_pretrained_test\n",
    "\n",
    "    print(\"Saving fine-tuned models...\")\n",
    "    save_object(dnn_pretrained, os.path.join(config_dir, \"dnn_pretrained_finetuned.pkl\"))\n",
    "\n",
    "    # Convert the tuple keys to strings for JSON compatibility\n",
    "    error_rates_pretrained_json = {str(k): v for k, v in error_rates_pretrained.items()}\n",
    "    error_rates_not_pretrained_json = {str(k): v for k, v in error_rates_not_pretrained.items()}\n",
    "    \n",
    "    # Save the dictionaries as JSON in the specified base directory\n",
    "    save_dict_to_json(error_rates_pretrained_json, os.path.join(base_dir, \"error_rates_pretrained_n_samples.json\"))\n",
    "    save_dict_to_json(error_rates_not_pretrained_json, os.path.join(base_dir, \"error_rates_not_pretrained_n_samples.json\"))\n",
    "    \n",
    "    print(\"Both dictionaries were saved as JSON files in the DNN_Models directory.\")\n",
    "\n",
    "print(\"All experiments completed. Plotting results...\")\n",
    "\n",
    "plot_error_rates(\n",
    "    error_rates_pretrained, \n",
    "    error_rates_not_pretrained, \n",
    "    sample_sizes,\n",
    "    \"Number of samples\", \n",
    "    \"experiment_samples.png\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}