CS412_Project_Group_GC_34.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ze74oLSOcAGc"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV\n",
        "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.svm import SVC"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# loading data\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gTpIwTqfcE2K",
        "outputId": "6ab3180b-251a-475c-ee57-e6e502c99417"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('/content/drive/My Drive/german_credit_data.csv')\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "_vXASk5rcVeL",
        "outputId": "97da497b-34f3-4ad0-9acb-aa634899bcbc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0  Age     Sex  Job Housing Saving accounts Checking account  \\\n",
              "0           0   67    male    2     own             NaN           little   \n",
              "1           1   22  female    2     own          little         moderate   \n",
              "2           2   49    male    1     own          little              NaN   \n",
              "3           3   45    male    2    free          little           little   \n",
              "4           4   53    male    2    free          little           little   \n",
              "\n",
              "   Credit amount  Duration              Purpose  Risk  \n",
              "0           1169         6             radio/TV  good  \n",
              "1           5951        48             radio/TV   bad  \n",
              "2           2096        12            education  good  \n",
              "3           7882        42  furniture/equipment  good  \n",
              "4           4870        24                  car   bad  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5a11ff8f-6ffc-4f66-8b38-a4815885f476\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Age</th>\n",
              "      <th>Sex</th>\n",
              "      <th>Job</th>\n",
              "      <th>Housing</th>\n",
              "      <th>Saving accounts</th>\n",
              "      <th>Checking account</th>\n",
              "      <th>Credit amount</th>\n",
              "      <th>Duration</th>\n",
              "      <th>Purpose</th>\n",
              "      <th>Risk</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>67</td>\n",
              "      <td>male</td>\n",
              "      <td>2</td>\n",
              "      <td>own</td>\n",
              "      <td>NaN</td>\n",
              "      <td>little</td>\n",
              "      <td>1169</td>\n",
              "      <td>6</td>\n",
              "      <td>radio/TV</td>\n",
              "      <td>good</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>female</td>\n",
              "      <td>2</td>\n",
              "      <td>own</td>\n",
              "      <td>little</td>\n",
              "      <td>moderate</td>\n",
              "      <td>5951</td>\n",
              "      <td>48</td>\n",
              "      <td>radio/TV</td>\n",
              "      <td>bad</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>49</td>\n",
              "      <td>male</td>\n",
              "      <td>1</td>\n",
              "      <td>own</td>\n",
              "      <td>little</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2096</td>\n",
              "      <td>12</td>\n",
              "      <td>education</td>\n",
              "      <td>good</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>45</td>\n",
              "      <td>male</td>\n",
              "      <td>2</td>\n",
              "      <td>free</td>\n",
              "      <td>little</td>\n",
              "      <td>little</td>\n",
              "      <td>7882</td>\n",
              "      <td>42</td>\n",
              "      <td>furniture/equipment</td>\n",
              "      <td>good</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>53</td>\n",
              "      <td>male</td>\n",
              "      <td>2</td>\n",
              "      <td>free</td>\n",
              "      <td>little</td>\n",
              "      <td>little</td>\n",
              "      <td>4870</td>\n",
              "      <td>24</td>\n",
              "      <td>car</td>\n",
              "      <td>bad</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a11ff8f-6ffc-4f66-8b38-a4815885f476')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5a11ff8f-6ffc-4f66-8b38-a4815885f476 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5a11ff8f-6ffc-4f66-8b38-a4815885f476');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Dropping the 'Unnamed: 0' column\n",
        "data = data.drop(columns=['Unnamed: 0'])\n",
        "\n",
        "# Categorical features and numerical features:\n",
        "categorical_features = ['Sex', 'Housing', 'Saving accounts', 'Checking account', 'Purpose']\n",
        "numerical_features = ['Age', 'Job', 'Credit amount', 'Duration']\n",
        "\n",
        "# Preprocessing pipeline:\n",
        "numeric_transformer = Pipeline(steps=[\n",
        "    ('imputer', SimpleImputer(strategy='median')),\n",
        "    ('scaler', StandardScaler())])\n",
        "\n",
        "categorical_transformer = Pipeline(steps=[\n",
        "    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n",
        "    ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
        "\n",
        "preprocessor = ColumnTransformer(\n",
        "    transformers=[\n",
        "        ('num', numeric_transformer, numerical_features),\n",
        "        ('cat', categorical_transformer, categorical_features)])\n",
        "\n",
        "# Defining our models\n",
        "models = [\n",
        "    ('knn', KNeighborsClassifier()),\n",
        "    ('dt', DecisionTreeClassifier()),\n",
        "    ('lr', LogisticRegression(max_iter=5000)), # increase max_iter\n",
        "    ('mlp', MLPClassifier(max_iter=5000, learning_rate_init=0.01)), # increase max_iter and adjust learning_rate_init\n",
        "    ('svm', SVC())\n",
        "]\n",
        "\n",
        "\n",
        "# Splitting our data into training and test sets:\n",
        "X = data.drop('Risk', axis=1)\n",
        "y = data['Risk']\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Training our models with hyperparameter tuning:\n",
        "for name, model in models:\n",
        "    pipe = Pipeline(steps=[('preprocessor', preprocessor),\n",
        "                      ('classifier', model)])\n",
        "    \n",
        "    # Different hyperparameters for different models\n",
        "    if name == 'knn':\n",
        "        param_grid = {\n",
        "            'classifier__n_neighbors': [3, 5, 11],\n",
        "        }\n",
        "    elif name == 'dt':\n",
        "        param_grid = {\n",
        "            'classifier__max_depth': [3, 5, 7],\n",
        "            'classifier__min_samples_split': [2, 3, 4]\n",
        "        }\n",
        "    elif name == 'lr':\n",
        "        param_grid = {\n",
        "            'classifier__C': [0.1, 1.0, 10.0],\n",
        "        }\n",
        "    elif name == 'mlp':\n",
        "        param_grid = {\n",
        "            'classifier__hidden_layer_sizes': [(50,), (100,), (50, 50)],\n",
        "            'classifier__alpha': [0.0001, 0.001, 0.01]\n",
        "        }\n",
        "    elif name == 'svm':\n",
        "        param_grid = {\n",
        "            'classifier__C': [0.1, 1.0, 10.0],\n",
        "            'classifier__gamma': [0.1, 1.0, 10.0]\n",
        "        }\n",
        "\n",
        "    search = GridSearchCV(pipe, param_grid, cv=5)\n",
        "\n",
        "    search.fit(X_train, y_train)\n",
        "\n",
        "    print(f\"For {name}, Best parameter (CV score={search.best_score_}): {search.best_params_}\")\n",
        "    print(f\"Test score: {search.score(X_test, y_test)}\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JmmKKOF_dQNt",
        "outputId": "9b0d10d5-c897-4316-f755-8188f16d9511"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "For knn, Best parameter (CV score=0.73125): {'classifier__n_neighbors': 11}\n",
            "Test score: 0.76\n",
            "\n",
            "For dt, Best parameter (CV score=0.6799999999999999): {'classifier__max_depth': 5, 'classifier__min_samples_split': 4}\n",
            "Test score: 0.71\n",
            "\n",
            "For lr, Best parameter (CV score=0.7375): {'classifier__C': 1.0}\n",
            "Test score: 0.76\n",
            "\n",
            "For mlp, Best parameter (CV score=0.71125): {'classifier__alpha': 0.001, 'classifier__hidden_layer_sizes': (100,)}\n",
            "Test score: 0.69\n",
            "\n",
            "For svm, Best parameter (CV score=0.74): {'classifier__C': 1.0, 'classifier__gamma': 0.1}\n",
            "Test score: 0.76\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
        "\n",
        "# Best parameters from GridSearchCV\n",
        "best_params = {\n",
        "    'knn': {'n_neighbors': 11},\n",
        "    'dt': {'max_depth': 5, 'min_samples_split': 3},\n",
        "    'lr': {'C': 1.0},\n",
        "    'mlp': {'hidden_layer_sizes': (100,), 'alpha': 0.01},\n",
        "    'svm': {'C': 1.0, 'gamma': 0.1},\n",
        "}\n",
        "\n",
        "# Models with best parameters\n",
        "models_best_params = [\n",
        "    ('knn', KNeighborsClassifier(n_neighbors=best_params['knn']['n_neighbors'])),\n",
        "    ('dt', DecisionTreeClassifier(max_depth=best_params['dt']['max_depth'], min_samples_split=best_params['dt']['min_samples_split'])),\n",
        "    ('lr', LogisticRegression(C=best_params['lr']['C'], max_iter=5000)),\n",
        "    ('mlp', MLPClassifier(hidden_layer_sizes=best_params['mlp']['hidden_layer_sizes'], alpha=best_params['mlp']['alpha'], max_iter=5000, learning_rate_init=0.01)),\n",
        "    ('svm', SVC(C=best_params['svm']['C'], gamma=best_params['svm']['gamma']))\n",
        "]\n",
        "\n",
        "# Fit models with best parameters\n",
        "for name, model in models_best_params:\n",
        "    pipe_best = Pipeline(steps=[('preprocessor', preprocessor),\n",
        "                                ('classifier', model)])\n",
        "    pipe_best.fit(X_train, y_train)\n",
        "\n",
        "    y_train_pred = pipe_best.predict(X_train)\n",
        "    y_test_pred = pipe_best.predict(X_test)\n",
        "\n",
        "    print(f\"{name} Model:\")\n",
        "    print(\"Training Accuracy:\", accuracy_score(y_train, y_train_pred))\n",
        "    print(\"Test Accuracy:\", accuracy_score(y_test, y_test_pred))\n",
        "    print(\"\\nClassification Report (Test Data):\")\n",
        "    print(classification_report(y_test, y_test_pred))\n",
        "    print(\"\\nConfusion Matrix (Test Data):\")\n",
        "    print(confusion_matrix(y_test, y_test_pred))\n",
        "    print(\"\\n===========================\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5b81V9eesioj",
        "outputId": "f3cdd8fc-1b6e-45f2-c98c-a2a158e8a8f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "knn Model:\n",
            "Training Accuracy: 0.76375\n",
            "Test Accuracy: 0.76\n",
            "\n",
            "Classification Report (Test Data):\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         bad       0.69      0.34      0.45        59\n",
            "        good       0.77      0.94      0.85       141\n",
            "\n",
            "    accuracy                           0.76       200\n",
            "   macro avg       0.73      0.64      0.65       200\n",
            "weighted avg       0.75      0.76      0.73       200\n",
            "\n",
            "\n",
            "Confusion Matrix (Test Data):\n",
            "[[ 20  39]\n",
            " [  9 132]]\n",
            "\n",
            "===========================\n",
            "\n",
            "dt Model:\n",
            "Training Accuracy: 0.78\n",
            "Test Accuracy: 0.71\n",
            "\n",
            "Classification Report (Test Data):\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         bad       0.51      0.31      0.38        59\n",
            "        good       0.75      0.88      0.81       141\n",
            "\n",
            "    accuracy                           0.71       200\n",
            "   macro avg       0.63      0.59      0.60       200\n",
            "weighted avg       0.68      0.71      0.68       200\n",
            "\n",
            "\n",
            "Confusion Matrix (Test Data):\n",
            "[[ 18  41]\n",
            " [ 17 124]]\n",
            "\n",
            "===========================\n",
            "\n",
            "lr Model:\n",
            "Training Accuracy: 0.74875\n",
            "Test Accuracy: 0.76\n",
            "\n",
            "Classification Report (Test Data):\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         bad       0.65      0.41      0.50        59\n",
            "        good       0.79      0.91      0.84       141\n",
            "\n",
            "    accuracy                           0.76       200\n",
            "   macro avg       0.72      0.66      0.67       200\n",
            "weighted avg       0.74      0.76      0.74       200\n",
            "\n",
            "\n",
            "Confusion Matrix (Test Data):\n",
            "[[ 24  35]\n",
            " [ 13 128]]\n",
            "\n",
            "===========================\n",
            "\n",
            "mlp Model:\n",
            "Training Accuracy: 0.99375\n",
            "Test Accuracy: 0.685\n",
            "\n",
            "Classification Report (Test Data):\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         bad       0.47      0.53      0.50        59\n",
            "        good       0.79      0.75      0.77       141\n",
            "\n",
            "    accuracy                           0.69       200\n",
            "   macro avg       0.63      0.64      0.63       200\n",
            "weighted avg       0.70      0.69      0.69       200\n",
            "\n",
            "\n",
            "Confusion Matrix (Test Data):\n",
            "[[ 31  28]\n",
            " [ 35 106]]\n",
            "\n",
            "===========================\n",
            "\n",
            "svm Model:\n",
            "Training Accuracy: 0.795\n",
            "Test Accuracy: 0.76\n",
            "\n",
            "Classification Report (Test Data):\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         bad       0.72      0.31      0.43        59\n",
            "        good       0.77      0.95      0.85       141\n",
            "\n",
            "    accuracy                           0.76       200\n",
            "   macro avg       0.74      0.63      0.64       200\n",
            "weighted avg       0.75      0.76      0.72       200\n",
            "\n",
            "\n",
            "Confusion Matrix (Test Data):\n",
            "[[ 18  41]\n",
            " [  7 134]]\n",
            "\n",
            "===========================\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
        "from sklearn.feature_selection import RFE\n",
        "\n",
        "# Train a RandomForestClassifier\n",
        "rf = Pipeline(steps=[('preprocessor', preprocessor),\n",
        "                      ('classifier', RandomForestClassifier(n_estimators=100, random_state=42))])\n",
        "\n",
        "rf.fit(X_train, y_train)\n",
        "print(\"RandomForestClassifier Model:\\n\")\n",
        "print(\"Training Accuracy: \", rf.score(X_train, y_train))\n",
        "print(\"Test Accuracy: \", rf.score(X_test, y_test))\n",
        "\n",
        "# Train a GradientBoostingClassifier\n",
        "gb = Pipeline(steps=[('preprocessor', preprocessor),\n",
        "                      ('classifier', GradientBoostingClassifier(n_estimators=100, random_state=42))])\n",
        "\n",
        "gb.fit(X_train, y_train)\n",
        "print(\"\\nGradientBoostingClassifier Model:\\n\")\n",
        "print(\"Training Accuracy: \", gb.score(X_train, y_train))\n",
        "print(\"Test Accuracy: \", gb.score(X_test, y_test))\n",
        "\n",
        "# Apply preprocessing on entire dataset:\n",
        "preprocessed_data = preprocessor.fit_transform(X)\n",
        "\n",
        "# Apply RFE\n",
        "# Here we will use Logistic Regression as the model\n",
        "selector = RFE(LogisticRegression(max_iter=5000), n_features_to_select=5, step=1)\n",
        "selector = selector.fit(preprocessed_data, y)\n",
        "print(\"\\nFeatures sorted by their rank:\")\n",
        "print(sorted(zip(map(lambda x: round(x, 4), selector.ranking_), range(preprocessed_data.shape[1]))))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-QTyGXG4KPAD",
        "outputId": "1bbee33c-5dfc-499b-a096-d9aa64269f24"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RandomForestClassifier Model:\n",
            "\n",
            "Training Accuracy:  1.0\n",
            "Test Accuracy:  0.755\n",
            "\n",
            "GradientBoostingClassifier Model:\n",
            "\n",
            "Training Accuracy:  0.8775\n",
            "Test Accuracy:  0.78\n",
            "\n",
            "Features sorted by their rank:\n",
            "[(1, 10), (1, 13), (1, 14), (1, 15), (1, 16), (2, 21), (3, 3), (4, 5), (5, 23), (6, 9), (7, 11), (8, 7), (9, 25), (10, 24), (11, 4), (12, 22), (13, 0), (14, 17), (15, 2), (16, 8), (17, 6), (18, 20), (19, 12), (20, 18), (21, 1), (22, 19)]\n"
          ]
        }
      ]
    }
  ]
}