demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bm25+dataset_search+fuzzy.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "G6YC5M9hrmyV"
      },
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00lFAKaIMszn"
      },
      "source": [
        "PATH = 'data/'"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCxnB_JGI3M4",
        "outputId": "d2102bdb-d1fd-45d0-c510-9b7d5519ba25",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        }
      },
      "source": [
        "from model.utils import *"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
            "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i_VVIGUgSlYx"
      },
      "source": [
        "# load everithing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjA9pzffI4sr"
      },
      "source": [
        "papers, f = load_data(PATH)\n",
        "papers = papers.sample(200) #-  for faster experiments"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cl72f-xkb9ZX",
        "outputId": "65ad6820-d7aa-492f-c6e9-24c9215019bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "papers.shape"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(200, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9jWgvFtqSyS8"
      },
      "source": [
        "# train models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8bPdzqOIJuBG",
        "outputId": "97958d3c-b200-4322-c22b-e0d670ab19a7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_models(papers)# по статьям"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "200it [00:14, 13.77it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZAdcCtIvxO2D",
        "outputId": "7ab9fb2f-611b-43e7-8ee0-16eafbe178f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "train_search_model(papers, f)# "
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ClFXuHVbZ75-"
      },
      "source": [
        "model = BM25Okapi_custom(path='./papers_search/')"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iIqBrZVXQ4li"
      },
      "source": [
        "# search обычный\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8BbeRjrkPyNx"
      },
      "source": [
        "kek = model.paper_search('sgd optimization')"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJZ8vYzfrUyz",
        "outputId": "c6849061-b11b-4ce3-e738-939d038d06d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        }
      },
      "source": [
        "kek[0]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(158,\n",
              " 'Painless Stochastic Gradient: Interpolation, Line-Search, and Convergence Rates',\n",
              " 2019,\n",
              " \"['Sharan Vaswani', 'Aaron Mishkin', 'Issam Laradji', 'Mark Schmidt', 'Gauthier Gidel', 'Simon Lacoste-Julien']\",\n",
              " ['A variant of korpelevichs method for variational inequalities with a new search strategy. Optimization 1997. 33 Prateek Jain Sham Kakade Rahul Kidambi Praneeth Netrapalli and Aaron Sidford',\n",
              "  '8 Aharon BenTal Laurent El Ghaoui and Arkadi Nemirovski. Robust optimization. Princeton University Press 2009',\n",
              "  '12  59 Jorge Nocedal and Stephen Wright. Numerical optimization. Springer Science  Business Media 2006'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M3evx0OsRRVR"
      },
      "source": [
        "# dataset search\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPMF2Nk1Q82r",
        "outputId": "6879aa30-f7e9-44de-9233-f69ea1b171fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "kek = model.dataset_search(\"SST\")\n",
        "len(kek)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p0N0cTtXRur6",
        "outputId": "50b71e38-263b-43cd-d3c7-0bb4bee2da36",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        }
      },
      "source": [
        "kek[0]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(63,\n",
              " 2019,\n",
              " \"['Wenbo Ren', 'Jia (Kevin) Liu', 'Ness Shroff']\",\n",
              " 'On Sample Complexity Upper and Lower Bounds for Exact Ranking from Noisy Comparisons')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1PIbVubEVynG"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}