BERT_OCR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT OCR.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOfwH06FwgAGvklT55rM/im",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hawc2/Text-Analysis-with-Python/blob/master/BERT_OCR.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEFY3olF6P4f"
      },
      "source": [
        "# Resources\n",
        "\n",
        "Tutorial on BERT OCR: \n",
        "\n",
        "https://www.statestitle.com/resource/using-nlp-bert-to-improve-ocr-accuracy/\n",
        "\n",
        "Related: https://medium.com/towards-artificial-intelligence/using-tesseract-ocr-for-text-recognition-with-google-colab-1c4513b9d3e0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UWMA-pW63GLb"
      },
      "source": [
        "# Libraries\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jPWXe_-O3YNU",
        "outputId": "1120a957-b290-4e89-ffe7-2e4b53ae6c1b"
      },
      "source": [
        "!pip install pytesseract\n",
        "!pip install pytorch-pretrained-bert\n",
        "!pip3 install pyenchant"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pytesseract\n",
            "  Downloading https://files.pythonhosted.org/packages/a0/e6/a4e9fc8a93c1318540e8de6d8d4beb5749b7960388a7c7f27799fc2dd016/pytesseract-0.3.7.tar.gz\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from pytesseract) (7.0.0)\n",
            "Building wheels for collected packages: pytesseract\n",
            "  Building wheel for pytesseract (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytesseract: filename=pytesseract-0.3.7-py2.py3-none-any.whl size=13945 sha256=386ea76f328ee4d6a31f7efd5e0a35b7957c6d76f9d1d266c1894b6440417c80\n",
            "  Stored in directory: /root/.cache/pip/wheels/81/20/7e/1dd0daad1575d5260916bb1e9781246430647adaef4b3ca3b3\n",
            "Successfully built pytesseract\n",
            "Installing collected packages: pytesseract\n",
            "Successfully installed pytesseract-0.3.7\n",
            "Collecting pytorch-pretrained-bert\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/e0/c08d5553b89973d9a240605b9c12404bcf8227590de62bae27acbcfe076b/pytorch_pretrained_bert-0.6.2-py3-none-any.whl (123kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 5.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.7.0+cu101)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2019.12.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (4.41.1)\n",
            "Collecting boto3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/65/c4/6a866d878e41d0fc4e29bc7782118e5202f21beee8fc492b10917e138a75/boto3-1.16.55-py2.py3-none-any.whl (130kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 5.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch>=0.4.1->pytorch-pretrained-bert) (3.7.4.3)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch>=0.4.1->pytorch-pretrained-bert) (0.8)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=0.4.1->pytorch-pretrained-bert) (0.16.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2020.12.5)\n",
            "Collecting jmespath<1.0.0,>=0.7.1\n",
            "  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n",
            "Collecting s3transfer<0.4.0,>=0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ea/43/4b4a1b26eb03a429a4c37ca7fdf369d938bd60018fc194e94b8379b0c77c/s3transfer-0.3.4-py2.py3-none-any.whl (69kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 4.7MB/s \n",
            "\u001b[?25hCollecting botocore<1.20.0,>=1.19.55\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/47/45d58c8a96646acd36b81899d9c6bdc2b3e7fe0aac253f1424e9f7071c00/botocore-1.19.55-py2.py3-none-any.whl (7.2MB)\n",
            "\u001b[K     |████████████████████████████████| 7.2MB 7.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.55->boto3->pytorch-pretrained-bert) (2.8.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.20.0,>=1.19.55->boto3->pytorch-pretrained-bert) (1.15.0)\n",
            "\u001b[31mERROR: botocore 1.19.55 has requirement urllib3<1.27,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: jmespath, botocore, s3transfer, boto3, pytorch-pretrained-bert\n",
            "Successfully installed boto3-1.16.55 botocore-1.19.55 jmespath-0.10.0 pytorch-pretrained-bert-0.6.2 s3transfer-0.3.4\n",
            "Collecting pyenchant\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/8c/bd224a5db562ac008edbfaf015f5d5c98ea13e745247cd4ab5fc5b683085/pyenchant-3.2.0-py3-none-any.whl (55kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 3.2MB/s \n",
            "\u001b[?25hInstalling collected packages: pyenchant\n",
            "Successfully installed pyenchant-3.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mWY0DHX5ejm6",
        "outputId": "77565f5c-d4b2-488b-c5a2-1d241558e381"
      },
      "source": [
        "!sudo apt-get -y install libenchant1c2a\n",
        "!sudo apt install tesseract-ocr\n",
        "!sudo apt install libtesseract-dev"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  aspell aspell-en dictionaries-common emacsen-common enchant hunspell-en-us\n",
            "  libaspell15 libhunspell-1.6-0 libtext-iconv-perl\n",
            "Suggested packages:\n",
            "  aspell-doc spellutils wordlist hunspell openoffice.org-hunspell\n",
            "  | openoffice.org-core libenchant-voikko\n",
            "The following NEW packages will be installed:\n",
            "  aspell aspell-en dictionaries-common emacsen-common enchant hunspell-en-us\n",
            "  libaspell15 libenchant1c2a libhunspell-1.6-0 libtext-iconv-perl\n",
            "0 upgraded, 10 newly installed, 0 to remove and 16 not upgraded.\n",
            "Need to get 1,310 kB of archives.\n",
            "After this operation, 5,353 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libtext-iconv-perl amd64 1.7-5build6 [13.0 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libaspell15 amd64 0.60.7~20110707-4ubuntu0.1 [309 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 emacsen-common all 2.0.8 [17.6 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic/main amd64 dictionaries-common all 1.27.2 [186 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 aspell amd64 0.60.7~20110707-4ubuntu0.1 [87.6 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 aspell-en all 2017.08.24-0-0.1 [298 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 hunspell-en-us all 1:2017.08.24 [168 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhunspell-1.6-0 amd64 1.6.2-1 [154 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 libenchant1c2a amd64 1.6.0-11.1 [64.4 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 enchant amd64 1.6.0-11.1 [12.2 kB]\n",
            "Fetched 1,310 kB in 1s (1,527 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 10.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libtext-iconv-perl.\n",
            "(Reading database ... 145483 files and directories currently installed.)\n",
            "Preparing to unpack .../0-libtext-iconv-perl_1.7-5build6_amd64.deb ...\n",
            "Unpacking libtext-iconv-perl (1.7-5build6) ...\n",
            "Selecting previously unselected package libaspell15:amd64.\n",
            "Preparing to unpack .../1-libaspell15_0.60.7~20110707-4ubuntu0.1_amd64.deb ...\n",
            "Unpacking libaspell15:amd64 (0.60.7~20110707-4ubuntu0.1) ...\n",
            "Selecting previously unselected package emacsen-common.\n",
            "Preparing to unpack .../2-emacsen-common_2.0.8_all.deb ...\n",
            "Unpacking emacsen-common (2.0.8) ...\n",
            "Selecting previously unselected package dictionaries-common.\n",
            "Preparing to unpack .../3-dictionaries-common_1.27.2_all.deb ...\n",
            "Adding 'diversion of /usr/share/dict/words to /usr/share/dict/words.pre-dictionaries-common by dictionaries-common'\n",
            "Unpacking dictionaries-common (1.27.2) ...\n",
            "Selecting previously unselected package aspell.\n",
            "Preparing to unpack .../4-aspell_0.60.7~20110707-4ubuntu0.1_amd64.deb ...\n",
            "Unpacking aspell (0.60.7~20110707-4ubuntu0.1) ...\n",
            "Selecting previously unselected package aspell-en.\n",
            "Preparing to unpack .../5-aspell-en_2017.08.24-0-0.1_all.deb ...\n",
            "Unpacking aspell-en (2017.08.24-0-0.1) ...\n",
            "Selecting previously unselected package hunspell-en-us.\n",
            "Preparing to unpack .../6-hunspell-en-us_1%3a2017.08.24_all.deb ...\n",
            "Unpacking hunspell-en-us (1:2017.08.24) ...\n",
            "Selecting previously unselected package libhunspell-1.6-0:amd64.\n",
            "Preparing to unpack .../7-libhunspell-1.6-0_1.6.2-1_amd64.deb ...\n",
            "Unpacking libhunspell-1.6-0:amd64 (1.6.2-1) ...\n",
            "Selecting previously unselected package libenchant1c2a:amd64.\n",
            "Preparing to unpack .../8-libenchant1c2a_1.6.0-11.1_amd64.deb ...\n",
            "Unpacking libenchant1c2a:amd64 (1.6.0-11.1) ...\n",
            "Selecting previously unselected package enchant.\n",
            "Preparing to unpack .../9-enchant_1.6.0-11.1_amd64.deb ...\n",
            "Unpacking enchant (1.6.0-11.1) ...\n",
            "Setting up libhunspell-1.6-0:amd64 (1.6.2-1) ...\n",
            "Setting up libaspell15:amd64 (0.60.7~20110707-4ubuntu0.1) ...\n",
            "Setting up emacsen-common (2.0.8) ...\n",
            "Setting up libtext-iconv-perl (1.7-5build6) ...\n",
            "Setting up dictionaries-common (1.27.2) ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76.)\n",
            "debconf: falling back to frontend: Readline\n",
            "Setting up aspell (0.60.7~20110707-4ubuntu0.1) ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76.)\n",
            "debconf: falling back to frontend: Readline\n",
            "Setting up hunspell-en-us (1:2017.08.24) ...\n",
            "Setting up libenchant1c2a:amd64 (1.6.0-11.1) ...\n",
            "Setting up aspell-en (2017.08.24-0-0.1) ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76.)\n",
            "debconf: falling back to frontend: Readline\n",
            "Setting up enchant (1.6.0-11.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.2) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for dictionaries-common (1.27.2) ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76.)\n",
            "debconf: falling back to frontend: Readline\n",
            "aspell-autobuildhash: processing: en [en-common].\n",
            "aspell-autobuildhash: processing: en [en-variant_0].\n",
            "aspell-autobuildhash: processing: en [en-variant_1].\n",
            "aspell-autobuildhash: processing: en [en-variant_2].\n",
            "aspell-autobuildhash: processing: en [en-w_accents-only].\n",
            "aspell-autobuildhash: processing: en [en-wo_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_AU-variant_0].\n",
            "aspell-autobuildhash: processing: en [en_AU-variant_1].\n",
            "aspell-autobuildhash: processing: en [en_AU-w_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_AU-wo_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_CA-variant_0].\n",
            "aspell-autobuildhash: processing: en [en_CA-variant_1].\n",
            "aspell-autobuildhash: processing: en [en_CA-w_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_CA-wo_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_GB-ise-w_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_GB-ise-wo_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_GB-ize-w_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_GB-ize-wo_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_GB-variant_0].\n",
            "aspell-autobuildhash: processing: en [en_GB-variant_1].\n",
            "aspell-autobuildhash: processing: en [en_US-w_accents-only].\n",
            "aspell-autobuildhash: processing: en [en_US-wo_accents-only].\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  tesseract-ocr-eng tesseract-ocr-osd\n",
            "The following NEW packages will be installed:\n",
            "  tesseract-ocr tesseract-ocr-eng tesseract-ocr-osd\n",
            "0 upgraded, 3 newly installed, 0 to remove and 16 not upgraded.\n",
            "Need to get 4,795 kB of archives.\n",
            "After this operation, 15.8 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr-eng all 4.00~git24-0e00fe6-1.2 [1,588 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr-osd all 4.00~git24-0e00fe6-1.2 [2,989 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr amd64 4.00~git2288-10f4998a-2 [218 kB]\n",
            "Fetched 4,795 kB in 1s (4,630 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 3.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package tesseract-ocr-eng.\n",
            "(Reading database ... 145897 files and directories currently installed.)\n",
            "Preparing to unpack .../tesseract-ocr-eng_4.00~git24-0e00fe6-1.2_all.deb ...\n",
            "Unpacking tesseract-ocr-eng (4.00~git24-0e00fe6-1.2) ...\n",
            "Selecting previously unselected package tesseract-ocr-osd.\n",
            "Preparing to unpack .../tesseract-ocr-osd_4.00~git24-0e00fe6-1.2_all.deb ...\n",
            "Unpacking tesseract-ocr-osd (4.00~git24-0e00fe6-1.2) ...\n",
            "Selecting previously unselected package tesseract-ocr.\n",
            "Preparing to unpack .../tesseract-ocr_4.00~git2288-10f4998a-2_amd64.deb ...\n",
            "Unpacking tesseract-ocr (4.00~git2288-10f4998a-2) ...\n",
            "Setting up tesseract-ocr-osd (4.00~git24-0e00fe6-1.2) ...\n",
            "Setting up tesseract-ocr-eng (4.00~git24-0e00fe6-1.2) ...\n",
            "Setting up tesseract-ocr (4.00~git2288-10f4998a-2) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  libleptonica-dev\n",
            "The following NEW packages will be installed:\n",
            "  libleptonica-dev libtesseract-dev\n",
            "0 upgraded, 2 newly installed, 0 to remove and 16 not upgraded.\n",
            "Need to get 2,755 kB of archives.\n",
            "After this operation, 13.8 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libleptonica-dev amd64 1.75.3-3 [1,308 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libtesseract-dev amd64 4.00~git2288-10f4998a-2 [1,447 kB]\n",
            "Fetched 2,755 kB in 1s (2,863 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 2.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libleptonica-dev.\n",
            "(Reading database ... 145944 files and directories currently installed.)\n",
            "Preparing to unpack .../libleptonica-dev_1.75.3-3_amd64.deb ...\n",
            "Unpacking libleptonica-dev (1.75.3-3) ...\n",
            "Selecting previously unselected package libtesseract-dev.\n",
            "Preparing to unpack .../libtesseract-dev_4.00~git2288-10f4998a-2_amd64.deb ...\n",
            "Unpacking libtesseract-dev (4.00~git2288-10f4998a-2) ...\n",
            "Setting up libleptonica-dev (1.75.3-3) ...\n",
            "Setting up libtesseract-dev (4.00~git2288-10f4998a-2) ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W15cdO8VSSu8",
        "outputId": "da2501d5-095c-4463-f158-e45b1242ce98"
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('maxent_ne_chunker')\n",
        "nltk.download('words')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package maxent_ne_chunker to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/words.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9zSdCbzf27S1"
      },
      "source": [
        "from PIL import Image\n",
        "import pytesseract\n",
        "from pytesseract import image_to_string\n",
        "import torch\n",
        "from pytorch_pretrained_bert import BertTokenizer, BertForMaskedLM\n",
        "import re\n",
        "import nltk\n",
        "from difflib import SequenceMatcher"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyDtoUvv-ZK9"
      },
      "source": [
        "# this maybe fixed pyenchant install: https://github.com/pyenchant/pyenchant/issues/214\n",
        "import enchant \n",
        "from enchant.checker import SpellChecker"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mwmNy0x_3Lyp"
      },
      "source": [
        "# File Import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": "OK"
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "ntAIwyTA3mHc",
        "outputId": "8c91573b-9fcb-44df-f40f-3b1c5b6be718"
      },
      "source": [
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "for fn in uploaded.keys():\n",
        "  print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "      name=fn, length=len(uploaded[fn])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8cc3fd84-63ea-467a-a603-3d0af7048f06\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-8cc3fd84-63ea-467a-a603-3d0af7048f06\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving 193696153.23_Page_134.png to 193696153.23_Page_134.png\n",
            "User uploaded file \"193696153.23_Page_134.png\" with length 6484767 bytes\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yjBRuUA-3FCh",
        "outputId": "7f2c8406-dfab-4f46-e6bc-3478bb79df67"
      },
      "source": [
        "filename = '19369615323_Page_134.png'\n",
        "text = image_to_string(Image.open(filename))\n",
        "text_original = str(text)\n",
        "print (text_original)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "124\n",
            "\n",
            "Coleoptera\n",
            "Pentamera.\n",
            "\n",
            "ae\n",
            "\n",
            "ENTOMOLOGY.\n",
            "\n",
            "SrcTion 4TH, SIMPLICIMANI.\n",
            "\n",
            "This section resembles the preceding so far as regards\n",
            "the termination of the elytra ;! but the two anterior tarsi\n",
            "only are dilated in the males, without however forming a\n",
            "square or orbicular pallet; sometimes the first three ar-\n",
            "ticles are obviously broader, and the succceding one 1s In\n",
            "that case always much less than its antecedent ; sometimes\n",
            "the latter and the two preceding ones are larger, almost\n",
            "equal, and in the form of a reversed or triangular heart.\n",
            "The joints of the four following tarsi are more slender and\n",
            "elongated, almost cylindrical, or in the form of a length-\n",
            "ened and reversed cone. | |\n",
            "\n",
            "The Simplicimani of Latreille consist of genera belong-\n",
            "ing to the tribe Carabiques L’eroniens of Dejean, with the\n",
            "addition of Tetragonoderus, Dejean (Harpalici), Catasco-\n",
            "pus, Kirby (Truncatipemnes), and a few others. _\n",
            "\n",
            "Genus ZABrus, Clairville. The first three articles of\n",
            "the anterior tarsi dilated in the males, broader than long,\n",
            "strongly cordiform. Terminal article of the palpi almost\n",
            "cylindrical, and truncated at the extremity. Antenne\n",
            "filiform, and but slightly lengthened. Labrum of a square\n",
            "form, broader than long, slightly notched anteriorly.\n",
            "Mandibles little advanced, rather strongly arcuated, al-\n",
            "most obtuse. A simple tooth in the middle of the emar-\n",
            "eination of the mentum. Body thick and convex. Tho-\n",
            "rax transverse, square, trapezoidal, or rounded on the sides.\n",
            "Elytra convex, rarely elongated, frequently very short, al-\n",
            "most parallel, and rounded at the extremity.\n",
            "\n",
            "These insects are usually found under stones, or march-\n",
            "ing about the fields, occasionally on the stalks of grass or\n",
            "other herbage. We are acquainted with about twenty\n",
            "species, of which not more than two are British. The\n",
            "majority of the others occur in the southern parts of Eu-\n",
            "rope. There is one from Teneriffe.\n",
            "\n",
            "Genus FeroniA, Lat. Dej. The first three articles of\n",
            "the anterior tarsi dilated in the males, broader than long,\n",
            "and strongly triangular or cordiform. Tcrminal article of\n",
            "the palpi more or less elongated, cylindrical, or slightly\n",
            "securiform. Antenne filiform, more or less lengthened.\n",
            "Labrum square, broader than long, sometimes almost trans-\n",
            "versal, quadrate anteriorly, or slightly notched. Man-\n",
            "dibles more or less advanced, and more or less arcuated\n",
            "and pointed. A bifid tooth in the middle of the emargi-\n",
            "nation of the mentum. ‘Thorax more or less cordiform,\n",
            "rounded, square, or trapezoidal, never transverse. Ely-\n",
            "tra more or less elongated, oval, or parallel. Intcrmediate\n",
            "tibiz always straight. .\n",
            "\n",
            "A great variety of opinion is maintained by entomolo-\n",
            "gists regarding the distribution of the component parts of\n",
            "this extensive genus. Bonelli, in the synoptical table\n",
            "annexed to the first part of his Observations Eintomolo-\n",
            "giques (in 1809), published the generic characters of Pla-\n",
            "tysma, Pecilus, Abax, Molops, Percus, Melanius, and Pter-\n",
            "ostichus. At an after period MM. Megerle and Ziegler\n",
            "established Argutor, Steropus, Cophosus, and Omaseus,.\n",
            "the latter genus, it is believed, corresponding to Melanius\n",
            "of Bonelli. All these wcre admitted provisionally into the\n",
            "first edition of Comte Dejean’s Catalogue, published in\n",
            "1821. In the third voluine, however, of the Species Gé-\n",
            "néral of that great collector, he observes as follows: “ Ma\n",
            "collection depuis cette époque s’etant considérablement\n",
            "augmentée, je me suis trouvé souvent tres-embarassé pour\n",
            "placer dans ces différents genres les espéces qui je rece-\n",
            "\n",
            " \n",
            "\n",
            "; |\n",
            "\n",
            "1 e ° , ° . ° ee\n",
            "a As “ai there are exceptions to the rule :—‘t Plusieurs Carabiques simplicimanes,” says Latreille, ‘ ont Vextrémité de leurs\n",
            "elytres fortemeut sinuée au bout, et se distinguant 4 peine, sous ce rapport, des Troncatipennes.”\n",
            "\n",
            "p. 399.)\n",
            "\n",
            "; For a further exposition of Latreille’s views, see Régne Animal, tom. iv. p. 395.\n",
            "lor the arrangement of the indigenous species, see Mr Stephens’ Nomenclature of British Insects, 2a edition.\n",
            "\n",
            "  \n",
            " \n",
            " \n",
            "  \n",
            "  \n",
            " \n",
            "  \n",
            " \n",
            "  \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            "  \n",
            " \n",
            " \n",
            " \n",
            "  \n",
            " \n",
            " \n",
            "  \n",
            "\n",
            "_\n",
            "ze\n",
            "\n",
            "vais, surtout les exotiques, et quelquefois plusieurs d’en- C\n",
            "tre elles me paraissaient intermédiaires entre deux genres,\n",
            "et appartenir autant a l'un qu’a l'autre. Plus je recevais\n",
            "d’insectes, plus je me trouvals embarassé, et lorsque je\n",
            "voulus classer definitivement toutes mes espéces dans —\n",
            "chaque genre, et etablir les. caracteres de chacun de ces '\n",
            "genres, aprés plusieurs essais infructueux, je finis par étre {\n",
            "convaincu que cela était réellement impossible.” Dejean —\n",
            "then thought of following tlic example of Sturm, who re- |\n",
            "duces (in his Deutschlands Fauna) all the genera above 1\n",
            "named to those of Abaz, Molops, Pterostichus, and Pla-—\n",
            "tysma ; but after a deliberate re-cxamination, even these ‘\n",
            "did not appear sufficiently marked ; and he finally deter- —\n",
            "mined to combine them all in one under the name of —\n",
            "FERONIA, a term previously used in a comprehensive sense 1\n",
            "by Latreille in the first edition of the Régne Animal. 1\n",
            "The genus Feronia, then, of Dejean is one of the most\n",
            "extensive in the coleopterous order, containing above 230 ©\n",
            "species. It is subdivided by the last-named author into |\n",
            "ten sections, corresponding to the same number of genera —\n",
            "as established by Bonelli, Megerle, Ziegler, and Sturm.\n",
            "According to Latreille’s views, the genus is divisible\n",
            "into three groups as follows: Is¢, The species, generally\n",
            "winged, of which the body, more or less oval, is a little\n",
            "convex or arcuated above, with the antennz more filiform, —\n",
            "the head proportionally narrower, and the mandibles some- —\n",
            "what less projecting. In their habits these insects ap-\n",
            "proach the genera Harpalus and Zabrus. Such are the —\n",
            "Amare, of which the thorax is transverse (and which —\n",
            "Dejean admits as a distinct genus); the Pecili, where 1\n",
            "is almost as long as broad, and where the rather short an-—\n",
            "tennz have the third article compressed and angular ; and —\n",
            "the Argutores, which resemble the Peecili, but have pro- _\n",
            "portionally shorter antenn, with the third joint not an-—\n",
            "gular.\n",
            "2d, The species usually furnished with wings, but of\n",
            "which the body is straight, plane, or horizontal above,”\n",
            "with the head nearly as wide as the body. Such is th\n",
            "genus Platysma of Bonelli, to which Latreille unites Oma--\n",
            "seus of Zicgler and Dejean, and Catadromus of Macleay.\n",
            "3d, This division contains Feronize, analogous to those\n",
            "of the preceding one in their general characters, but dis-_\n",
            "tinguished by the want of wings.?\n",
            "- We shall conclude by observing that our English en-\n",
            "tomologists have adopted all those minor genera, in so far —\n",
            "as they are applicable to the British species, and have\n",
            "even proposed several others of still more limited applica- |\n",
            "tion. We regret that our necessarily narrow limits pre-\n",
            "vent our entering into minute details.2 We shall merely —\n",
            "indicate a few of the species. /*. madida 1s shining black,\n",
            "the elytra ovate, rather convex, subpunctate-striate, with\n",
            "a puncture on the third interstice a little behind the mid-\n",
            "dle. This is the F. concinna of Sturm and Dejean. A\n",
            "variety occurs with the thighs and sometimes the tibie\n",
            "rufous; it constitutes the F. madida of the latter author.\n",
            "They are not distinct. Both varieties are in this country\n",
            "among the most common of coleopterous insects, occur-\n",
            "ring bencath stones, at the roots of trees, &c. both on low\n",
            "grounds and on mountains. They seem less common in\n",
            "Germany than in any other country in Europe. P*. nt-\n",
            "grita is also of frequent occurrence in all parts of Britain.\n",
            "It is of a shining black, the thorax subquadrate, narrowed\n",
            "behind, the elytra with obsoletely punctured stria, and\n",
            "three impressed dots on the disk. £. orinomum is like-\n",
            "\n",
            "one\n",
            "\n",
            "—7 term ae ot le, _— wo, ras gaat\n",
            "\n",
            "ee\n",
            "ats a —— = tn —~ oo or\n",
            "\n",
            " \n",
            "\n",
            "sil —— cae:\n",
            "\n",
            " \n",
            "\n",
            "(Note to Régne Animal, t. iv.\n",
            "\n",
            "-\n",
            "\f\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gO-FXteY6I7G"
      },
      "source": [
        "# Clean Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7bu4bUru3Nno"
      },
      "source": [
        "# cleanup text\n",
        "rep = { '\\n': ' ', '\\\\': ' ', '\\\"': '\"', '-': ' ', '\"': ' \" ', \n",
        "        '\"': ' \" ', '\"': ' \" ', ',':' , ', '.':' . ', '!':' ! ', \n",
        "        '?':' ? ', \"n't\": \" not\" , \"'ll\": \" will\", '*':' * ', \n",
        "        '(': ' ( ', ')': ' ) ', \"s'\": \"s '\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zaycur3QSEtD"
      },
      "source": [
        "rep = dict((re.escape(k), v) for k, v in rep.items()) \n",
        "pattern = re.compile(\"|\".join(rep.keys()))\n",
        "text = pattern.sub(lambda m: rep[re.escape(m.group(0))], text)\n",
        "def get_personslist(text):\n",
        "    personslist=[]\n",
        "    for sent in nltk.sent_tokenize(text):\n",
        "        for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):\n",
        "            if isinstance(chunk, nltk.tree.Tree) and chunk.label() == 'PERSON':\n",
        "                personslist.insert(0, (chunk.leaves()[0][0]))\n",
        "    return list(set(personslist))\n",
        "personslist = get_personslist(text)\n",
        "ignorewords = personslist + [\"!\", \",\", \".\", \"\\\"\", \"?\", '(', ')', '*', \"'\"]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MV_h990OR_kL"
      },
      "source": [
        "# using enchant.checker.SpellChecker, identify incorrect words\n",
        "# maybe replace SpellChecker\n",
        "d = SpellChecker(\"en_US\")\n",
        "words = text.split()\n",
        "\n",
        "incorrectwords = [w for w in words if not d.check(w) and w not in ignorewords]\n",
        "# using enchant.checker.SpellChecker, get suggested replacements\n",
        "suggestedwords = [d.suggest(w) for w in incorrectwords]\n",
        "\n",
        "# replace incorrect words with [MASK]\n",
        "for w in incorrectwords:\n",
        "    text = text.replace(w, '[MASK]')\n",
        "    text_original = text_original.replace(w, '[MASK]')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oF7fiB96R8nB",
        "outputId": "9d0b8c2c-cc70-455d-f610-5861b4d8cfe8"
      },
      "source": [
        "print(text)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "124  [MASK] [MASK] .   [MASK]  ENTOMOLOGY .   [MASK] 4TH ,  [MASK] .   This [MASK]c[MASK] re[MASK]mb[MASK] [MASK]e p[MASK]d[MASK] so f[MASK] as reg[MASK]ds [MASK]e termina[MASK] of [MASK]e [MASK] [MASK] !  but [MASK]e t[MASK] a[MASK]erior t[MASK]si only [MASK]e dilated in [MASK]e m[MASK]es ,  wi[MASK]out however form[MASK] a squ[MASK]e or orbicul[MASK] p[MASK]l[MASK][MASK] som[MASK]imes [MASK]e first [MASK]r[MASK] [MASK]  [MASK] [MASK]e obviously broa[MASK]r ,  and [MASK]e [MASK] one [MASK] In [MASK]at ca[MASK] [MASK]ways much [MASK]s [MASK]an its a[MASK]e[MASK][MASK][MASK] [MASK] som[MASK]imes [MASK]e latter and [MASK]e t[MASK] p[MASK]d[MASK] ones [MASK]e l[MASK]ger ,  [MASK]most equ[MASK] ,  and in [MASK]e form of a rever[MASK]d or triangul[MASK] he[MASK]t .  The joi[MASK]s of [MASK]e four follow[MASK] t[MASK]si [MASK]e more s[MASK]n[MASK]r and elongated ,  [MASK]most cylindric[MASK] ,  or in [MASK]e form of a [MASK]ng[MASK]  [MASK] and rever[MASK]d cone .  [MASK] [MASK]  The [MASK] of La[MASK]il[MASK] consist of genera belong  [MASK] to [MASK]e tribe C[MASK]abi[MASK]s [MASK] of [MASK] ,  wi[MASK] [MASK]e addi[MASK] of [MASK] ,  [MASK]  ( H[MASK]p[MASK]ici )  ,  [MASK]  pus ,  Kirby  ( [MASK] )  ,  and a few o[MASK]ers .  [MASK]  Genus [MASK] ,  [MASK] .  The first [MASK]r[MASK] [MASK][MASK] of [MASK]e a[MASK]erior t[MASK]si dilated in [MASK]e m[MASK]es ,  broa[MASK]r [MASK]an long ,  strongly [MASK] .  Termin[MASK] [MASK]tic[MASK] of [MASK]e [MASK] [MASK]most cylindric[MASK] ,  and truncated at [MASK]e ex[MASK]mity .  [MASK] [MASK] ,  and but slightly [MASK]ng[MASK][MASK] .  [MASK] of a squ[MASK]e form ,  broa[MASK]r [MASK]an long ,  slightly n[MASK]ched [MASK] .  Man[MASK] litt[MASK] advan[MASK]d ,  ra[MASK]er strongly [MASK]cuated ,  [MASK]  most obtu[MASK] .  A simp[MASK] t[MASK][MASK] in [MASK]e mid[MASK] of [MASK]e em[MASK]  [MASK] of [MASK]e [MASK] .  Body [MASK]ick and convex .  Tho  [MASK] [MASK]nsver[MASK] ,  squ[MASK]e ,  [MASK]pezoid[MASK] ,  or roun[MASK]d on [MASK]e si[MASK]s .  [MASK] convex ,  r[MASK]ely elongated ,  fre[MASK][MASK]ly very short ,  [MASK]  most p[MASK][MASK][MASK]l ,  and roun[MASK]d at [MASK]e ex[MASK]mity .   The[MASK] in[MASK]cts [MASK]e usu[MASK]ly found un[MASK]r stones ,  or m[MASK]ch  [MASK] about [MASK]e fields ,  occasion[MASK]ly on [MASK]e st[MASK]ks of g[MASK]s or o[MASK]er herbage .  We [MASK]e acquai[MASK]ed wi[MASK] about twe[MASK]y species ,  of which n[MASK] more [MASK]an t[MASK] [MASK]e British .  The majority of [MASK]e o[MASK]ers occur in [MASK]e sou[MASK]ern p[MASK]ts of Eu  rope .  There is one from [MASK] .   Genus [MASK] ,  Lat .  [MASK] .  The first [MASK]r[MASK] [MASK][MASK] of [MASK]e a[MASK]erior t[MASK]si dilated in [MASK]e m[MASK]es ,  broa[MASK]r [MASK]an long ,  and strongly triangul[MASK] or [MASK] .  Tcrmin[MASK] [MASK]tic[MASK] of [MASK]e [MASK] more or [MASK]s elongated ,  cylindric[MASK] ,  or slightly [MASK] .  [MASK] [MASK] ,  more or [MASK]s [MASK]ng[MASK][MASK] .  [MASK] squ[MASK]e ,  broa[MASK]r [MASK]an long ,  som[MASK]imes [MASK]most [MASK]ns  vers[MASK] ,  [MASK] [MASK] ,  or slightly n[MASK]ched .  Man  [MASK] more or [MASK]s advan[MASK]d ,  and more or [MASK]s [MASK]cuated and poi[MASK]ed .  A [MASK] t[MASK][MASK] in [MASK]e mid[MASK] of [MASK]e em[MASK]gi  na[MASK] of [MASK]e [MASK] .  [MASK]Tho[MASK] more or [MASK]s [MASK] ,  roun[MASK]d ,  squ[MASK]e ,  or [MASK]pezoid[MASK] ,  never [MASK]nsver[MASK] .  [MASK]  [MASK] more or [MASK]s elongated ,  ov[MASK] ,  or p[MASK][MASK][MASK]l .  [MASK] [MASK] [MASK]ways s[MASK]ight .   .   A great v[MASK]i[MASK]y of opinion is mai[MASK]ained by [MASK]  [MASK] reg[MASK]d[MASK] [MASK]e distribu[MASK] of [MASK]e compone[MASK] p[MASK]ts of [MASK]is extensive genus .  Bonelli ,  in [MASK]e synoptic[MASK] tab[MASK] annexed to [MASK]e first p[MASK]t of his Ob[MASK]rva[MASK]s [MASK]  [MASK]  ( in 1809 )  ,  published [MASK]e generic ch[MASK]acters of Pla  [MASK] ,  Pecilus ,  Abax ,  [MASK] ,  Percus ,  Melanius ,  and Pter  [MASK] .  At an after period MM .  Meger[MASK] and Zieg[MASK]r established Argutor ,  Steropus ,  [MASK] ,  and Oma[MASK]us ,  .  [MASK]e latter genus ,  it is believed ,  correspond[MASK] to Melanius of Bonelli .  All [MASK]e[MASK] [MASK] admitted provision[MASK]ly i[MASK]o [MASK]e first edi[MASK] of Comte [MASK]’s Cat[MASK]ogue ,  published in 1821 .  In [MASK]e [MASK]ird [MASK] ,  however ,  of [MASK]e Species [MASK]  nér[MASK] of [MASK]at great col[MASK]ctor ,  he ob[MASK]rves as [MASK] [MASK] Ma col[MASK]c[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] ,  [MASK] me [MASK] [MASK] [MASK] [MASK] emb[MASK]assé pour pla[MASK]r [MASK] [MASK] [MASK] genres [MASK] espé[MASK] [MASK] [MASK] [MASK]      [MASK] [MASK]  1 e [MASK]  ,  [MASK]  .  [MASK] [MASK] a As [MASK]ai [MASK]ere [MASK]e ex[MASK]p[MASK]s to [MASK]e ru[MASK] [MASK] Plusieurs C[MASK]abi[MASK]s [MASK] , [MASK] says La[MASK]il[MASK] ,  [MASK] [MASK] [MASK] [MASK] [MASK] ely[MASK] [MASK] [MASK] [MASK] bout ,  [MASK] [MASK] dist[MASK]ua[MASK] 4 [MASK] ,  sous [MASK] r[MASK]port ,  [MASK]s Troncatipennes . [MASK]  p .  399 .  )   [MASK] For a fur[MASK]er exposi[MASK] of [MASK] views ,  s[MASK] Régne Anim[MASK] ,  tom .  iv .  p .  395 .  [MASK] [MASK]e [MASK]rangeme[MASK] of [MASK]e indigenous species ,  s[MASK] Mr [MASK] Nomenclature of British In[MASK]cts ,  [MASK] edi[MASK] .                                                                          [MASK] [MASK]  [MASK] ,  [MASK] [MASK] [MASK] ,  [MASK] [MASK] [MASK] [MASK]  C [MASK] el[MASK] me p[MASK]aissaie[MASK] [MASK] en[MASK] [MASK]ux genres ,  [MASK] [MASK]p[MASK]tenir [MASK]ta[MASK] a [MASK] [MASK] l'[MASK][MASK] .  Plus [MASK] [MASK][MASK] d’in[MASK]ctes ,  plus [MASK] me trouv[MASK]s emb[MASK]assé ,  [MASK] [MASK]s[MASK] [MASK] [MASK] clas[MASK]r [MASK]finitiveme[MASK] [MASK] mes espé[MASK] [MASK] [MASK] [MASK] genre ,  [MASK] [MASK]ablir [MASK] .  c[MASK]acteres [MASK] [MASK] [MASK] [MASK] ' genres ,  [MASK] [MASK] [MASK] [MASK] ,  [MASK] finis p[MASK] é[MASK] [MASK] [MASK] [MASK] [MASK]la [MASK] [MASK] impossib[MASK] . [MASK] [MASK] [MASK] [MASK]en [MASK]ought of follow[MASK] [MASK] examp[MASK] of Sturm ,  who re  [MASK] du[MASK]  ( in his [MASK] F[MASK]na )  [MASK]l [MASK]e genera above 1 named to [MASK]o[MASK] of [MASK] ,  [MASK] ,  Pter[MASK] ,  and Pla [MASK] [MASK] [MASK] but after a [MASK]liberate re [MASK] ,  even [MASK]e[MASK] [MASK] did n[MASK] [MASK]pe[MASK] sufficie[MASK]ly m[MASK]ked [MASK] and he fin[MASK]ly [MASK]ter  [MASK] mined to combine [MASK]em [MASK]l in one un[MASK]r [MASK]e name of [MASK] [MASK] ,  a term [MASK]viously u[MASK]d in a com[MASK]hensive [MASK]n[MASK] 1 by La[MASK]il[MASK] in [MASK]e first edi[MASK] of [MASK]e Régne Anim[MASK] .  1 The genus [MASK] ,  [MASK]en ,  of [MASK] is one of [MASK]e most extensive in [MASK]e [MASK] or[MASK]r ,  c[MASK]ain[MASK] above 230 [MASK] species .  It is subdivi[MASK]d by [MASK]e last named [MASK][MASK]or i[MASK]o [MASK] ten [MASK]c[MASK]s ,  correspond[MASK] to [MASK]e same number of genera [MASK] as established by Bonelli ,  Meger[MASK] ,  Zieg[MASK]r ,  and Sturm .  Accord[MASK] to [MASK] views ,  [MASK]e genus is divisib[MASK] i[MASK]o [MASK]r[MASK] groups as [MASK] [MASK] ,  The species ,  gener[MASK]ly w[MASK]ed ,  of which [MASK]e body ,  more or [MASK]s ov[MASK] ,  is a litt[MASK] convex or [MASK]cuated above ,  wi[MASK] [MASK]e [MASK] more [MASK] ,  [MASK] [MASK]e head propor[MASK][MASK]ly n[MASK]rower ,  and [MASK]e man[MASK] some  [MASK] what [MASK]s pro[MASK]ct[MASK] .  In [MASK]eir habits [MASK]e[MASK] in[MASK]cts [MASK]  [MASK] [MASK]e genera H[MASK]p[MASK]us and [MASK] .  Such [MASK]e [MASK]e [MASK] Am[MASK]e ,  of which [MASK]e [MASK]o[MASK] is [MASK]nsver[MASK]  ( and which [MASK] [MASK] admits as a distinct genus ) [MASK] [MASK]e [MASK] ,  where 1 is [MASK]most as long as broad ,  and where [MASK]e ra[MASK]er short an [MASK] [MASK] have [MASK]e [MASK]ird [MASK]tic[MASK] com[MASK]s[MASK]d and angul[MASK] [MASK] and [MASK] [MASK]e [MASK] ,  which re[MASK]mb[MASK] [MASK]e P[MASK]cili ,  but have pro  [MASK] por[MASK][MASK]ly shorter [MASK] ,  wi[MASK] [MASK]e [MASK]ird joi[MASK] n[MASK] an [MASK] gul[MASK] .  [MASK] ,  The species usu[MASK]ly furnished wi[MASK] w[MASK]s ,  but of which [MASK]e body is s[MASK]ight ,  plane ,  or horiz[MASK][MASK] above , [MASK] wi[MASK] [MASK]e head ne[MASK]ly as wi[MASK] as [MASK]e body .  Such is [MASK] genus Pla[MASK] of Bonelli ,  to which La[MASK]il[MASK] unites Oma   [MASK]us of Zicg[MASK]r and [MASK] ,  and [MASK] of [MASK] .  [MASK] ,  This division c[MASK]ains Feroni[MASK] ,  an[MASK]ogous to [MASK]o[MASK] of [MASK]e p[MASK]d[MASK] one in [MASK]eir gener[MASK] ch[MASK]acters ,  but dis [MASK] t[MASK]uished by [MASK]e wa[MASK] of w[MASK]s .  ?    We sh[MASK]l conclu[MASK] by ob[MASK]rv[MASK] [MASK]at our English en  tomolo[MASK] have adopted [MASK]l [MASK]o[MASK] minor genera ,  in so f[MASK] [MASK] as [MASK]ey [MASK]e [MASK]plicab[MASK] to [MASK]e British species ,  and have even propo[MASK]d [MASK]ver[MASK] o[MASK]ers of still more limited [MASK]plica  [MASK] [MASK] .  We regr[MASK] [MASK]at our ne[MASK]s[MASK]ily n[MASK]row limits [MASK]  ve[MASK] our e[MASK]er[MASK] i[MASK]o minute [MASK]tails . 2 We sh[MASK]l merely [MASK] indicate a few of [MASK]e species .  [MASK] *  .  [MASK] [MASK] shin[MASK] black ,  [MASK]e [MASK] ovate ,  ra[MASK]er convex ,  [MASK] [MASK] ,  wi[MASK] a puncture on [MASK]e [MASK]ird i[MASK]ersti[MASK] a litt[MASK] behind [MASK]e mid  [MASK] .  This is [MASK]e F .  [MASK] of Sturm and [MASK] .  A v[MASK]i[MASK]y occurs wi[MASK] [MASK]e [MASK]ighs and som[MASK]imes [MASK]e [MASK] rufous[MASK] it constitutes [MASK]e F .  [MASK] of [MASK]e latter [MASK][MASK]or .  They [MASK]e n[MASK] distinct .  Bo[MASK] v[MASK]i[MASK]ies [MASK]e in [MASK]is cou[MASK]ry among [MASK]e most common of [MASK] in[MASK]cts ,  occur  r[MASK] benca[MASK] stones ,  at [MASK]e ro[MASK]s of tr[MASK]s ,  [MASK] .  bo[MASK] on low grounds and on mou[MASK]ains .  They s[MASK]m [MASK]s common in Germany [MASK]an in any o[MASK]er cou[MASK]ry in Europe .  P *  .  [MASK]  [MASK] is [MASK]so of fre[MASK][MASK] occurren[MASK] in [MASK]l p[MASK]ts of Britain .  It is of a shin[MASK] black ,  [MASK]e [MASK]o[MASK] sub[MASK] ,  n[MASK]rowed behind ,  [MASK]e [MASK] wi[MASK] obsol[MASK]ely punctured stria ,  and [MASK]r[MASK] im[MASK]s[MASK]d d[MASK]s on [MASK]e disk .  [MASK] .  [MASK] is like   one  [MASK]7 term [MASK] [MASK] [MASK] ,  [MASK][MASK] [MASK] ,  [MASK] [MASK]  [MASK] [MASK] a [MASK][MASK] [MASK] tn [MASK]~ [MASK] or     [MASK] [MASK][MASK] c[MASK]:      ( N[MASK]e to Régne Anim[MASK] ,  t .  iv .     \f\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EXSMODYO6Ngu"
      },
      "source": [
        "# Load BERT Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 533
        },
        "id": "aIvda_2n6Kc-",
        "outputId": "a9f8a6d5-0b41-4181-b751-102dba48762f"
      },
      "source": [
        "# Load, train and predict using pre-trained model\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "tokenized_text = tokenizer.tokenize(text)\n",
        "indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "MASKIDS = [i for i, e in enumerate(tokenized_text) if e == '[MASK]']\n",
        "# Create the segments tensors\n",
        "segs = [i for i, e in enumerate(tokenized_text) if e == \".\"]\n",
        "segments_ids=[]\n",
        "prev=-1\n",
        "for k, s in enumerate(segs):\n",
        "    segments_ids = segments_ids + [k] * (s-prev)\n",
        "    prev=s\n",
        "segments_ids = segments_ids + [len(segs)] * (len(tokenized_text) - len(segments_ids))\n",
        "segments_tensors = torch.tensor([segments_ids])\n",
        "# prepare Torch inputs \n",
        "tokens_tensor = torch.tensor([indexed_tokens])\n",
        "# Load pre-trained model\n",
        "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
        "# Predict all tokens\n",
        "with torch.no_grad():\n",
        "    predictions = model(tokens_tensor, segments_tensors)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 231508/231508 [00:00<00:00, 4184994.29B/s]\n",
            "Token indices sequence length is longer than the specified maximum  sequence length for this BERT model (3699 > 512). Running this sequence through BERT will result in indexing errors\n",
            "100%|██████████| 407873900/407873900 [00:06<00:00, 60055149.04B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-cf188817fc88>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;31m# Predict all tokens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msegments_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pytorch_pretrained_bert/modeling.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, token_type_ids, attention_mask, masked_lm_labels)\u001b[0m\n\u001b[1;32m    860\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmasked_lm_labels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    861\u001b[0m         sequence_output, _ = self.bert(input_ids, token_type_ids, attention_mask,\n\u001b[0;32m--> 862\u001b[0;31m                                        output_all_encoded_layers=False)\n\u001b[0m\u001b[1;32m    863\u001b[0m         \u001b[0mprediction_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequence_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    864\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pytorch_pretrained_bert/modeling.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, token_type_ids, attention_mask, output_all_encoded_layers)\u001b[0m\n\u001b[1;32m    728\u001b[0m         \u001b[0mextended_attention_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1.0\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mextended_attention_mask\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m10000.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 730\u001b[0;31m         \u001b[0membedding_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    731\u001b[0m         encoded_layers = self.encoder(embedding_output,\n\u001b[1;32m    732\u001b[0m                                       \u001b[0mextended_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pytorch_pretrained_bert/modeling.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, token_type_ids)\u001b[0m\n\u001b[1;32m    266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \u001b[0mwords_embeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mword_embeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 268\u001b[0;31m         \u001b[0mposition_embeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mposition_embeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mposition_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    269\u001b[0m         \u001b[0mtoken_type_embeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoken_type_embeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/sparse.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    124\u001b[0m         return F.embedding(\n\u001b[1;32m    125\u001b[0m             \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpadding_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_norm\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 126\u001b[0;31m             self.norm_type, self.scale_grad_by_freq, self.sparse)\n\u001b[0m\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   1850\u001b[0m         \u001b[0;31m# remove once script supports set_grad_enabled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1851\u001b[0m         \u001b[0m_no_grad_embedding_renorm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_norm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnorm_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1852\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadding_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscale_grad_by_freq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msparse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1853\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1854\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: index out of range in self"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Ic1XDbikRN8"
      },
      "source": [
        "With Peter's dataset: https://stackoverflow.com/questions/56010551/pytorch-embedding-index-out-of-range"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sDC089QZ7OBV"
      },
      "source": [
        "# Refine BERT predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R3g7rbnm7NPr"
      },
      "source": [
        "#Predict words for mask using BERT; \n",
        "#refine prediction by matching with proposals from SpellChecker\n",
        "def predict_word(text_original, predictions, maskids):\n",
        "    pred_words=[]\n",
        "    for i in range(len(MASKIDS)):\n",
        "        preds = torch.topk(predictions[0, MASKIDS[i]], k=50) \n",
        "        indices = preds.indices.tolist()\n",
        "        list1 = tokenizer.convert_ids_to_tokens(indices)\n",
        "        list2 = suggestedwords[i]\n",
        "        simmax=0\n",
        "        predicted_token=''\n",
        "        for word1 in list1:\n",
        "            for word2 in list2:\n",
        "                s = SequenceMatcher(None, word1, word2).ratio()\n",
        "                if s is not None and s > simmax:\n",
        "                    simmax = s\n",
        "                    predicted_token = word1\n",
        "        text_original = text_original.replace('[MASK]', predicted_token, 1)\n",
        "    return text_original\n",
        "text_original = predict_word(text_original, predictions, MASKIDS)\n",
        "print (text_original)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}