deal_with_problematic_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_type = 'thread'\n",
    "field_name = 'title'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_file = open(f'{file_type}_{field_name}_token_lists.pickle', \"rb\" )\n",
    "corpus = pickle.load( pickle_file )\n",
    "pickle_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pronounes_letters = 'מושכלתמהב'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Creating a list of stop words and adding custom stopwords\n",
    "import codecs\n",
    "with codecs.open('bgulex.utf8.hr', 'r', encoding='utf8') as f:\n",
    "    text = f.read()\n",
    "lines = text.split('\\n')\n",
    "words_in_heb_dict = set()\n",
    "for line in lines:\n",
    "    word = line.split(':')[0].split(' ')[0].strip().replace(' DEF','')\n",
    "    if len(word) > 0:\n",
    "        words_in_heb_dict.add(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for sentence in corpus:\n",
    "    for i in range(len(sentence)):\n",
    "        word = sentence[i]\n",
    "        new_sentence = list\n",
    "        if word in sentence:\n",
    "            new_sentence.append(word)\n",
    "        elif i < len(sentence) and str(word + sentence[i]): #try to join words\n",
    "            new_sentence.append(str(word + sentence[i]))\n",
    "        else:\n",
    "            if word[0] in pronounes_letters:\n",
    "                \n",
    "        i += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}