create_heb_dictionary_from_wiki.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.corpora import WikiCorpus\n",
    "import urllib.request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file_wiki = \"hewiki-latest-pages-articles.xml.bz2\"\n",
    "output_file_wiki = \"wiki.he.text\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using existing wikipedia dump\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "my_file = Path(\"input_file_wiki\")\n",
    "url = 'https://dumps.wikimedia.org/hewiki/latest/hewiki-latest-pages-articles.xml.bz2'\n",
    "if not my_file.is_file():\n",
    "    print('using existing wikipedia dump')\n",
    "else:\n",
    "    print('downlowding the new wikipedia dump')\n",
    "    urllib.request.urlretrieve(url, input_file_wiki)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_words = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting to create wiki corpus\n"
     ]
    }
   ],
   "source": [
    "print(\"Starting to create wiki corpus\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = open(output_file_wiki, 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "space = \" \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = WikiCorpus(input_file_wiki, lemmatize=False, dictionary={})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved 1000 articles\n",
      "Saved 2000 articles\n",
      "Saved 3000 articles\n",
      "Saved 4000 articles\n",
      "Saved 5000 articles\n",
      "Saved 6000 articles\n",
      "Saved 7000 articles\n",
      "Saved 8000 articles\n",
      "Saved 9000 articles\n",
      "Saved 10000 articles\n",
      "Saved 11000 articles\n",
      "Saved 12000 articles\n",
      "Saved 13000 articles\n",
      "Saved 14000 articles\n",
      "Saved 15000 articles\n",
      "Saved 16000 articles\n",
      "Saved 17000 articles\n",
      "Saved 18000 articles\n",
      "Saved 19000 articles\n",
      "Saved 20000 articles\n",
      "Saved 21000 articles\n",
      "Saved 22000 articles\n",
      "Saved 23000 articles\n",
      "Saved 24000 articles\n",
      "Saved 25000 articles\n",
      "Saved 26000 articles\n",
      "Saved 27000 articles\n",
      "Saved 28000 articles\n",
      "Saved 29000 articles\n",
      "Saved 30000 articles\n",
      "Saved 31000 articles\n",
      "Saved 32000 articles\n",
      "Saved 33000 articles\n",
      "Saved 34000 articles\n",
      "Saved 35000 articles\n",
      "Saved 36000 articles\n",
      "Saved 37000 articles\n",
      "Saved 38000 articles\n",
      "Saved 39000 articles\n",
      "Saved 40000 articles\n",
      "Saved 41000 articles\n",
      "Saved 42000 articles\n",
      "Saved 43000 articles\n",
      "Saved 44000 articles\n",
      "Saved 45000 articles\n",
      "Saved 46000 articles\n",
      "Saved 47000 articles\n",
      "Saved 48000 articles\n",
      "Saved 49000 articles\n",
      "Saved 50000 articles\n",
      "Saved 51000 articles\n",
      "Saved 52000 articles\n",
      "Saved 53000 articles\n",
      "Saved 54000 articles\n",
      "Saved 55000 articles\n",
      "Saved 56000 articles\n",
      "Saved 57000 articles\n",
      "Saved 58000 articles\n",
      "Saved 59000 articles\n",
      "Saved 60000 articles\n",
      "Saved 61000 articles\n",
      "Saved 62000 articles\n",
      "Saved 63000 articles\n",
      "Saved 64000 articles\n",
      "Saved 65000 articles\n",
      "Saved 66000 articles\n",
      "Saved 67000 articles\n",
      "Saved 68000 articles\n",
      "Saved 69000 articles\n",
      "Saved 70000 articles\n",
      "Saved 71000 articles\n",
      "Saved 72000 articles\n",
      "Saved 73000 articles\n",
      "Saved 74000 articles\n",
      "Saved 75000 articles\n",
      "Saved 76000 articles\n",
      "Saved 77000 articles\n",
      "Saved 78000 articles\n",
      "Saved 79000 articles\n",
      "Saved 80000 articles\n",
      "Saved 81000 articles\n",
      "Saved 82000 articles\n",
      "Saved 83000 articles\n",
      "Saved 84000 articles\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "for text in wiki.get_texts():\n",
    "    set_words = set_words.union(set([t for t in text]))\n",
    "    i += 1\n",
    "    if (i % 1000 == 0):\n",
    "        print(\"Saved \" + str(i) + \" articles\")\n",
    "output.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('wiki.heb.txt','w')\n",
    "for word in set_words:\n",
    "\tf.write(word.strip())\n",
    "\tf.write('\\n')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}