Cleanup publishers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Cleaning up names of publishers to match standard forms."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nullifications = [\n",
    "    # Regular expression that are simply deleted before normalizing.\n",
    "    # So \"scribner's sons\" becomes \"scribners sons\"\n",
    "    r\"\\.\",\n",
    "    \",\",\n",
    "    \"'\",\n",
    "    r\"\\[\",\n",
    "    r\"\\]\",\n",
    "    r\"\\?\",\n",
    "    \" and co(mpany)?\"\n",
    "]\n",
    "spacifications = [\n",
    "    # Regular expressions that are replaced with a space\n",
    "    \"and co \",\n",
    "    \"and company\"\n",
    "]\n",
    "regex_replacements = [\n",
    "    (\"  +\",\" \"),\n",
    "    (\" ?& ?\",\" and \")\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'm doing some real native python for the actual normalization. The PublisherNormalizer is a subclass of the broader \"Normalizer\" object, which could work on placenames, titles, and so forth as well.\n",
    "\n",
    "I've bundled some unit tests below that demonstrate what sort of changes the normalization code is supposed to make."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "======================================================================\n",
      "FAIL: testOne (__main__.TestNormalization)\n",
      "----------------------------------------------------------------------\n",
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-91-f7ebcfd2ab4e>\", line 31, in testOne\n",
      "    self.failUnless(normer.norm(\"[Scribner's Sons and compay  ]?\")==\"scribners sons\")\n",
      "AssertionError: False is not true\n",
      "\n",
      "----------------------------------------------------------------------\n",
      "Ran 1 test in 0.000s\n",
      "\n",
      "FAILED (failures=1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<unittest.runner.TextTestResult run=1 errors=0 failures=1>"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "class Normalizer(object):\n",
    "    \"\"\"\n",
    "    Normalizes a string while maintaining a lookup dictionary\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self.cache = dict()\n",
    "    def norm(self,string):\n",
    "        try:\n",
    "            return self.cache[string]\n",
    "        except KeyError:\n",
    "            return self.normalize(string)\n",
    "        \n",
    "class PublisherNormalizer(Normalizer):\n",
    "    def normalize(self,string):        \n",
    "        string = string.lower()\n",
    "        string = re.sub(r\"|\".join(nullifications),\"\",string)\n",
    "        string = re.sub(r\"|\".join(spacifications),\" \",string)\n",
    "        for (a,b) in regex_replacements:\n",
    "            string = re.sub(a,b,string)\n",
    "        string = string.strip(\" \").rstrip(\" \")\n",
    "        return string\n",
    "    \n",
    "    \n",
    "import unittest\n",
    "class TestNormalization(unittest.TestCase):\n",
    "    def testOne(self):\n",
    "        normer = PublisherNormalizer()\n",
    "        self.failUnless(normer.norm(\"foo.bar,?\")==\"foobar\")\n",
    "        self.failUnless(normer.norm(\"FOOBAR\")==\"foobar\")\n",
    "        self.failUnless(normer.norm(\"[Scribner's Sons and company  ]?\")==\"scribners sons\")\n",
    "\n",
    "suite = unittest.TestLoader().loadTestsFromTestCase(TestNormalization)\n",
    "unittest.TextTestRunner(verbosity=0).run(suite)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "counts = Counter()\n",
    "\n",
    "terms = []\n",
    "i = 0\n",
    "for line in open(\"publishers.txt\"):\n",
    "    i += 1\n",
    "    (filename,publisher) = line.rstrip(\"\\n\").split(\"\\t\")\n",
    "    terms.append(publisher)\n",
    "    if i > 3000000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "311545\t\n",
      "47460\tus gpo\n",
      "43195\tsn\n",
      "38529\tus gpo :\n",
      "38098\tus govt print off\n",
      "23325\tgovt print off\n",
      "20210\tmacmillan\n",
      "17607\tgpo\n",
      "14785\tlongmans green\n",
      "13503\tharper and brothers\n",
      "12891\tc scribners sons\n",
      "12587\thoughton mifflin\n",
      "12243\td appleton\n",
      "11215\tthe macmillan company\n",
      "9675\tthe society\n",
      "9069\tj murray\n",
      "8708\tus dept of the interior bureau of mines\n",
      "8350\tu s govt print off\n",
      "7777\tlittle brown\n",
      "7471\tus dept of commerce bureau of the census :\n",
      "6506\tdivision of the federal register the national archives\n",
      "6414\thoughton mifflin company\n",
      "5902\tthe bureau :\n",
      "5771\tclarendon press\n",
      "5424\tthe service\n",
      "5314\tthe office\n",
      "5308\tg p putnams sons\n",
      "4892\th holt\n",
      "4828\tgp putnams sons\n",
      "4693\tdodd mead\n",
      "4620\tuniversity press\n",
      "4532\tus dept of agriculture\n",
      "4053\tharper\n",
      "4038\tthe century co\n",
      "3909\tbg teubner\n"
     ]
    }
   ],
   "source": [
    "normer = PublisherNormalizer()\n",
    "for term in terms:\n",
    "    counts[normer.norm(term)] += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "311545\t\n",
      "47460\tus gpo\n",
      "43195\tsn\n",
      "38529\tus gpo :\n",
      "38098\tus govt print off\n",
      "23325\tgovt print off\n",
      "20210\tmacmillan\n",
      "17607\tgpo\n",
      "14785\tlongmans green\n",
      "13503\tharper and brothers\n",
      "12891\tc scribners sons\n",
      "12587\thoughton mifflin\n",
      "12243\td appleton\n",
      "11215\tthe macmillan company\n",
      "9675\tthe society\n",
      "9069\tj murray\n",
      "8708\tus dept of the interior bureau of mines\n",
      "8350\tu s govt print off\n",
      "7777\tlittle brown\n",
      "7471\tus dept of commerce bureau of the census :\n",
      "6506\tdivision of the federal register the national archives\n",
      "6414\thoughton mifflin company\n",
      "5902\tthe bureau :\n",
      "5771\tclarendon press\n",
      "5424\tthe service\n",
      "5314\tthe office\n",
      "5308\tg p putnams sons\n",
      "4892\th holt\n",
      "4828\tgp putnams sons\n",
      "4693\tdodd mead\n",
      "4620\tuniversity press\n",
      "4532\tus dept of agriculture\n",
      "4053\tharper\n",
      "4038\tthe century co\n",
      "3909\tbg teubner\n",
      "3773\ta and c black\n",
      "3763\tg fischer\n",
      "3703\tchapman and hall\n",
      "3632\tr bentley\n",
      "3612\tg reimer\n",
      "3612\tus dept of agriculture :\n",
      "3609\tja barth\n",
      "3524\tj springer\n",
      "3442\tmcgraw-hill\n",
      "3440\tthe university\n",
      "3422\tthe association\n",
      "3381\tthe commission\n",
      "3363\tb g teubner\n",
      "3216\tf alcan\n",
      "3215\tw engelmann\n",
      "3176\tginn\n",
      "3105\tscribner\n",
      "3065\tthe survey\n",
      "2994\tthe bureau\n",
      "2990\tthe board\n"
     ]
    }
   ],
   "source": [
    "for (k,v) in counts.most_common(55):\n",
    "    print \"{}\\t{}\".format(v,k)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}