Bookworm-Hathi-MARC_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "This is a quick demo of what sort of data I'm pulling out with Hathi-Specific features of the Bookworm-MARC library.\n",
    "\n",
    "\n",
    "First just some basic imports, including from this library."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import pymarc\n",
    "import random\n",
    "import json\n",
    "from bookwormMARC.bookwormMARC import parse_record\n",
    "from bookwormMARC.hathi_methods import hathi_record_yielder\n",
    "from bookwormMARC.bookwormMARC import LCCallNumber\n",
    "\n",
    "import bookwormMARC\n",
    "import sys\n",
    "import os\n",
    "from collections import defaultdict\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "all_files = hathi_record_yielder([\"/Users/bschmidt/Downloads/bb3.xml\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cataloging_source': u' ',\n",
       " 'cntry': u'enk',\n",
       " 'contributing_library': 'Unknown',\n",
       " 'date': 1883,\n",
       " 'filename': u'coo1.ark:/13960/t5q81vr3g',\n",
       " 'first_author_birth': 1816,\n",
       " 'first_author_death': 1909,\n",
       " 'first_author_name': u'Martin, Theodore, Sir, 1816-1909.',\n",
       " 'first_place': u'London :',\n",
       " 'first_publisher': u'J. Murray,',\n",
       " 'government_document': u' ',\n",
       " 'item_date': 1883,\n",
       " 'language': u'eng',\n",
       " 'lc0': 'D',\n",
       " 'lc1': 'DA',\n",
       " 'lc2': '536.',\n",
       " 'lc_class_from_lc': True,\n",
       " 'literary_form': 'Not fiction',\n",
       " 'marc_record_created': u'1982-12-10',\n",
       " 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t5q81vr3g',\n",
       " 'record_date': 1883,\n",
       " 'resource_type': 'book',\n",
       " 'rights_changed_date': u'2015-08-20',\n",
       " 'scanner': u'cornell-ms',\n",
       " 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t5q81vr3g><em>A life of Lord Lyndhurst from letters and papers in possession of his family.</em> (1883)',\n",
       " 'serial_killer_guess': 'serial',\n",
       " 'target_audience': 'Unknown or not specified',\n",
       " 'title': u'A life of Lord Lyndhurst from letters and papers in possession of his family.'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec = all_files.next()\n",
    "rec.hathi_bookworm_dicts().next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rec' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ff8466ca3247>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhathi_bookworm_dicts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'rec' is not defined"
     ]
    }
   ],
   "source": [
    "rec.hathi_bookworm_dicts().next()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Example output\n",
    "\n",
    "Here is an example of the output of this script on Hathi books: 5 randomly selected records from the first 50000 or so in the DPLA dump. This is usually, note, more than 5 *items*: Hathi groups multiple items into a single record.\n",
    "\n",
    "Note that we're using a custom superset of the pymarc.Record class called `BRecord`. This adds a number of functions that make it easier--for instance--to pull out a dictionary with the categories that may be useful for Bookworms in a variety of ways.\n",
    "\n",
    "Each of the keys here is something that might make sense to chart or analyze. We want to know the scanner so that we can see if there are OCR effects or something that might be relevant. We want the library so we can see how shifting library composition affects time series. It might make sense to build up miniature bookworms for particular authors, or publishers, etc.\n",
    "\n",
    "To start with, I just print four random records from the first 500 or so."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"cataloging_source\": \" \",\n",
      "  \"cntry\": \"mau\",\n",
      "  \"contributing_library\": \"Cornell University\",\n",
      "  \"date\": 1900,\n",
      "  \"filename\": \"coo1.ark:/13960/t9s18j800\",\n",
      "  \"first_author_birth\": 1862,\n",
      "  \"first_author_death\": 1930,\n",
      "  \"first_author_name\": \"Stratemeyer, Edward, 1862-1930.\",\n",
      "  \"first_place\": \"Boston,\",\n",
      "  \"first_publisher\": \"Lee and Shepard,\",\n",
      "  \"government_document\": \" \",\n",
      "  \"item_date\": 1900,\n",
      "  \"language\": \"eng\",\n",
      "  \"lc0\": \"P\",\n",
      "  \"lc1\": \"PZ\",\n",
      "  \"lc2\": \"7.\",\n",
      "  \"lc_class_from_lc\": true,\n",
      "  \"literary_form\": \"Fiction\",\n",
      "  \"marc_record_created\": \"1972-07-24\",\n",
      "  \"permalink\": \"https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t9s18j800\",\n",
      "  \"record_date\": 1900,\n",
      "  \"resource_type\": \"book\",\n",
      "  \"rights_changed_date\": \"2015-07-19\",\n",
      "  \"scanner\": \"cornell-ms\",\n",
      "  \"searchstring\": \"<a href=https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t9s18j800><em>The campaign of the jungle; or, Under Lawton through Luzon /</em> (1900)\",\n",
      "  \"serial_killer_guess\": \"book\",\n",
      "  \"subject_places\": [\n",
      "    \"a-ph---\"\n",
      "  ],\n",
      "  \"target_audience\": \"Unknown or not specified\",\n",
      "  \"title\": \"The campaign of the jungle; or, Under Lawton through Luzon /\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"cataloging_source\": \" \",\n",
      "  \"cntry\": \"mau\",\n",
      "  \"contributing_library\": \"Cornell University\",\n",
      "  \"date\": 1910,\n",
      "  \"filename\": \"coo1.ark:/13960/t8qc0kj10\",\n",
      "  \"first_author_birth\": 1803,\n",
      "  \"first_author_death\": 1882,\n",
      "  \"first_author_name\": \"Emerson, Ralph Waldo, 1803-1882.\",\n",
      "  \"first_place\": \"Boston ;\",\n",
      "  \"first_publisher\": \"Houghton Mifflin Company,\",\n",
      "  \"government_document\": \" \",\n",
      "  \"item_date\": 1910,\n",
      "  \"language\": \"eng\",\n",
      "  \"lc0\": \"P\",\n",
      "  \"lc1\": \"PS\",\n",
      "  \"lc2\": \"1633\",\n",
      "  \"lc_class_from_lc\": true,\n",
      "  \"literary_form\": \"Letters\",\n",
      "  \"marc_record_created\": \"1972-08-09\",\n",
      "  \"permalink\": \"https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t8qc0kj10\",\n",
      "  \"record_date\": 1910,\n",
      "  \"resource_type\": \"book\",\n",
      "  \"rights_changed_date\": \"2015-07-21\",\n",
      "  \"scanner\": \"cornell-ms\",\n",
      "  \"searchstring\": \"<a href=https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t8qc0kj10><em>Records of a lifelong friendship, 1807-1882 : Ralph Waldo Emerson and William Henry Furness /</em> (1910)\",\n",
      "  \"serial_killer_guess\": \"book\",\n",
      "  \"target_audience\": \"Unknown or not specified\",\n",
      "  \"title\": \"Records of a lifelong friendship, 1807-1882 : Ralph Waldo Emerson and William Henry Furness /\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"cataloging_source\": \"u\",\n",
      "  \"cntry\": \"nyu\",\n",
      "  \"contributing_library\": \"isrlf\",\n",
      "  \"date\": 1807,\n",
      "  \"filename\": \"uc2.ark:/13960/t1zc7t220\",\n",
      "  \"first_author_birth\": 1742,\n",
      "  \"first_author_death\": 1823,\n",
      "  \"first_author_name\": \"Combe, William, 1742-1823.\",\n",
      "  \"first_place\": \"Troy, N.Y. :\",\n",
      "  \"first_publisher\": \"Printed and sold by Wright, Goodenow, & Stockwell,\",\n",
      "  \"government_document\": \" \",\n",
      "  \"item_date\": 1807,\n",
      "  \"language\": \"eng\",\n",
      "  \"literary_form\": \"Not fiction\",\n",
      "  \"marc_record_created\": \"1972-08-10\",\n",
      "  \"permalink\": \"https://babel.hathitrust.org/cgi/pt?id=uc2.ark:/13960/t1zc7t220\",\n",
      "  \"record_date\": 1807,\n",
      "  \"resource_type\": \"book\",\n",
      "  \"rights_changed_date\": \"2013-08-10\",\n",
      "  \"scanner\": \"ia\",\n",
      "  \"searchstring\": \"<a href=https://babel.hathitrust.org/cgi/pt?id=uc2.ark:/13960/t1zc7t220><em>Letters of the late Lord Lyttleton; to which is now added, a memoir concerning the author, including an account of some extraordinary circumstances attending his death.</em> (1807)\",\n",
      "  \"serial_killer_guess\": \"book\",\n",
      "  \"target_audience\": \"Unknown or not specified\",\n",
      "  \"title\": \"Letters of the late Lord Lyttleton; to which is now added, a memoir concerning the author, including an account of some extraordinary circumstances attending his death.\"\n",
      "}\n",
      "\n",
      ".\n",
      "{\n",
      "  \"cataloging_source\": \" \",\n",
      "  \"cntry\": \"enk\",\n",
      "  \"contributing_library\": \"Cornell University\",\n",
      "  \"date\": 1883,\n",
      "  \"filename\": \"coo1.ark:/13960/t5q81vr3g\",\n",
      "  \"first_author_birth\": 1816,\n",
      "  \"first_author_death\": 1909,\n",
      "  \"first_author_name\": \"Martin, Theodore, Sir, 1816-1909.\",\n",
      "  \"first_place\": \"London :\",\n",
      "  \"first_publisher\": \"J. Murray,\",\n",
      "  \"government_document\": \" \",\n",
      "  \"item_date\": 1883,\n",
      "  \"language\": \"eng\",\n",
      "  \"lc0\": \"D\",\n",
      "  \"lc1\": \"DA\",\n",
      "  \"lc2\": \"536.\",\n",
      "  \"lc_class_from_lc\": true,\n",
      "  \"literary_form\": \"Not fiction\",\n",
      "  \"marc_record_created\": \"1982-12-10\",\n",
      "  \"permalink\": \"https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t5q81vr3g\",\n",
      "  \"record_date\": 1883,\n",
      "  \"resource_type\": \"book\",\n",
      "  \"rights_changed_date\": \"2015-08-20\",\n",
      "  \"scanner\": \"cornell-ms\",\n",
      "  \"searchstring\": \"<a href=https://babel.hathitrust.org/cgi/pt?id=coo1.ark:/13960/t5q81vr3g><em>A life of Lord Lyndhurst from letters and papers in possession of his family.</em> (1883)\",\n",
      "  \"serial_killer_guess\": \"serial\",\n",
      "  \"target_audience\": \"Unknown or not specified\",\n",
      "  \"title\": \"A life of Lord Lyndhurst from letters and papers in possession of his family.\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"cataloging_source\": \"d\",\n",
      "  \"cntry\": \"enk\",\n",
      "  \"contributing_library\": \"inrlf\",\n",
      "  \"date\": 1920,\n",
      "  \"filename\": \"uc2.ark:/13960/t1sf2n69b\",\n",
      "  \"first_author_name\": \"MacCarthy, Edward Thomas.\",\n",
      "  \"first_place\": \"London,\",\n",
      "  \"first_publisher\": \"Routledge,\",\n",
      "  \"government_document\": \" \",\n",
      "  \"item_date\": 1920,\n",
      "  \"language\": \"eng\",\n",
      "  \"literary_form\": \"Not fiction\",\n",
      "  \"marc_record_created\": \"1985-04-27\",\n",
      "  \"permalink\": \"https://babel.hathitrust.org/cgi/pt?id=uc2.ark:/13960/t1sf2n69b\",\n",
      "  \"record_date\": 1920,\n",
      "  \"resource_type\": \"book\",\n",
      "  \"rights_changed_date\": \"2015-08-05\",\n",
      "  \"scanner\": \"ia\",\n",
      "  \"searchstring\": \"<a href=https://babel.hathitrust.org/cgi/pt?id=uc2.ark:/13960/t1sf2n69b><em>Further incidents in the life of a mining engineer,</em> (1920)\",\n",
      "  \"serial_killer_guess\": \"book\",\n",
      "  \"target_audience\": \"Unknown or not specified\",\n",
      "  \"title\": \"Further incidents in the life of a mining engineer,\"\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from bookwormMARC.bookwormMARC import BRecord\n",
    "\n",
    "n=0\n",
    "for rec in all_files:\n",
    "    if random.random()>.5:\n",
    "        print \".\"\n",
    "        continue\n",
    "    for entry in rec.hathi_bookworm_dicts():\n",
    "        # Pretty print the dictionary entry.\n",
    "        print json.dumps(entry,sort_keys=True, indent=2, separators=(',', ': ') )\n",
    "        print \"\"\n",
    "    n+=1\n",
    "    if n>4:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "Experimental: testing the goodness of record 043 codes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "if False:\n",
    "    all_files = hathi_record_yielder()\n",
    "    knowledge = open(\"/drobo/knowledge_directions.tsv\",\"w\")\n",
    "\n",
    "    for record in all_files:\n",
    "        if record['043'] is not None:\n",
    "            try:\n",
    "                dicto = record.bookworm_dict()\n",
    "                subjects = dicto['subject_places']\n",
    "                p1 = record.first_place()\n",
    "                cntry = dicto['cntry']\n",
    "                year = dicto['date']\n",
    "                for subject in subjects:\n",
    "                    knowledge.write(\"\\t\".join(map(str,[subject,p1,year,cntry,record['001'].value(),dicto['title'].encode(\"utf-8\")]))+ \"\\n\")\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### The world of available fields\n",
    "\n",
    "This code creates a list of fields that appear in more than 10% of a randomly selected subset of records. They include the control fields; author and title information; and some more esoteric things including country of study."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "n = 0\n",
    "global_counts = defaultdict(int)\n",
    "\n",
    "for record in all_files:\n",
    "    if random.random() >.2:\n",
    "        continue\n",
    "    already_seen = set([])\n",
    "    n+=1\n",
    "    from collections import defaultdict\n",
    "    for dicto in record.as_dict()['fields']:\n",
    "        name = dicto.keys()[0]\n",
    "        if 'subfields' in dicto[name]:\n",
    "            for subfield in dicto[name]['subfields']:\n",
    "                tupo = (name,subfield.keys()[0])\n",
    "        else:\n",
    "            tupo = (name,None)\n",
    "        if not tupo in already_seen:\n",
    "            global_counts[tupo] +=1\n",
    "            already_seen.add(tupo)\n",
    "    if n > 10000:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "a = [((k,v),count) for ((k,v),count) in global_counts.iteritems()]\n",
    "a.sort()\n",
    "for elem in a:\n",
    "    if elem[1] > 1000:\n",
    "        print elem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Better years\n",
    "\n",
    "One of the big things I've noticed is that the 974 field has better year information than the record information, such as individual fields. \n",
    "\n",
    "The following block shows that something like 1 in 3 items, in about one in ten records, have a different entry in the 974y field from the native date field. That suggests huge possibilities for improving dates if we're not already using the 974y fields: I suspect we are not based on the serial volumes that include 974y fields I see in the online browser.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "hathi_record_yielder() takes at least 1 argument (0 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-dc250696dd2b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mall_files\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhathi_record_yielder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mrecords\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: hathi_record_yielder() takes at least 1 argument (0 given)"
     ]
    }
   ],
   "source": [
    "all_files = hathi_record_yielder()\n",
    "\n",
    "import collections\n",
    "\n",
    "records = 0\n",
    "diff_records = 0\n",
    "items = 0\n",
    "diff_items = 0\n",
    "date_diffs = collections.defaultdict(int)\n",
    "for rec in all_files:\n",
    "    if random.random() > .1:\n",
    "        # Print just one in one hundred files each time for debugging\n",
    "        continue\n",
    "    records += 1\n",
    "    line_counted = False\n",
    "    for dicto in rec.hathi_bookworm_dicts():\n",
    "        try:\n",
    "            if dicto[\"item_date\"] != dicto[\"record_date\"]:\n",
    "                date_diffs[(dicto[\"item_date\"],dicto[\"record_date\"])] += 1\n",
    "        except KeyError:\n",
    "            pass\n",
    "    if records>1000:\n",
    "        break\n",
    "print \"%i out of %i records and %i out of %i items have differing dates\" %(diff_records,records,diff_items,items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Assessing differences in dates between 974 and the main MARC record\n",
    "\n",
    "]The most common pattern is that I'm replacing a \"None\" value with an actual year, or vice versa. It would be wise to see if there isn't sometimes a better solution than the Nones for the original fields. (Eg; am I overrelying on F008?)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'date_diffs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-4c779291654c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mflattened\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mf008\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mf974\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf974\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mf008\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdate_diffs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mflattened\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'date_diffs' is not defined"
     ]
    }
   ],
   "source": [
    "flattened = sorted([(-val,f008,f974,val) for ((f974,f008),val) in date_diffs.iteritems()])\n",
    "flattened[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Looking at only places where we have years, most are the realm of reasonableness here.\n",
    "(With just 1000 examples, I'm certainly getting a lot of repeat entries.)\n",
    "\n",
    "There are, though, a number of places where f974 instates an earlier entry than does the native date field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "flattened = sorted([(-val,f008,f974,val) for ((f974,f008),val) in date_diffs.iteritems() \n",
    "                    if f974 is not None and f008 is not None])\n",
    "flattened[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's look to see what those are. Here are thirty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "records = 0\n",
    "for rec in all_files:\n",
    "    if random.random() > .5:\n",
    "        # Print just one in two for debugging\n",
    "        continue\n",
    "    rec.__class__ = BRecord\n",
    "    for field in rec.get_fields('974'):\n",
    "        items += 1\n",
    "        if str(field['y']) != str(rec.date()) and field['y'] is not None and rec.date() is not None:\n",
    "            if int(field['y']) < int(rec.date()):\n",
    "                records += 1\n",
    "                print rec\n",
    "    if records>30:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The basic problem in all of these seems to be that in the original record, field 260c and field 008 disagree on the date. Pymarc prefers 260 in these cases; Zephir prefers field 008. Fair enough."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Cataloging Errors: What country is Vienna in?\n",
    "\n",
    "I've noticed that cataloging librarians often just dump the city \"Wien\" into Germany because, I guess, they look only at the language. How often does this happen? Here's some code to check, 1000 entries at a time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n = 0\n",
    "right = 0\n",
    "wrong = 0\n",
    "all_files = hathi_record_yielder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('oops', u'gw ', u'Wien,')\n",
      "So far, 7 wrong and 85 right (7.61%)\n"
     ]
    }
   ],
   "source": [
    "n = 0\n",
    "\n",
    "for rec in all_files:\n",
    "    n += 1\n",
    "    if n>100000:\n",
    "        break\n",
    "    dicto =  rec.bookworm_dict()\n",
    "    try:\n",
    "        if \"Wien\" in dicto['first_place'] and not \"und Wien\" in dicto['first_place']:\n",
    "            if dicto['cntry'] != 'au ' and dicto['cntry'] != 'au#' and dicto['cntry'] != \"xx \":\n",
    "                wrong += 1\n",
    "                #a = bookwormMARC.bookwormMARC.F008(rec)\n",
    "                print (\"oops\", dicto['cntry'],dicto['first_place'])\n",
    "            else:\n",
    "                #print dicto['cntry']\n",
    "                right += 1\n",
    "    except KeyError:\n",
    "        continue\n",
    "try:\n",
    "    print \"So far, {} wrong and {} right ({:0.2f}%)\".format(wrong,right,100*float(wrong)/float(wrong+right))\n",
    "except ZeroDivisionError:\n",
    "    print \"Haven't found any yet: try running again\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n = 0\n",
    "\n",
    "dictee = defaultdict(int)\n",
    "\n",
    "for rec in all_files:\n",
    "    n+=1\n",
    "    if n > 10:\n",
    "        break\n",
    "        \n",
    "    for item in rec.hathi_bookworm_dicts():\n",
    "        if item['serial_killer_guess'] != item['resource_type']:\n",
    "            print item['title']\n",
    "        try:\n",
    "            dictee[(item['serial_killer_guess'],item['resource_type'])] += 1\n",
    "        except KeyError:\n",
    "            pass\n",
    "        break\n",
    "for (k,v) in dictee.iteritems():\n",
    "    print (k,v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cataloging_source': u' ',\n",
       " 'cntry': u'nyu',\n",
       " 'contributing_library': u'nrlf',\n",
       " 'date': 1949,\n",
       " 'filename': u'uc1.$b243514',\n",
       " 'first_author_birth': 1902,\n",
       " 'first_author_name': u'Meeker, Arthur, 1902-',\n",
       " 'first_place': u'New York,',\n",
       " 'first_publisher': u'A. A. Knopf,',\n",
       " 'government_document': u' ',\n",
       " 'item_date': 1949,\n",
       " 'language': u'eng',\n",
       " 'lc0': 'P',\n",
       " 'lc1': 'PZ',\n",
       " 'lc2': '3.',\n",
       " 'lc_class_from_lc': True,\n",
       " 'literary_form': 'Fiction',\n",
       " 'marc_record_created': u'1985-04-08',\n",
       " 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=uc1.$b243514',\n",
       " 'record_date': 1949,\n",
       " 'resource_type': 'book',\n",
       " 'rights_changed_date': u'2013-10-15',\n",
       " 'scanner': u'google',\n",
       " 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=uc1.$b243514><em>Prairie Avenue.</em> (1949)',\n",
       " 'serial_killer_guess': 'book',\n",
       " 'target_audience': 'Unknown or not specified',\n",
       " 'title': u'Prairie Avenue.'}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bar\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "testrow = \"CLASS C - AUXILIARY SCIENCES OF HISTORY\"\n",
    "file = open(\"/drobo/hathi_metadata/vocabularies/lc_class.txt\",\"r\")\n",
    "line = line.rstrip(\"\\n\")\n",
    "line = line.rstrip(\"\\r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def parse_row(string):\n",
    "    \"\"\"\n",
    "    Each row is 4-tuple; the level,\n",
    "    lower bound,\n",
    "    the upper bound, and the label.\n",
    "    \"\"\"\n",
    "    l1 = re.search(r\"CLASS ([A-Z]) - (\\w+)\",string)\n",
    "    if l1:\n",
    "        groups = l1.groups()\n",
    "        return (0,groups[0],groups[0],groups[1])\n",
    "    l2 = re.search(r\"Subclass ([A-Z]{1,3})\",string)\n",
    "    if l2:\n",
    "        groups = l2.groups()\n",
    "        return (1,groups[0],groups[0],None)\n",
    "    l3 = re.search(r'([A-Z]+)\\(?(\\d+(?:\\.[A-Z]?\\d*)?)\\)?-?\\(?(\\d+(?:\\.[A-Z]?\\d*)?)?\\)?(\\t+)(.*)',string)\n",
    "    if l3:\n",
    "        groups = list(l3.groups())\n",
    "        if groups[2] is None:\n",
    "            groups[2] = groups[1]\n",
    "        return (1 + len(groups[3]),groups[1],groups[2],groups[4])\n",
    "\n",
    "line = file.readline()\n",
    "if line ==\"\\n\": line = file.readline()\n",
    "\n",
    "print line\n",
    "print parse_row(line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "for i in xrange(20):\n",
    "    foo = all_files.next()\n",
    "    print foo['008'].data[32] + '-' + foo['008'].data[33]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000 complete\n",
      "200000 complete\n",
      "300000 complete\n",
      "400000 complete\n",
      "500000 complete\n",
      "600000 complete\n",
      "700000 complete\n",
      "800000 complete\n",
      "900000 complete\n",
      "1000000 complete\n",
      "1100000 complete\n",
      "1200000 complete\n",
      "1300000 complete\n",
      "1400000 complete\n",
      "1500000 complete\n",
      "1600000 complete\n",
      "1700000 complete\n",
      "1800000 complete\n",
      "1900000 complete\n",
      "2000000 complete\n",
      "2100000 complete\n",
      "2200000 complete\n",
      "2300000 complete\n",
      "2400000 complete\n",
      "2500000 complete\n",
      "2600000 complete\n",
      "2700000 complete\n",
      "2800000 complete\n",
      "2900000 complete\n",
      "3000000 complete\n",
      "3100000 complete\n",
      "3200000 complete\n",
      "3300000 complete\n",
      "3400000 complete\n",
      "3500000 complete\n",
      "3600000 complete\n",
      "3700000 complete\n",
      "3800000 complete\n",
      "3900000 complete\n",
      "4000000 complete\n",
      "4100000 complete\n",
      "4200000 complete\n",
      "4300000 complete\n",
      "4400000 complete\n",
      "4500000 complete\n",
      "4600000 complete\n",
      "4700000 complete\n",
      "4800000 complete\n",
      "4900000 complete\n",
      "5000000 complete\n",
      "5100000 complete\n",
      "5200000 complete\n",
      "5300000 complete\n",
      "5400000 complete\n",
      "5500000 complete\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "output = open(\"/drobo/jsoncatalog.txt\",\"w\")\n",
    "all_files = hathi_record_yielder()\n",
    "for record in hathi_record_yielder():\n",
    "    for item in record.hathi_bookworm_dicts():\n",
    "        output.write(json.dumps(item) + \"\\n\")\n",
    "        i += 1\n",
    "        if i % 100000 == 0:\n",
    "            print \"{} complete\".format(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rec' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ea91ade2ae20>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrec\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'082'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'rec' is not defined"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}