09.genderize.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gender predictions for names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pathlib\n",
    "import pandas\n",
    "import ratelimit\n",
    "import requests\n",
    "import jsonlines\n",
    "import backoff\n",
    "try:\n",
    "    from utils.secrets import genderize_api_key\n",
    "except ImportError:\n",
    "    genderize_api_key = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fore_name_simple</th>\n",
       "      <th>n_authors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>david</td>\n",
       "      <td>13221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>michael</td>\n",
       "      <td>12381</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  fore_name_simple  n_authors\n",
       "0            david      13221\n",
       "1          michael      12381"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fore_name_df = pandas.read_csv('data/names/fore-names.tsv.xz', sep='\\t', keep_default_na=False, na_values=[''])\n",
    "# read_csv keep_default_na=True (deafult) interprets some names as NaN\n",
    "assert fore_name_df[fore_name_df.fore_name.isna()].empty\n",
    "fore_name_df = (\n",
    "    fore_name_df\n",
    "    .groupby('fore_name_simple')\n",
    "    .n_authors.sum()\n",
    "    .sort_values(ascending=False)\n",
    "    .reset_index()\n",
    ")\n",
    "fore_name_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "@backoff.on_exception(backoff.expo, Exception)\n",
    "@ratelimit.limits(calls=90*5, period=86_400)\n",
    "def request_genderize_io(names, service='genderize', api_key=None):\n",
    "    \"\"\"\n",
    "    service='nationalize' is also available\n",
    "    \"\"\"\n",
    "    assert 1 <= len(names) <= 10\n",
    "    url = f\"https://api.{service}.io\"\n",
    "    params = {f\"name[{i}]\": name for i, name in enumerate(names)}\n",
    "    if api_key:\n",
    "        params[\"apikey\"] = api_key\n",
    "    response = requests.get(url, params)\n",
    "    response.raise_for_status()\n",
    "    results = response.json()\n",
    "    for result in results:\n",
    "        result['query_date'] = datetime.datetime.utcnow().date().isoformat()\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grouper(iterable, n):\n",
    "    \"\"\"\n",
    "    Collect data into fixed-length chunks or blocks\n",
    "    https://docs.python.org/3/library/itertools.html\n",
    "    grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx\"\n",
    "    \"\"\"\n",
    "    from itertools import zip_longest\n",
    "    args = [iter(iterable)] * n\n",
    "    return zip_longest(*args)\n",
    "\n",
    "def genderize_names(names, service='genderize'):\n",
    "    for subset in grouper(names, n=10):\n",
    "        yield from request_genderize_io(subset, service=service, api_key=genderize_api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Genderize fore names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76,539 total names: 76,205 already queried, 784 new\n"
     ]
    }
   ],
   "source": [
    "path_genderize = pathlib.Path('data/gender/genderize.jsonl')\n",
    "lines = jsonlines.open(path_genderize) if path_genderize.exists() else []\n",
    "existing_names = {row['name'] for row in lines}\n",
    "all_names = fore_name_df.sort_values('n_authors', ascending=False).fore_name_simple\n",
    "new_names = all_names[~all_names.isin(existing_names)]\n",
    "print(f\"{len(all_names):,} total names: {len(existing_names):,} already queried, {len(new_names):,} new\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with jsonlines.open(path_genderize, mode='a') as writer:\n",
    "    results = genderize_names(new_names, service='genderize')\n",
    "    for result in results:\n",
    "        writer.write(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_male_probability(result):\n",
    "    \"\"\"\n",
    "    Add probability_male field to a genderize.io result\n",
    "    \"\"\"\n",
    "    if result['gender']:\n",
    "        result['probability_male'] = (\n",
    "            result['probability']\n",
    "            if result['gender'] == 'male' else\n",
    "            1 - result['probability']\n",
    "        )\n",
    "    else:\n",
    "        result['probability_male'] = None\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fore_name_simple</th>\n",
       "      <th>n_authors</th>\n",
       "      <th>genderize_sample_size</th>\n",
       "      <th>query_date</th>\n",
       "      <th>probability_male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>david</td>\n",
       "      <td>13221</td>\n",
       "      <td>501011</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>michael</td>\n",
       "      <td>12381</td>\n",
       "      <td>245870</td>\n",
       "      <td>2019-09-20</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  fore_name_simple  n_authors  genderize_sample_size  query_date  \\\n",
       "0            david      13221                 501011  2019-09-20   \n",
       "1          michael      12381                 245870  2019-09-20   \n",
       "\n",
       "   probability_male  \n",
       "0              0.99  \n",
       "1              0.99  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = jsonlines.open(path_genderize)\n",
    "lines = map(add_male_probability, lines)\n",
    "gender_df = pandas.DataFrame(lines)\n",
    "gender_df = gender_df.rename(columns={'name': 'fore_name_simple', 'count': 'genderize_sample_size'})\n",
    "gender_df = gender_df.drop(columns=['gender', 'probability'])\n",
    "gender_df = gender_df.sort_values(['fore_name_simple', 'query_date']).drop_duplicates(subset=['fore_name_simple', 'query_date'], keep='last')\n",
    "gender_df = fore_name_df.merge(gender_df)\n",
    "gender_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_df.to_csv('data/gender/genderize.tsv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nationalize fore names\n",
    "\n",
    "The following chunks are commented out because `nationalize.io` returns only the top three nationalities regardless of number - not useful at the moment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path_nationalize = pathlib.Path('data/gender/nationalize.jsonl')\n",
    "# lines = jsonlines.open(path_nationalize) if path_nationalize.exists() else []\n",
    "# existing_names = {row['name'] for row in lines}\n",
    "# new_names = all_names[~all_names.isin(existing_names)]\n",
    "# print(f\"{len(all_names):,} total names: {len(existing_names):,} already queried, {len(new_names):,} new\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with jsonlines.open(path_nationalize, mode='a') as writer:\n",
    "#     results = genderize_names(new_names, service='nationalize')\n",
    "#     for result in results:\n",
    "#         writer.write(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}