patent_authors_assignees.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import random\n",
    "import urllib\n",
    "import time\n",
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "\n",
    "import rdkit.Chem as Chem"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Goal: Analyze MA change by authors & assignees (companies)\n",
    "\n",
    "First test: sample patents, then authors, then all patents (& therefore compounds) made by authors\n",
    "\n",
    "\n",
    "(do the same as above, but with assignees)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find Authors & Assignees\n",
    "\n",
    "From JSON files, find all authors & assignees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_authors_and_assignees(data):\n",
    "    \"\"\" Use the Pubchem-downloaded patent JSON files to get all authors & assignees \n",
    "\n",
    "    Args:\n",
    "        data (json dict): dictionary of patent JSON records\n",
    "    \"\"\"\n",
    "    inventor_dict = [x for x in data[\"Record\"][\"Section\"] if x[\"TOCHeading\"] == \"Inventor\"][0][\"Information\"][0][\"Value\"][\"StringWithMarkup\"]\n",
    "    authors = [x[\"String\"] for x in inventor_dict]\n",
    "    \n",
    "    assignee_dict = [x for x in data[\"Record\"][\"Section\"] if x[\"TOCHeading\"] == \"Assignee\"][0][\"Information\"][0][\"Value\"][\"StringWithMarkup\"]\n",
    "    assignees = [x[\"String\"] for x in assignee_dict]\n",
    "\n",
    "    return authors, assignees\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 74/74 [00:00<00:00, 1113.23it/s]\n"
     ]
    }
   ],
   "source": [
    "## Get info from all patents in the Data/Patents directory\n",
    "fp = \"Data/Patents/\"\n",
    "files = os.listdir(fp)\n",
    "authors = []\n",
    "assignees = []\n",
    "\n",
    "for file in tqdm(files):\n",
    "\n",
    "    data = json.load(open(fp + file))\n",
    "\n",
    "    new_authors, new_assignees = get_authors_and_assignees(data)\n",
    "\n",
    "    authors.extend(new_authors)\n",
    "    assignees.extend(new_assignees)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove duplicates (if present) and save\n",
    "authors = list(set(authors))\n",
    "assignees = list(set(assignees))\n",
    "\n",
    "pickle.dump(authors, file=open(\"Data/Patents/authors.p\", \"wb\"))\n",
    "pickle.dump(assignees, file=open(\"Data/Patents/assignees.p\", \"wb\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find patent ids associated with authors\n",
    "\n",
    "Scraped patents associated with authors - see *patent_scraping.py*, now I want to\n",
    "find all patents (and eventually compounds) associated with each author"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 21947/21947 [14:30<00:00, 25.22it/s]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       ID                         author  \\\n",
      "0           US-8653233-B2  HOLLINGSWORTH MICHAEL ANTHONY   \n",
      "1           US-8653233-B2                  KOHLGRAF KARL   \n",
      "2           US-8653233-B2                    CAFFREY TOM   \n",
      "3        US-2011312290-A1                 BEELER MICHAEL   \n",
      "4        US-2011312290-A1   CANNON RICHARD HOLLINGSWORTH   \n",
      "...                   ...                            ...   \n",
      "7056907     US-8663196-B2          ZYZELEWSKI MARK EDWIN   \n",
      "7056908     US-9610166-B2              GUNTHER STEPHEN B   \n",
      "7056909     US-9610166-B2              O'FARRELL DESMOND   \n",
      "7056910     US-9610166-B2          ZYZELEWSKI MARK EDWIN   \n",
      "7056911     US-9610166-B2         RODENHOUSE ANDREW JOHN   \n",
      "\n",
      "                                       assignees  \\\n",
      "0                     [UNIV NEBRASKA MEDICAL CT]   \n",
      "1                     [UNIV NEBRASKA MEDICAL CT]   \n",
      "2                     [UNIV NEBRASKA MEDICAL CT]   \n",
      "3                         [COMTECH EF DATA CORP]   \n",
      "4                         [COMTECH EF DATA CORP]   \n",
      "...                                          ...   \n",
      "7056907  [KASSAB KUGHN ENDOVASCULAR DEVICES LLC]   \n",
      "7056908               [SHOULDER INNOVATIONS LLC]   \n",
      "7056909               [SHOULDER INNOVATIONS LLC]   \n",
      "7056910               [SHOULDER INNOVATIONS LLC]   \n",
      "7056911               [SHOULDER INNOVATIONS LLC]   \n",
      "\n",
      "                                            classification  \n",
      "0                                   [A61K38/00, A61K38/17]  \n",
      "1                                   [A61K38/00, A61K38/17]  \n",
      "2                                   [A61K38/00, A61K38/17]  \n",
      "3                                     [H04B1/04, H04B1/10]  \n",
      "4                                     [H04B1/04, H04B1/10]  \n",
      "...                                                    ...  \n",
      "7056907                             [A61M25/00, A61M25/06]  \n",
      "7056908  [A61B17/00, A61B17/15, A61B17/16, A61B17/86, A...  \n",
      "7056909  [A61B17/00, A61B17/15, A61B17/16, A61B17/86, A...  \n",
      "7056910  [A61B17/00, A61B17/15, A61B17/16, A61B17/86, A...  \n",
      "7056911  [A61B17/00, A61B17/15, A61B17/16, A61B17/86, A...  \n",
      "\n",
      "[7056912 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "## Getting patent IDs from all JSON author files\n",
    "full_data = []\n",
    "\n",
    "fp = \"Data/Patents/Patent_Author_Records/\"\n",
    "files = os.listdir(fp)\n",
    "\n",
    "patent_data = []\n",
    "\n",
    "for f in tqdm(files):\n",
    "    try:\n",
    "        data = json.load(open(file=fp + f))\n",
    "    except ValueError:\n",
    "        data = [] \n",
    "\n",
    "    for patent in data:\n",
    "        if \"inventors\" in patent:\n",
    "            inventors = patent[\"inventors\"]\n",
    "            #Get all authors (in list form)\n",
    "            if type(inventors) != list:\n",
    "                authors = []\n",
    "                authors.append(inventors)\n",
    "            else:\n",
    "                authors = inventors\n",
    "        else:\n",
    "            authors = [\"\"]\n",
    "        \n",
    "        #Get assignees which are not part of author list (meant to find only businesses/universities/other)\n",
    "        if \"assignees\" in patent:\n",
    "            a = patent[\"assignees\"]\n",
    "            if type(a) != list:\n",
    "                assignees = []\n",
    "                assignees.append(a)\n",
    "            else:\n",
    "                assignees = a\n",
    "\n",
    "            assignees = list(set(assignees) - set(authors))\n",
    "        else:\n",
    "            assignees = \"\"\n",
    "\n",
    "        if \"classification\" in patent:\n",
    "            classification = patent[\"classification\"]\n",
    "        else:\n",
    "            classification = \"\"\n",
    "    \n",
    "        for author in authors:\n",
    "            patent_data.append({\"ID\": patent[\"publicationnumber\"], \"author\": author, \"assignees\": assignees,\n",
    "                \"classification\": classification})\n",
    "\n",
    "    \n",
    "df = pd.DataFrame.from_dict(patent_data)\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Data/Patents/patent_author_records.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Link patent IDs to compounds\n",
    "\n",
    "Based on testing (below) - using patent_cpd_edges to search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Data/Patents/patent_author_records.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000\n"
     ]
    }
   ],
   "source": [
    "## 1: get all unique patents\n",
    "patents = set(df[\"ID\"])\n",
    "\n",
    "### Randomly sampling from patents in order to make this computationally feasible (goal: ~1 million cpds)\n",
    "patents = random.sample(patents, 100000)\n",
    "\n",
    "print(len(patents))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete patent dataframe\n",
    "\n",
    "del(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_month_increments(start, stop):\n",
    "    \"\"\" Build month increments in the form YYYY-MM\n",
    "\n",
    "    Args:\n",
    "        start (int): Starting year\n",
    "        stop (int): Ending year\n",
    "\n",
    "    Returns:\n",
    "        list: list of strings in the form YYYY-MM (e.g., \"1980-01\")\n",
    "    \"\"\"\n",
    "    months = []\n",
    "    while start <= stop:\n",
    "        for month in [\n",
    "                \"01\", \"02\", \"03\", \"04\", \"05\", \"06\", \"07\", \"08\", \"09\", \"10\",\n",
    "                \"11\", \"12\"\n",
    "        ]:\n",
    "            months.append(str(start) + \"-\" + month)\n",
    "        start += 1\n",
    "\n",
    "    return months"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 492/492 [04:44<00:00,  1.73it/s]\n"
     ]
    }
   ],
   "source": [
    "### Tangent - put all patent_cpd_edgesXXX into a single dictionary to search from\n",
    "\n",
    "#NOTE: ONLY SHOULD BE RUN ONCE\n",
    "\n",
    "patent_cpd_edges = {}\n",
    "\n",
    "months = build_month_increments(1980, 2020)\n",
    "\n",
    "files = []\n",
    "for month in months:\n",
    "    files.append(\"Data/CpdPatentIdsDates/Patent_Cpd_Edges/patent_cpd_edges_\" + month + \".p\")\n",
    "\n",
    "cpds = []\n",
    "for file in tqdm(files):\n",
    "    with open(file, \"rb\") as f:\n",
    "        try:\n",
    "            edges = pickle.load(f)\n",
    "        except EOFError:\n",
    "            edges = {}\n",
    "\n",
    "        patent_cpd_edges.update(edges)\n",
    "\n",
    "# pickle.dump(patent_cpd_edges, file=open(\"Data/CpdPatentIdsDates/Patent_Cpd_Edges/full_patent_cpd_edges.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100000/100000 [00:00<00:00, 100615.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1034394\n",
      "['SCHEMBL6077928', 'SCHEMBL1050012', 'SCHEMBL2848301', 'SCHEMBL48933', 'SCHEMBL8729586', 'SCHEMBL7002444', 'SCHEMBL12741818', 'SCHEMBL14350314', 'SCHEMBL22645514', 'SCHEMBL7919644']\n"
     ]
    }
   ],
   "source": [
    "## Get all compound IDs - filter down into a non-repeating list\n",
    "\n",
    "# #NOTE: UNCOMMENT PICKLE.LOAD WHEN RUNNING THIS WITHOUT BLOCK ABOVE\n",
    "# patent_cpd_edges = pickle.load(file=open(\"Data/CpdPatentIdsDates/Patent_Cpd_Edges/full_patent_cpd_edges.p\", \"rb\"))\n",
    "cpds = []\n",
    "patents = pickle.load(file=open(\"Data/Patents/author_patent_samples.p\", \"rb\"))\n",
    "#selected_patents = [p for p in patent_cpd_edges.keys() if p in patents]\n",
    "    \n",
    "for p in tqdm(patents):\n",
    "    if p in patent_cpd_edges:\n",
    "        cpds.extend(patent_cpd_edges[p])\n",
    "\n",
    "cpds = list(set(cpds))\n",
    "\n",
    "## NOTE: goal of this is to have ~1 million (change number of patents otherwise)\n",
    "print(len(cpds))\n",
    "print(cpds[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-44207cab27a9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m# Delete edges from memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mdel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpatent_cpd_edges\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "## Save compounds\n",
    "pickle.dump(cpds, file=open(\"Data/Patents/author_cpds_sampled.p\", \"wb\"))\n",
    "\n",
    "## Save sampled patents too\n",
    "pickle.dump(patents, file=open(\"Data/Patents/author_patent_samples.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete edges from memory\n",
    "del(patent_cpd_edges)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find patent cpds associated with assignees\n",
    "\n",
    "Goal - less than 1 million unique cpds (different from patent samples - over 1e6 is fine, as long as unique cpds stay below 1e6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6166\n",
      "['', '10X_GENOMICS_INC', '21ST_CENTURY_MEDICINE', '3D_ECO_OIL_LTD', '3D_SYSTEMS_INC', 'A-DEC_INC', 'A_P_C_A_ASSEMBLEE_PERMANENTE_DES_CHAMBRES_D_AGRICULTURE', 'AAKESSON_PER', 'AASBERG-PETERSEN_KIM', 'ABAECHERLI_ROGER']\n"
     ]
    }
   ],
   "source": [
    "## Step 1: get all assignees, and find those which are companies (filter by required naming)\n",
    "\n",
    "assignees = [a[:-5] for a in os.listdir(\"Data/Patents/Patent_Assignee_Records/\")]\n",
    "\n",
    "print(len(assignees))\n",
    "print(assignees[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1850\n"
     ]
    }
   ],
   "source": [
    "# Filtering - from companiesinc.com (might be better ways to do this?), as well as adding universities\n",
    "terms = [\"CORP\", \"INC\", \"CO\", \"LTD\", \"LLC\", \"LLLP\", \"RLLLP\", \"CORPORATION\", \"INCORPORATED\", \"LIMITED\", \"COMPANY\", \"UNIV\", \"UNIVERSITY\"]\n",
    "\n",
    "assignees = [a for a in assignees if any(term in a for term in terms)]\n",
    "\n",
    "print(len(assignees))\n",
    "\n",
    "# #Sample 1000 assigness from this list (otherwise too many compounds)\n",
    "# assignees = random.sample(assignees, 1000)\n",
    "\n",
    "# print(len(assignees))\n",
    "# print(assignees[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1850/1850 [01:53<00:00, 16.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                      ID                                             author  \\\n",
      "0       WO-2021226290-A1  [MCDONNELL WYATT, STUBBINGTON MICHAEL JOHN, MC...   \n",
      "1       WO-2021247618-A1  [MCDONNELL WYATT JAMES, PFEIFFER KATHERINE, RA...   \n",
      "2         US-10544413-B2  [BHARADWAJ RAJIV, SCHNALL-LEVIN MICHAEL, MAKAR...   \n",
      "3       US-2018312873-A1                                    [ZHENG XINYING]   \n",
      "4          US-9975122-B2  [MASQUELIER DONALD A, HINDSON BENJAMIN, NESS K...   \n",
      "...                  ...                                                ...   \n",
      "667160     WO-9621025-A1                                           [LOK SI]   \n",
      "667161     WO-9739767-A1  [DURNAM DIANE M, KUIJPER JOSEPH L, WEIGLE DAVI...   \n",
      "667162     WO-9955869-A1  [CONKLIN DARRELL C, LEWIS KENNETH B JR, GILBER...   \n",
      "667163     US-7829076-B2  [SHEPPARD PAUL O, FOX BRIAN A, KLUCHER KEVIN M...   \n",
      "667164     US-7618791-B2                              [WHITMORE THEODORE E]   \n",
      "\n",
      "               assignees                                     classification  \n",
      "0       10X_GENOMICS_INC     [G01N33/537, G01N33/577, G01N33/58, G01N33/68]  \n",
      "1       10X_GENOMICS_INC                                         C12Q1/6844  \n",
      "2       10X_GENOMICS_INC  [B01L3/00, C12N15/10, C12Q1/6806, C12Q1/6834, ...  \n",
      "3       10X_GENOMICS_INC                                          C12N15/88  \n",
      "4       10X_GENOMICS_INC          [B01L3/00, B01L9/00, C12Q1/68, F16K99/00]  \n",
      "...                  ...                                                ...  \n",
      "667160  ZYMOGENETICS_INC                   [A61K38/00, C12N15/54, C12N9/10]  \n",
      "667161  ZYMOGENETICS_INC  [A61K31/00, A61K35/12, A61K38/00, A61K38/22, A...  \n",
      "667162  ZYMOGENETICS_INC                 [A61K48/00, C07K14/515, C12N15/12]  \n",
      "667163  ZYMOGENETICS_LLC       [A61K38/00, A61K38/20, A61K38/21, C07K14/00]  \n",
      "667164  ZYMOGENETICS_LLC  [A61K39/395, A61K45/00, A61P1/00, A61P11/06, A...  \n",
      "\n",
      "[667165 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "## Step 2: get all patents from JSON files\n",
    "full_data = []\n",
    "\n",
    "fp = \"Data/Patents/Patent_Assignee_Records/\"\n",
    "\n",
    "#Use filtered assignee list to narrow down files\n",
    "files = [f for f in os.listdir(fp) if f[:-5] in assignees]\n",
    "\n",
    "patent_data = []\n",
    "\n",
    "for f in tqdm(files):\n",
    "    try:\n",
    "        data = json.load(open(file=fp + f))\n",
    "    except ValueError:\n",
    "        data = [] \n",
    "\n",
    "    for patent in data:\n",
    "        if \"inventors\" in patent:\n",
    "            inventors = patent[\"inventors\"]\n",
    "            #Get all authors (in list form)\n",
    "            if type(inventors) != list:\n",
    "                authors = []\n",
    "                authors.append(inventors)\n",
    "            else:\n",
    "                authors = inventors\n",
    "        else:\n",
    "            authors = [\"\"]\n",
    "\n",
    "        if \"classification\" in patent:\n",
    "            classification = patent[\"classification\"]\n",
    "        else:\n",
    "            classification = \"\"\n",
    "\n",
    "        #Save patent data by assignee - one patent per assignee, which is obtained from the file name\n",
    "        try:\n",
    "            patent_data.append({\"ID\": patent[\"publicationnumber\"], \"author\": authors, \"assignees\": f[:-5],\n",
    "                \"classification\": classification})\n",
    "        except TypeError:\n",
    "            pass\n",
    "\n",
    "    \n",
    "df = pd.DataFrame.from_dict(patent_data)\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Data/Patents/patent_assignee_records.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 340940/340940 [00:09<00:00, 36930.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2048622\n",
      "['SCHEMBL2960146', 'SCHEMBL2819623', 'SCHEMBL1564641', 'SCHEMBL9194335', 'SCHEMBL1337775', 'SCHEMBL3560392', 'SCHEMBL10007518', 'SCHEMBL5850481', 'SCHEMBL16528158', 'SCHEMBL48933']\n"
     ]
    }
   ],
   "source": [
    "## Step 3: Get all compound IDs - filter down into a non-repeating list\n",
    "## NOTE: NEEDS TO BE RUN WITH PATENT_CPD_EDGES BUILDING BLOCK ABOVE\n",
    "\n",
    "cpds = []\n",
    "patents = df[\"ID\"].tolist()\n",
    "#selected_patents = [p for p in patent_cpd_edges.keys() if p in patents]\n",
    "    \n",
    "for p in tqdm(patents):\n",
    "    if p in patent_cpd_edges:\n",
    "        cpds.extend(patent_cpd_edges[p])\n",
    "\n",
    "cpds = list(set(cpds))\n",
    "\n",
    "print(len(cpds))\n",
    "print(cpds[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1751784\n",
      "['SCHEMBL2960146', 'SCHEMBL2819623', 'SCHEMBL1564641', 'SCHEMBL9194335', 'SCHEMBL1337775', 'SCHEMBL3560392', 'SCHEMBL10007518', 'SCHEMBL5850481', 'SCHEMBL16528158', 'SCHEMBL1941238']\n"
     ]
    }
   ],
   "source": [
    "## Step 4: filter cpds with patent author cpds, goal is to have ~1 million unique cpds (otherwise, sample!)\n",
    "author_cpds = pickle.load(file=open(\"Data/Patents/author_cpds_sampled.p\", \"rb\"))\n",
    "\n",
    "cpds = list(set(cpds) - set(author_cpds))\n",
    "\n",
    "print(len(cpds))\n",
    "print(cpds[0:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Save compounds\n",
    "pickle.dump(cpds, file=open(\"Data/Patents/assignee_cpds_sampled.p\", \"wb\"))\n",
    "\n",
    "## Save sampled patents too\n",
    "pickle.dump(patents, file=open(\"Data/Patents/assignee_patent_samples.p\", \"wb\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Link cpds with structures\n",
    "\n",
    "Will be done regardless of author/assignee origin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Load compounds (START HERE FOR ID -> STRUCTURES)\n",
    "\n",
    "cpds = pickle.load(file=open(\"Data/Patents/assignee_cpds_sampled.p\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "surechembl_allcpds = pickle.load(file=open(\"Data/Cpd_Data/SureChemBL_allCpds.p\", \"rb\"))\n",
    "print(type(surechembl_allcpds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpd_df = surechembl_allcpds[surechembl_allcpds[\"SureChEMBL_ID\"].isin(cpds)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(surechembl_allcpds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpd_df.to_csv(\"Data/Patents/assignee_cpds_structures.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inchi_to_mol(inchi, ID):\n",
    "    \"\"\" Translates a smiles string to a mol file and saves it in the appropriate location\n",
    "\n",
    "    Args:\n",
    "        smiles (str): smiles description of a molecule\n",
    "        index (int): identifying number of the molecule (unique to database)\n",
    "        database (str): name of database (corresponds to a directory in 'Data')\n",
    "    \"\"\"\n",
    "    try:\n",
    "        mol = Chem.MolFromInchi(inchi)\n",
    "        print(Chem.MolToMolBlock(mol),\n",
    "              file=open(\n",
    "                  \"Data/AssemblyValues/Patent_Authors/\" + ID + \".mol\",\n",
    "                  \"w+\"))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 535989/5963485 [5:54:14<8389:24:52,  5.56s/it] "
     ]
    }
   ],
   "source": [
    "## Get mol files\n",
    "tqdm.pandas()\n",
    "\n",
    "cpd_df.progress_apply(lambda x: inchi_to_mol(x[\"InChI\"], x[\"SureChEMBL_ID\"]), axis=1)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linking patent IDs to compounds: testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('US-4181664-A', ['SCHEMBL1812', 'SCHEMBL2454118', 'SCHEMBL11454132', 'SCHEMBL11470144', 'SCHEMBL11472968', 'SCHEMBL11455057', 'SCHEMBL1816', 'SCHEMBL11476626', 'SCHEMBL11472939', 'SCHEMBL11470688', 'SCHEMBL11462969', 'SCHEMBL11452721', 'SCHEMBL10498831', 'SCHEMBL10940552', 'SCHEMBL9458455', 'SCHEMBL1009808', 'SCHEMBL11483372', 'SCHEMBL11467835', 'SCHEMBL11454317', 'SCHEMBL120896', 'SCHEMBL1247', 'SCHEMBL11454025', 'SCHEMBL11454227', 'SCHEMBL11471395', 'SCHEMBL1967', 'SCHEMBL11476624', 'SCHEMBL10564539', 'SCHEMBL11471592', 'SCHEMBL9489027', 'SCHEMBL11481038', 'SCHEMBL11467833', 'SCHEMBL11475722', 'SCHEMBL2262', 'SCHEMBL11454052', 'SCHEMBL11462594', 'SCHEMBL11460770', 'SCHEMBL1798', 'SCHEMBL11085564', 'SCHEMBL25158', 'SCHEMBL11481104', 'SCHEMBL11453131', 'SCHEMBL11467831', 'SCHEMBL11470679', 'SCHEMBL11470687', 'SCHEMBL11454040', 'SCHEMBL9096638', 'SCHEMBL1586', 'SCHEMBL393249', 'SCHEMBL6966505', 'SCHEMBL9807176', 'SCHEMBL10699570', 'SCHEMBL11451896', 'SCHEMBL11462970', 'SCHEMBL11472941', 'SCHEMBL1499381', 'SCHEMBL11467452', 'SCHEMBL1259', 'SCHEMBL11468769', 'SCHEMBL6960516', 'SCHEMBL1044835', 'SCHEMBL11454049', 'SCHEMBL10434682', 'SCHEMBL6968975', 'SCHEMBL11469629', 'SCHEMBL11470146', 'SCHEMBL283639', 'SCHEMBL11453961', 'SCHEMBL11477212', 'SCHEMBL15458', 'SCHEMBL34', 'SCHEMBL34', 'SCHEMBL215763', 'SCHEMBL17836', 'SCHEMBL8169713', 'SCHEMBL11454130', 'SCHEMBL11480518', 'SCHEMBL11470669', 'SCHEMBL11467281', 'SCHEMBL7469279', 'SCHEMBL11451895', 'SCHEMBL11452720', 'SCHEMBL11471593', 'SCHEMBL1241', 'SCHEMBL1071609', 'SCHEMBL11465084', 'SCHEMBL11085562', 'SCHEMBL11471394', 'SCHEMBL11469530', 'SCHEMBL11477211', 'SCHEMBL11472945', 'SCHEMBL11470405', 'SCHEMBL1353', 'SCHEMBL641759', 'SCHEMBL11454051', 'SCHEMBL11468331', 'SCHEMBL573', 'SCHEMBL11471974', 'SCHEMBL573', 'SCHEMBL11454131', 'SCHEMBL11453962', 'SCHEMBL11480971', 'SCHEMBL11468313', 'SCHEMBL11480562', 'SCHEMBL11454024', 'SCHEMBL11454318', 'SCHEMBL11477063', 'SCHEMBL1296', 'SCHEMBL11462593', 'SCHEMBL11460771', 'SCHEMBL11453130', 'SCHEMBL7413', 'SCHEMBL11471613', 'SCHEMBL9705116', 'SCHEMBL10551521', 'SCHEMBL11477197', 'SCHEMBL1430822', 'SCHEMBL11468317', 'SCHEMBL1414', 'SCHEMBL2457986', 'SCHEMBL11454319', 'SCHEMBL11454252', 'SCHEMBL11454253', 'SCHEMBL29364']), ('US-4181734-A', ['SCHEMBL675157', 'SCHEMBL80292', 'SCHEMBL15111', 'SCHEMBL11467752', 'SCHEMBL675158', 'SCHEMBL10784476', 'SCHEMBL16226', 'SCHEMBL22722', 'SCHEMBL36992', 'SCHEMBL15134', 'SCHEMBL15112', 'SCHEMBL19157', 'SCHEMBL15110', 'SCHEMBL1829048', 'SCHEMBL105872', 'SCHEMBL21656', 'SCHEMBL743626', 'SCHEMBL11032756', 'SCHEMBL1296', 'SCHEMBL11210831', 'SCHEMBL20411', 'SCHEMBL18492', 'SCHEMBL3416'])]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nSummary: has a list of all patents in a month, associated with SureChemBL compounds\\n\\nThis works! May be a pain to search over every month, but it's possible...\\n\""
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Test 1: patent_cpd_edges\n",
    "\n",
    "patent_cpd_edges = pickle.load(file=open(\"Data/PubchemTesting/patent_cpd_edges_1980-01.p\", \"rb\"))\n",
    "\n",
    "print(list(patent_cpd_edges.items())[:2])\n",
    "\n",
    "\"\"\"\n",
    "Summary: has a list of all patents in a month, associated with SureChemBL compounds\n",
    "\n",
    "This works! May be a pain to search over every month, but it's possible...\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Testing 2: SureChemBL data files\n",
    "\n",
    "\"\"\"\n",
    "Summary: holds all cpd:structure pairs\n",
    "\n",
    "Useful for linking ids to structures, but nothing else right now...\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "[(21642937, 13500164), (21642937, 13496703), (21642937, 11697068), (21642937, 13000345), (21642937, 13008282), (21642937, 11213625), (21642937, 14644888), (21642937, 8925145), (21642937, 14299252), (21642937, 11867437)]\n",
      "{21642937: 11867437}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nSummary: I think this is the igraph format of the graph structure...using igraph's ids\\n\\nNot helpful, I'll have to link ids to patent/cpd ids anyway\\n\\n\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Testing 3: index_edgelist_bipartite.p\n",
    "\n",
    "index_edgelist = pickle.load(file=open(\"Data/CpdPatentIdsDates/index_edgelist_bipartite.p\", \"rb\"))\n",
    "print(type(index_edgelist))\n",
    "\n",
    "print(index_edgelist[0:10])\n",
    "\n",
    "test_list = index_edgelist[0:10]\n",
    "\n",
    "print(dict(test_list))\n",
    "\"\"\"\n",
    "Summary: I think this is the igraph format of the graph structure...using igraph's ids\n",
    "\n",
    "Not helpful, I'll have to link ids to patent/cpd ids anyway\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Unnamed: 0    month  newPatents  totalPatents\n",
      "0           635  1962-01           1             1\n",
      "1           634  1962-02           0             1\n",
      "2           633  1962-03           0             1\n",
      "3           632  1962-04           0             1\n",
      "4           631  1962-05           0             1\n",
      "..          ...      ...         ...           ...\n",
      "703         703  2020-08       17772       4718060\n",
      "704         702  2020-09       18737       4736797\n",
      "705         707  2020-10       22580       4759377\n",
      "706         706  2020-11       17575       4776952\n",
      "707         705  2020-12       22717       4799669\n",
      "\n",
      "[708 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "## Testing 4: patent_summary.csv\n",
    "patent_summary = pd.read_csv(\"Data/PubchemTesting/0-patent_summary.csv\")\n",
    "\n",
    "print(patent_summary)\n",
    "\n",
    "\"\"\"\n",
    "Summary: holds the numbers of patents (total & new) for each month\n",
    "\n",
    "Not useful for this, unfortunately\n",
    "\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('SCHEMBL11310284', 'US-4186208-A'), ('SCHEMBL7622', 'EP-0007112-A2')]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nSummary: holds all cpd:patent id pairs, showing where that particular compounds is (first?) found\\n\\nNot as useful as patent_cpd_edges\\n'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Testing 5: cpd_patent_edges\n",
    "\n",
    "cpd_patent_edges = pickle.load(file=open(\"Data/PubchemTesting/cpd_patent_edges_1980-01.p\", \"rb\"))\n",
    "\n",
    "print(cpd_patent_edges[:2])\n",
    "\n",
    "\"\"\"\n",
    "Summary: holds all cpd:patent id pairs, showing where that particular compounds is (first?) found\n",
    "\n",
    "Not as useful as patent_cpd_edges\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('EP-0269274-B1', 0), ('EP-1567545-A2', 1), ('EP-0835920-B1', 2), ('US-20050277593-A1', 3), ('EP-1937634-A4', 4), ('EP-1523328-A2', 5), ('EP-1474414-B1', 6), ('JP-H02174-A', 7), ('EP-1897683-B1', 8), ('US-3906010-A', 9)]\n"
     ]
    }
   ],
   "source": [
    "## Testing 6: patent_ID_index_dict\n",
    "\n",
    "patent_ID_index_dict = pickle.load(file=open(\"Data/patent_ID_index_dict.p\", \"rb\"))\n",
    "\n",
    "print(list(patent_ID_index_dict.items())[0:10])\n",
    "\n",
    "\n",
    "\"\"\" \n",
    "Not useful this is the iGraph IDs, which could potentially be useful, but still needs to loop through everything...\n",
    "\n",
    "Eh...wait...maybe if I can get a dictionary of all numbers associated with cpds from the master edgelist...\n",
    "\"\"\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                      Cpd    Month     Index\n",
      "0          SCHEMBL7356245  1962-01  17422145\n",
      "1          SCHEMBL7247395  1963-08    992275\n",
      "2          SCHEMBL7340057  1963-08   8302315\n",
      "3           SCHEMBL180193  1965-02   9088770\n",
      "4           SCHEMBL180194  1965-02   4821631\n",
      "...                   ...      ...       ...\n",
      "18845995  SCHEMBL21610019  2019-12  20096324\n",
      "18845996  SCHEMBL21609918  2019-12    563495\n",
      "18845997  SCHEMBL21609646  2019-12  13053968\n",
      "18845998  SCHEMBL21609627  2019-12  12059745\n",
      "18845999  SCHEMBL21610082  2019-12  12689868\n",
      "\n",
      "[18846000 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "cpd_df = pickle.load(file=open(\"Data/Cpd_Data/master_cpd_date_index_df.p\", \"rb\"))\n",
    "\n",
    "print(cpd_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1\n"
     ]
    }
   ],
   "source": [
    "print(min(cpd_df[\"Index\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my-rdkit-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a9f8f612ec1a1c2a2d1194d542dbb1244b288dc77a8f67f9e6b61794420b3e36"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}