processed_tsv_issue.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f51a4141",
   "metadata": {},
   "source": [
    "# Enrichment Analysis Notebook\n",
    "\n",
    "Compares the results of SPINDOCTOR gene set summarization vs statistical ontological enrichment.\n",
    "\n",
    "Draft: https://docs.google.com/document/d/1H103ux6Dd1_bPM0un4RwutBLcYJx-0ybil2AwlAvG_Q/edit#"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01aa8c37",
   "metadata": {},
   "source": [
    "## Initial setup\n",
    "\n",
    "Here we take care of imports, defining the data dictionary for the pandas dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fec960ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "from yaml import Loader\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import kstest, ttest_ind, wilcoxon, mannwhitneyu\n",
    "import math\n",
    "from statsmodels.stats.multitest import multipletests\n",
    "\n",
    "#import itertools as it\n",
    "#import collections as ct\n",
    "#import more_itertools as mit\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.patches import Rectangle\n",
    "import colorsys\n",
    "import seaborn as sns\n",
    "\n",
    "from oaklib import get_adapter\n",
    "from oaklib.datamodels.vocabulary import IS_A, PART_OF\n",
    "from ontogpt.evaluation.enrichment.eval_enrichment import EvalEnrichment\n",
    "go = get_adapter(\"sqlite:obo:go\")\n",
    "hgnc = get_adapter(\"sqlite:obo:hgnc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebcc3593",
   "metadata": {},
   "outputs": [],
   "source": [
    "TURBO = \"gpt-3.5-turbo\"\n",
    "DAVINCI = \"text-davinci-003\"\n",
    "GPT4 = \"gpt-4\"\n",
    "MODELS = [TURBO, DAVINCI, GPT4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "79068722",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>go_term_ids</th>\n",
       "      <th>name</th>\n",
       "      <th>cutoff</th>\n",
       "      <th>closure</th>\n",
       "      <th>top_n</th>\n",
       "      <th>source</th>\n",
       "      <th>model</th>\n",
       "      <th>method</th>\n",
       "      <th>method_desc</th>\n",
       "      <th>run</th>\n",
       "      <th>...</th>\n",
       "      <th>true_positive_terms</th>\n",
       "      <th>false_positive_terms</th>\n",
       "      <th>false_negative_terms_example20</th>\n",
       "      <th>unparsed_terms</th>\n",
       "      <th>gene_set_size</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>recall_general</th>\n",
       "      <th>recall_specific</th>\n",
       "      <th>f1_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4.0</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4.0</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096</td>\n",
       "      <td>GO:0006006</td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4.0</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096</td>\n",
       "      <td>GO:0006006</td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>25</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4.0</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096|GO:0006006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.148148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>5000</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4.0</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096|GO:0006006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.051282</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.097561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99355</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.000</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99356</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.000</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180</td>\n",
       "      <td>0.000794</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99357</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.000</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180</td>\n",
       "      <td>0.001720</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99358</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.000</td>\n",
       "      <td>True</td>\n",
       "      <td>25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>GO:0008525|GO:0019899|GO:0050709|GO:0035094|GO...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180</td>\n",
       "      <td>0.004761</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.009477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99359</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.000</td>\n",
       "      <td>True</td>\n",
       "      <td>5000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99360 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             go_term_ids  \\\n",
       "0      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "1      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "2      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "3      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "4      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "...                                                  ...   \n",
       "99355  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99356  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99357  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99358  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99359  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "\n",
       "                                 name  cutoff  closure  top_n source  model  \\\n",
       "0            glycolysis-gocam-0-0.005   0.005    False      1   NONE    4.0   \n",
       "1            glycolysis-gocam-0-0.005   0.005    False      5   NONE    4.0   \n",
       "2            glycolysis-gocam-0-0.005   0.005    False     10   NONE    4.0   \n",
       "3            glycolysis-gocam-0-0.005   0.005    False     25   NONE    4.0   \n",
       "4            glycolysis-gocam-0-0.005   0.005    False   5000   NONE    4.0   \n",
       "...                               ...     ...      ...    ...    ...    ...   \n",
       "99355  HALLMARK_MTORC1_SIGNALING-1-99  99.000     True      1    NaN    NaN   \n",
       "99356  HALLMARK_MTORC1_SIGNALING-1-99  99.000     True      5    NaN    NaN   \n",
       "99357  HALLMARK_MTORC1_SIGNALING-1-99  99.000     True     10    NaN    NaN   \n",
       "99358  HALLMARK_MTORC1_SIGNALING-1-99  99.000     True     25    NaN    NaN   \n",
       "99359  HALLMARK_MTORC1_SIGNALING-1-99  99.000     True   5000    NaN    NaN   \n",
       "\n",
       "      method    method_desc                   run  ...  \\\n",
       "0        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "1        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "2        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "3        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "4        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "...      ...            ...                   ...  ...   \n",
       "99355    NaN            NaN               closure  ...   \n",
       "99356    NaN            NaN               closure  ...   \n",
       "99357    NaN            NaN               closure  ...   \n",
       "99358    NaN            NaN               closure  ...   \n",
       "99359    NaN            NaN               closure  ...   \n",
       "\n",
       "                                     true_positive_terms  \\\n",
       "0                                             GO:0006096   \n",
       "1                                             GO:0006096   \n",
       "2                                             GO:0006096   \n",
       "3                                  GO:0006096|GO:0006006   \n",
       "4                                  GO:0006096|GO:0006006   \n",
       "...                                                  ...   \n",
       "99355                                         GO:0044283   \n",
       "99356  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "99357  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "99358  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "99359  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "\n",
       "                                    false_positive_terms  \\\n",
       "0                                                    NaN   \n",
       "1                                             GO:0006006   \n",
       "2                                             GO:0006006   \n",
       "3                                                    NaN   \n",
       "4                                                    NaN   \n",
       "...                                                  ...   \n",
       "99355                                                NaN   \n",
       "99356  GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...   \n",
       "99357  GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...   \n",
       "99358  GO:0008525|GO:0019899|GO:0050709|GO:0035094|GO...   \n",
       "99359                                                NaN   \n",
       "\n",
       "       false_negative_terms_example20                    unparsed_terms  \\\n",
       "0                                 NaN                               NaN   \n",
       "1                GO:0006090|GO:001605  energy production|atp generation   \n",
       "2                GO:0006090|GO:001605  energy production|atp generation   \n",
       "3                GO:0006090|GO:001605  energy production|atp generation   \n",
       "4                GO:0006090|GO:001605  energy production|atp generation   \n",
       "...                               ...                               ...   \n",
       "99355                             NaN                               NaN   \n",
       "99356                             NaN                               NaN   \n",
       "99357                             NaN                               NaN   \n",
       "99358                             NaN                               NaN   \n",
       "99359                             NaN                               NaN   \n",
       "\n",
       "       gene_set_size  precision    recall  recall_general  recall_specific  \\\n",
       "0                 10   1.000000  1.000000            1.00         1.000000   \n",
       "1                 10   0.500000  0.200000            1.00         0.200000   \n",
       "2                 10   0.500000  0.100000            0.20         0.200000   \n",
       "3                 10   1.000000  0.080000            0.25         0.153846   \n",
       "4                 10   1.000000  0.051282            0.25         0.142857   \n",
       "...              ...        ...       ...             ...              ...   \n",
       "99355            180   1.000000  1.000000            1.00         1.000000   \n",
       "99356            180   0.000794  1.000000            1.00         1.000000   \n",
       "99357            180   0.001720  1.000000            1.00         1.000000   \n",
       "99358            180   0.004761  1.000000            1.00         1.000000   \n",
       "99359            180   1.000000  1.000000            1.00         1.000000   \n",
       "\n",
       "       f1_score  \n",
       "0      1.000000  \n",
       "1      0.285714  \n",
       "2      0.166667  \n",
       "3      0.148148  \n",
       "4      0.097561  \n",
       "...         ...  \n",
       "99355  1.000000  \n",
       "99356  0.001587  \n",
       "99357  0.003434  \n",
       "99358  0.009477  \n",
       "99359  1.000000  \n",
       "\n",
       "[99360 rows x 31 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bad = pd.read_csv('results/processed.tsv', sep='\\t', header=0, low_memory=False)\n",
    "df_bad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "457306a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>go_term_ids</th>\n",
       "      <th>name</th>\n",
       "      <th>cutoff</th>\n",
       "      <th>closure</th>\n",
       "      <th>top_n</th>\n",
       "      <th>source</th>\n",
       "      <th>model</th>\n",
       "      <th>method</th>\n",
       "      <th>method_desc</th>\n",
       "      <th>run</th>\n",
       "      <th>...</th>\n",
       "      <th>true_positive_terms</th>\n",
       "      <th>false_positive_terms</th>\n",
       "      <th>false_negative_terms_example20</th>\n",
       "      <th>unparsed_terms</th>\n",
       "      <th>gene_set_size</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>recall_general</th>\n",
       "      <th>recall_specific</th>\n",
       "      <th>f1_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096</td>\n",
       "      <td>GO:0006006</td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2857142857142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096</td>\n",
       "      <td>GO:0006006</td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.16666666666666666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>25</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096|GO:0006006</td>\n",
       "      <td></td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.15384615384615385</td>\n",
       "      <td>0.14814814814814814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...</td>\n",
       "      <td>glycolysis-gocam-0-0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>False</td>\n",
       "      <td>5000</td>\n",
       "      <td>NONE</td>\n",
       "      <td>4</td>\n",
       "      <td>gpt</td>\n",
       "      <td>no_synopsis-4</td>\n",
       "      <td>gpt-4.no_synopsis.v1</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0006096|GO:0006006</td>\n",
       "      <td></td>\n",
       "      <td>GO:0006090|GO:001605</td>\n",
       "      <td>energy production|atp generation</td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.05128205128205128</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.14285714285714285</td>\n",
       "      <td>0.0975609756097561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99355</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>180</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99356</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>180</td>\n",
       "      <td>0.0007942811755361397</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0015873015873015873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99357</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>180</td>\n",
       "      <td>0.0017199862401100791</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.003434065934065934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99358</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>25</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td>GO:0008525|GO:0019899|GO:0050709|GO:0035094|GO...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>180</td>\n",
       "      <td>0.004760997905160921</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.009476876421531463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99359</th>\n",
       "      <td>{'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...</td>\n",
       "      <td>HALLMARK_MTORC1_SIGNALING-1-99</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>5000</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>closure</td>\n",
       "      <td>...</td>\n",
       "      <td>GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>180</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99360 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             go_term_ids  \\\n",
       "0      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "1      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "2      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "3      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "4      {'BFO:0000003', 'GO:0016052', 'GO:0006090', 'G...   \n",
       "...                                                  ...   \n",
       "99355  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99356  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99357  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99358  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "99359  {'GO:0006997', 'GO:0023051', 'GO:0048762', 'GO...   \n",
       "\n",
       "                                 name cutoff closure top_n source model  \\\n",
       "0            glycolysis-gocam-0-0.005  0.005   False     1   NONE     4   \n",
       "1            glycolysis-gocam-0-0.005  0.005   False     5   NONE     4   \n",
       "2            glycolysis-gocam-0-0.005  0.005   False    10   NONE     4   \n",
       "3            glycolysis-gocam-0-0.005  0.005   False    25   NONE     4   \n",
       "4            glycolysis-gocam-0-0.005  0.005   False  5000   NONE     4   \n",
       "...                               ...    ...     ...   ...    ...   ...   \n",
       "99355  HALLMARK_MTORC1_SIGNALING-1-99   99.0    True     1                \n",
       "99356  HALLMARK_MTORC1_SIGNALING-1-99   99.0    True     5                \n",
       "99357  HALLMARK_MTORC1_SIGNALING-1-99   99.0    True    10                \n",
       "99358  HALLMARK_MTORC1_SIGNALING-1-99   99.0    True    25                \n",
       "99359  HALLMARK_MTORC1_SIGNALING-1-99   99.0    True  5000                \n",
       "\n",
       "      method    method_desc                   run  ...  \\\n",
       "0        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "1        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "2        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "3        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "4        gpt  no_synopsis-4  gpt-4.no_synopsis.v1  ...   \n",
       "...      ...            ...                   ...  ...   \n",
       "99355                                     closure  ...   \n",
       "99356                                     closure  ...   \n",
       "99357                                     closure  ...   \n",
       "99358                                     closure  ...   \n",
       "99359                                     closure  ...   \n",
       "\n",
       "                                     true_positive_terms  \\\n",
       "0                                             GO:0006096   \n",
       "1                                             GO:0006096   \n",
       "2                                             GO:0006096   \n",
       "3                                  GO:0006096|GO:0006006   \n",
       "4                                  GO:0006096|GO:0006006   \n",
       "...                                                  ...   \n",
       "99355                                         GO:0044283   \n",
       "99356  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "99357  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "99358  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "99359  GO:0044283|GO:0005737|GO:0019752|GO:0036094|GO...   \n",
       "\n",
       "                                    false_positive_terms  \\\n",
       "0                                                          \n",
       "1                                             GO:0006006   \n",
       "2                                             GO:0006006   \n",
       "3                                                          \n",
       "4                                                          \n",
       "...                                                  ...   \n",
       "99355                                                      \n",
       "99356  GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...   \n",
       "99357  GO:0008525|GO:0009749|GO:0019899|GO:0050709|GO...   \n",
       "99358  GO:0008525|GO:0019899|GO:0050709|GO:0035094|GO...   \n",
       "99359                                                      \n",
       "\n",
       "      false_negative_terms_example20                    unparsed_terms  \\\n",
       "0                                                                        \n",
       "1               GO:0006090|GO:001605  energy production|atp generation   \n",
       "2               GO:0006090|GO:001605  energy production|atp generation   \n",
       "3               GO:0006090|GO:001605  energy production|atp generation   \n",
       "4               GO:0006090|GO:001605  energy production|atp generation   \n",
       "...                              ...                               ...   \n",
       "99355                                                                    \n",
       "99356                                                                    \n",
       "99357                                                                    \n",
       "99358                                                                    \n",
       "99359                                                                    \n",
       "\n",
       "      gene_set_size              precision               recall  \\\n",
       "0                10                    1.0                  1.0   \n",
       "1                10                    0.5                  0.2   \n",
       "2                10                    0.5                  0.1   \n",
       "3                10                    1.0                 0.08   \n",
       "4                10                    1.0  0.05128205128205128   \n",
       "...             ...                    ...                  ...   \n",
       "99355           180                    1.0                  1.0   \n",
       "99356           180  0.0007942811755361397                  1.0   \n",
       "99357           180  0.0017199862401100791                  1.0   \n",
       "99358           180   0.004760997905160921                  1.0   \n",
       "99359           180                    1.0                  1.0   \n",
       "\n",
       "      recall_general      recall_specific               f1_score  \n",
       "0                1.0                  1.0                    1.0  \n",
       "1                1.0                  0.2     0.2857142857142857  \n",
       "2                0.2                  0.2    0.16666666666666666  \n",
       "3               0.25  0.15384615384615385    0.14814814814814814  \n",
       "4               0.25  0.14285714285714285     0.0975609756097561  \n",
       "...              ...                  ...                    ...  \n",
       "99355            1.0                  1.0                    1.0  \n",
       "99356            1.0                  1.0  0.0015873015873015873  \n",
       "99357            1.0                  1.0   0.003434065934065934  \n",
       "99358            1.0                  1.0   0.009476876421531463  \n",
       "99359            1.0                  1.0                    1.0  \n",
       "\n",
       "[99360 rows x 31 columns]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "with open('results/processed.tsv', mode='r', encoding='utf-8') as file:\n",
    "    tsv_reader = csv.DictReader(file, delimiter='\\t')\n",
    "\n",
    "    data = [row for row in tsv_reader]\n",
    "\n",
    "df2 = pd.DataFrame(data)\n",
    "\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "e0eb2191",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Discrepancy found in line 1: CSV - , Pandas - nan\n",
      "Discrepancy found in line 3: CSV - 0.16666666666666666, Pandas - 0.1666666666666666\n",
      "Discrepancy found in line 4: CSV - , Pandas - nan\n",
      "Discrepancy found in line 5: CSV - , Pandas - nan\n",
      "Discrepancy found in line 6: CSV - , Pandas - nan\n",
      "Discrepancy found in line 7: CSV - , Pandas - nan\n",
      "Discrepancy found in line 8: CSV - , Pandas - nan\n",
      "Discrepancy found in line 9: CSV - , Pandas - nan\n",
      "Discrepancy found in line 10: CSV - , Pandas - nan\n",
      "Discrepancy found in line 11: CSV - , Pandas - nan\n",
      "Discrepancy found in line 13: CSV - 0.16666666666666666, Pandas - 0.1666666666666666\n",
      "Discrepancy found in line 14: CSV - , Pandas - nan\n",
      "Discrepancy found in line 15: CSV - , Pandas - nan\n",
      "Discrepancy found in line 16: CSV - , Pandas - nan\n",
      "Discrepancy found in line 17: CSV - , Pandas - nan\n",
      "Discrepancy found in line 18: CSV - , Pandas - nan\n",
      "Discrepancy found in line 19: CSV - , Pandas - nan\n",
      "Discrepancy found in line 20: CSV - , Pandas - nan\n",
      "Discrepancy found in line 21: CSV - , Pandas - nan\n",
      "Discrepancy found in line 23: CSV - 0.16666666666666666, Pandas - 0.1666666666666666\n",
      "Discrepancy found in line 24: CSV - , Pandas - nan\n",
      "Discrepancy found in line 25: CSV - , Pandas - nan\n",
      "Discrepancy found in line 26: CSV - , Pandas - nan\n",
      "Discrepancy found in line 27: CSV - , Pandas - nan\n",
      "Discrepancy found in line 28: CSV - , Pandas - nan\n",
      "Discrepancy found in line 29: CSV - , Pandas - nan\n",
      "Discrepancy found in line 30: CSV - , Pandas - nan\n",
      "Discrepancy found in line 31: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 32: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 33: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 34: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 35: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 36: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 37: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 38: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 39: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 40: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 41: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 42: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 43: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 44: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 45: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 46: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 47: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 48: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 49: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 50: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 51: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 52: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 53: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 54: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 55: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 56: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 57: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 58: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 59: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 60: CSV - 56, Pandas - 56.0\n",
      "Discrepancy found in line 61: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 62: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 63: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 64: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 65: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 66: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 67: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 68: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 69: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 70: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 71: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 72: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 73: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 74: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 75: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 76: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 77: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 78: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 79: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 80: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 81: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 82: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 83: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 84: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 85: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 86: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 87: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 88: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 89: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 90: CSV - 72, Pandas - 72.0\n",
      "Discrepancy found in line 91: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 92: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 93: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 94: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 95: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 96: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 97: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 98: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 99: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 100: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 101: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 102: CSV - 143, Pandas - 143.0\n",
      "Discrepancy found in line 103: CSV - 143, Pandas - 143.0\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import pandas as pd\n",
    "\n",
    "def compare_tsv(file_path):\n",
    "    df = pd.read_csv(file_path, sep='\\t')\n",
    "\n",
    "    discrepancies = []\n",
    "    line_number = 0\n",
    "\n",
    "    with open(file_path, 'r', newline='', encoding='utf-8') as file:\n",
    "        tsv_reader = csv.DictReader(file, delimiter='\\t')\n",
    "\n",
    "        for dict_row in tsv_reader:\n",
    "            line_number += 1\n",
    "\n",
    "            if line_number <= len(df):\n",
    "                pandas_row = df.iloc[line_number - 1].to_dict()\n",
    "                \n",
    "                for key in dict_row:\n",
    "                    if str(dict_row[key]) != str(pandas_row[key]):\n",
    "                        \n",
    "                        if dict_row[key] != \"4\" and dict_row[key] != \"67\":\n",
    "                        #    continue\n",
    "        \n",
    "                            #print(type(dict_row[key]))\n",
    "                            discrepancies.append(f\"Discrepancy found in line {line_number}: CSV - {dict_row[key]}, Pandas - {pandas_row[key]}\")\n",
    "                            break\n",
    "            else:\n",
    "                discrepancies.append(f\"Extra line in CSV at line {line_number}\")\n",
    "\n",
    "    if line_number < len(df):\n",
    "        discrepancies.append(f\"Pandas has extra rows from line {line_number + 1} onwards\")\n",
    "\n",
    "    return discrepancies\n",
    "\n",
    "file_path = 'results/processed.tsv'\n",
    "discrepancies = compare_tsv(file_path)\n",
    "countd = 0\n",
    "for discrepancy in discrepancies:\n",
    "    if countd < 100:\n",
    "        print(discrepancy)\n",
    "        countd = countd + 1\n",
    "    else:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4702da5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-ASCII character found in line 5343\n",
      "Non-ASCII character found in line 5344\n",
      "Non-ASCII character found in line 5345\n",
      "Non-ASCII character found in line 5346\n",
      "Non-ASCII character found in line 5348\n",
      "Non-ASCII character found in line 5349\n",
      "Non-ASCII character found in line 5350\n",
      "Non-ASCII character found in line 5351\n",
      "Non-ASCII character found in line 5353\n",
      "Non-ASCII character found in line 5354\n",
      "Non-ASCII character found in line 5355\n",
      "Non-ASCII character found in line 5356\n",
      "Non-ASCII character found in line 5358\n",
      "Non-ASCII character found in line 5359\n",
      "Non-ASCII character found in line 5360\n",
      "Non-ASCII character found in line 5361\n",
      "Non-ASCII character found in line 5363\n",
      "Non-ASCII character found in line 5364\n",
      "Non-ASCII character found in line 5365\n",
      "Non-ASCII character found in line 5366\n",
      "Non-ASCII character found in line 5368\n",
      "Non-ASCII character found in line 5369\n",
      "Non-ASCII character found in line 5370\n",
      "Non-ASCII character found in line 5371\n",
      "Non-ASCII character found in line 17043\n",
      "Non-ASCII character found in line 17044\n",
      "Non-ASCII character found in line 17045\n",
      "Non-ASCII character found in line 17046\n",
      "Non-ASCII character found in line 17048\n",
      "Non-ASCII character found in line 17049\n",
      "Non-ASCII character found in line 17050\n",
      "Non-ASCII character found in line 17051\n",
      "Non-ASCII character found in line 17053\n",
      "Non-ASCII character found in line 17054\n",
      "Non-ASCII character found in line 17055\n",
      "Non-ASCII character found in line 17056\n",
      "Non-ASCII character found in line 17058\n",
      "Non-ASCII character found in line 17059\n",
      "Non-ASCII character found in line 17060\n",
      "Non-ASCII character found in line 17061\n",
      "Non-ASCII character found in line 17063\n",
      "Non-ASCII character found in line 17064\n",
      "Non-ASCII character found in line 17065\n",
      "Non-ASCII character found in line 17066\n",
      "Non-ASCII character found in line 17068\n",
      "Non-ASCII character found in line 17069\n",
      "Non-ASCII character found in line 17070\n",
      "Non-ASCII character found in line 17071\n",
      "Non-ASCII character found in line 41913\n",
      "Non-ASCII character found in line 41914\n",
      "Non-ASCII character found in line 41915\n",
      "Non-ASCII character found in line 41916\n",
      "Non-ASCII character found in line 41918\n",
      "Non-ASCII character found in line 41919\n",
      "Non-ASCII character found in line 41920\n",
      "Non-ASCII character found in line 41921\n",
      "Non-ASCII character found in line 41923\n",
      "Non-ASCII character found in line 41924\n",
      "Non-ASCII character found in line 41925\n",
      "Non-ASCII character found in line 41926\n",
      "Non-ASCII character found in line 41928\n",
      "Non-ASCII character found in line 41929\n",
      "Non-ASCII character found in line 41930\n",
      "Non-ASCII character found in line 41931\n",
      "Non-ASCII character found in line 41933\n",
      "Non-ASCII character found in line 41934\n",
      "Non-ASCII character found in line 41935\n",
      "Non-ASCII character found in line 41936\n",
      "Non-ASCII character found in line 41938\n",
      "Non-ASCII character found in line 41939\n",
      "Non-ASCII character found in line 41940\n",
      "Non-ASCII character found in line 41941\n",
      "Non-ASCII character found in line 44583\n",
      "Non-ASCII character found in line 44584\n",
      "Non-ASCII character found in line 44585\n",
      "Non-ASCII character found in line 44586\n",
      "Non-ASCII character found in line 44588\n",
      "Non-ASCII character found in line 44589\n",
      "Non-ASCII character found in line 44590\n",
      "Non-ASCII character found in line 44591\n",
      "Non-ASCII character found in line 44593\n",
      "Non-ASCII character found in line 44594\n",
      "Non-ASCII character found in line 44595\n",
      "Non-ASCII character found in line 44596\n",
      "Non-ASCII character found in line 44598\n",
      "Non-ASCII character found in line 44599\n",
      "Non-ASCII character found in line 44600\n",
      "Non-ASCII character found in line 44601\n",
      "Non-ASCII character found in line 44603\n",
      "Non-ASCII character found in line 44604\n",
      "Non-ASCII character found in line 44605\n",
      "Non-ASCII character found in line 44606\n",
      "Non-ASCII character found in line 44608\n",
      "Non-ASCII character found in line 44609\n",
      "Non-ASCII character found in line 44610\n",
      "Non-ASCII character found in line 44611\n",
      "Non-ASCII character found in line 44673\n",
      "Non-ASCII character found in line 44674\n",
      "Non-ASCII character found in line 44675\n",
      "Non-ASCII character found in line 44676\n",
      "Non-ASCII character found in line 44678\n",
      "Non-ASCII character found in line 44679\n",
      "Non-ASCII character found in line 44680\n",
      "Non-ASCII character found in line 44681\n",
      "Non-ASCII character found in line 44683\n",
      "Non-ASCII character found in line 44684\n",
      "Non-ASCII character found in line 44685\n",
      "Non-ASCII character found in line 44686\n",
      "Non-ASCII character found in line 44688\n",
      "Non-ASCII character found in line 44689\n",
      "Non-ASCII character found in line 44690\n",
      "Non-ASCII character found in line 44691\n",
      "Non-ASCII character found in line 44693\n",
      "Non-ASCII character found in line 44694\n",
      "Non-ASCII character found in line 44695\n",
      "Non-ASCII character found in line 44696\n",
      "Non-ASCII character found in line 44698\n",
      "Non-ASCII character found in line 44699\n",
      "Non-ASCII character found in line 44700\n",
      "Non-ASCII character found in line 44701\n",
      "Non-ASCII character found in line 50823\n",
      "Non-ASCII character found in line 50824\n",
      "Non-ASCII character found in line 50825\n",
      "Non-ASCII character found in line 50826\n",
      "Non-ASCII character found in line 50828\n",
      "Non-ASCII character found in line 50829\n",
      "Non-ASCII character found in line 50830\n",
      "Non-ASCII character found in line 50831\n",
      "Non-ASCII character found in line 50833\n",
      "Non-ASCII character found in line 50834\n",
      "Non-ASCII character found in line 50835\n",
      "Non-ASCII character found in line 50836\n",
      "Non-ASCII character found in line 50838\n",
      "Non-ASCII character found in line 50839\n",
      "Non-ASCII character found in line 50840\n",
      "Non-ASCII character found in line 50841\n",
      "Non-ASCII character found in line 50843\n",
      "Non-ASCII character found in line 50844\n",
      "Non-ASCII character found in line 50845\n",
      "Non-ASCII character found in line 50846\n",
      "Non-ASCII character found in line 50848\n",
      "Non-ASCII character found in line 50849\n",
      "Non-ASCII character found in line 50850\n",
      "Non-ASCII character found in line 50851\n",
      "Non-ASCII character found in line 59493\n",
      "Non-ASCII character found in line 59494\n",
      "Non-ASCII character found in line 59495\n",
      "Non-ASCII character found in line 59496\n",
      "Non-ASCII character found in line 59498\n",
      "Non-ASCII character found in line 59499\n",
      "Non-ASCII character found in line 59500\n",
      "Non-ASCII character found in line 59501\n",
      "Non-ASCII character found in line 59503\n",
      "Non-ASCII character found in line 59504\n",
      "Non-ASCII character found in line 59505\n",
      "Non-ASCII character found in line 59506\n",
      "Non-ASCII character found in line 59508\n",
      "Non-ASCII character found in line 59509\n",
      "Non-ASCII character found in line 59510\n",
      "Non-ASCII character found in line 59511\n",
      "Non-ASCII character found in line 59513\n",
      "Non-ASCII character found in line 59514\n",
      "Non-ASCII character found in line 59515\n",
      "Non-ASCII character found in line 59516\n",
      "Non-ASCII character found in line 59518\n",
      "Non-ASCII character found in line 59519\n",
      "Non-ASCII character found in line 59520\n",
      "Non-ASCII character found in line 59521\n",
      "Non-ASCII character found in line 66603\n",
      "Non-ASCII character found in line 66604\n",
      "Non-ASCII character found in line 66605\n",
      "Non-ASCII character found in line 66606\n",
      "Non-ASCII character found in line 66608\n",
      "Non-ASCII character found in line 66609\n",
      "Non-ASCII character found in line 66610\n",
      "Non-ASCII character found in line 66611\n",
      "Non-ASCII character found in line 66613\n",
      "Non-ASCII character found in line 66614\n",
      "Non-ASCII character found in line 66615\n",
      "Non-ASCII character found in line 66616\n",
      "Non-ASCII character found in line 66618\n",
      "Non-ASCII character found in line 66619\n",
      "Non-ASCII character found in line 66620\n",
      "Non-ASCII character found in line 66621\n",
      "Non-ASCII character found in line 66623\n",
      "Non-ASCII character found in line 66624\n",
      "Non-ASCII character found in line 66625\n",
      "Non-ASCII character found in line 66626\n",
      "Non-ASCII character found in line 66628\n",
      "Non-ASCII character found in line 66629\n",
      "Non-ASCII character found in line 66630\n",
      "Non-ASCII character found in line 66631\n",
      "Non-ASCII character found in line 67202\n",
      "Non-ASCII character found in line 67203\n",
      "Non-ASCII character found in line 67204\n",
      "Non-ASCII character found in line 67205\n",
      "Non-ASCII character found in line 67206\n",
      "Non-ASCII character found in line 67207\n",
      "Non-ASCII character found in line 67208\n",
      "Non-ASCII character found in line 67209\n",
      "Non-ASCII character found in line 67210\n",
      "Non-ASCII character found in line 67211\n",
      "Non-ASCII character found in line 67212\n",
      "Non-ASCII character found in line 67213\n",
      "Non-ASCII character found in line 67214\n",
      "Non-ASCII character found in line 67215\n",
      "Non-ASCII character found in line 67216\n",
      "Non-ASCII character found in line 67217\n",
      "Non-ASCII character found in line 67218\n",
      "Non-ASCII character found in line 67219\n",
      "Non-ASCII character found in line 67220\n",
      "Non-ASCII character found in line 67221\n",
      "Non-ASCII character found in line 67222\n",
      "Non-ASCII character found in line 67223\n",
      "Non-ASCII character found in line 67224\n",
      "Non-ASCII character found in line 67225\n",
      "Non-ASCII character found in line 67226\n",
      "Non-ASCII character found in line 67227\n",
      "Non-ASCII character found in line 67228\n",
      "Non-ASCII character found in line 67229\n",
      "Non-ASCII character found in line 67230\n",
      "Non-ASCII character found in line 67231\n",
      "Non-ASCII character found in line 79713\n",
      "Non-ASCII character found in line 79714\n",
      "Non-ASCII character found in line 79715\n",
      "Non-ASCII character found in line 79716\n",
      "Non-ASCII character found in line 79718\n",
      "Non-ASCII character found in line 79719\n",
      "Non-ASCII character found in line 79720\n",
      "Non-ASCII character found in line 79721\n",
      "Non-ASCII character found in line 79723\n",
      "Non-ASCII character found in line 79724\n",
      "Non-ASCII character found in line 79725\n",
      "Non-ASCII character found in line 79726\n",
      "Non-ASCII character found in line 79728\n",
      "Non-ASCII character found in line 79729\n",
      "Non-ASCII character found in line 79730\n",
      "Non-ASCII character found in line 79731\n",
      "Non-ASCII character found in line 79733\n",
      "Non-ASCII character found in line 79734\n",
      "Non-ASCII character found in line 79735\n",
      "Non-ASCII character found in line 79736\n",
      "Non-ASCII character found in line 79738\n",
      "Non-ASCII character found in line 79739\n",
      "Non-ASCII character found in line 79740\n",
      "Non-ASCII character found in line 79741\n",
      "Non-ASCII character found in line 85383\n",
      "Non-ASCII character found in line 85384\n",
      "Non-ASCII character found in line 85385\n",
      "Non-ASCII character found in line 85386\n",
      "Non-ASCII character found in line 85388\n",
      "Non-ASCII character found in line 85389\n",
      "Non-ASCII character found in line 85390\n",
      "Non-ASCII character found in line 85391\n",
      "Non-ASCII character found in line 85393\n",
      "Non-ASCII character found in line 85394\n",
      "Non-ASCII character found in line 85395\n",
      "Non-ASCII character found in line 85396\n",
      "Non-ASCII character found in line 85398\n",
      "Non-ASCII character found in line 85399\n",
      "Non-ASCII character found in line 85400\n",
      "Non-ASCII character found in line 85401\n",
      "Non-ASCII character found in line 85403\n",
      "Non-ASCII character found in line 85404\n",
      "Non-ASCII character found in line 85405\n",
      "Non-ASCII character found in line 85406\n",
      "Non-ASCII character found in line 85408\n",
      "Non-ASCII character found in line 85409\n",
      "Non-ASCII character found in line 85410\n",
      "Non-ASCII character found in line 85411\n",
      "Non-ASCII character found in line 88113\n",
      "Non-ASCII character found in line 88114\n",
      "Non-ASCII character found in line 88115\n",
      "Non-ASCII character found in line 88116\n",
      "Non-ASCII character found in line 88118\n",
      "Non-ASCII character found in line 88119\n",
      "Non-ASCII character found in line 88120\n",
      "Non-ASCII character found in line 88121\n",
      "Non-ASCII character found in line 88123\n",
      "Non-ASCII character found in line 88124\n",
      "Non-ASCII character found in line 88125\n",
      "Non-ASCII character found in line 88126\n",
      "Non-ASCII character found in line 88128\n",
      "Non-ASCII character found in line 88129\n",
      "Non-ASCII character found in line 88130\n",
      "Non-ASCII character found in line 88131\n",
      "Non-ASCII character found in line 88133\n",
      "Non-ASCII character found in line 88134\n",
      "Non-ASCII character found in line 88135\n",
      "Non-ASCII character found in line 88136\n",
      "Non-ASCII character found in line 88138\n",
      "Non-ASCII character found in line 88139\n",
      "Non-ASCII character found in line 88140\n",
      "Non-ASCII character found in line 88141\n",
      "Non-ASCII character found in line 96423\n",
      "Non-ASCII character found in line 96424\n",
      "Non-ASCII character found in line 96425\n",
      "Non-ASCII character found in line 96426\n",
      "Non-ASCII character found in line 96428\n",
      "Non-ASCII character found in line 96429\n",
      "Non-ASCII character found in line 96430\n",
      "Non-ASCII character found in line 96431\n",
      "Non-ASCII character found in line 96433\n",
      "Non-ASCII character found in line 96434\n",
      "Non-ASCII character found in line 96435\n",
      "Non-ASCII character found in line 96436\n",
      "Non-ASCII character found in line 96438\n",
      "Non-ASCII character found in line 96439\n",
      "Non-ASCII character found in line 96440\n",
      "Non-ASCII character found in line 96441\n",
      "Non-ASCII character found in line 96443\n",
      "Non-ASCII character found in line 96444\n",
      "Non-ASCII character found in line 96445\n",
      "Non-ASCII character found in line 96446\n",
      "Non-ASCII character found in line 96448\n",
      "Non-ASCII character found in line 96449\n",
      "Non-ASCII character found in line 96450\n",
      "Non-ASCII character found in line 96451\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "file_path = 'results/processed.tsv'\n",
    "\n",
    "\n",
    "standard_characters = set('\\t\\n\\rabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,;:\"\\'!?-()[]{}')\n",
    "\n",
    "\n",
    "def detect_problems(file_path):\n",
    "    problems = []\n",
    "    expected_delimiter_count = None\n",
    "\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line_number, line in enumerate(file, 1):\n",
    "            # Count the number of tab characters\n",
    "            delimiter_count = line.count('\\t')\n",
    "\n",
    "            if expected_delimiter_count is None:\n",
    "                expected_delimiter_count = delimiter_count\n",
    "            elif delimiter_count != expected_delimiter_count:\n",
    "                problems.append(f\"Inconsistent number of tabs in line {line_number}\")\n",
    "\n",
    "            # Check for non-ASCII characters\n",
    "            if any(ord(char) > 127 for char in line):\n",
    "                problems.append(f\"Non-ASCII character found in line {line_number}\")\n",
    "            \n",
    "            #for char in line:    \n",
    "            #    if char not in standard_characters:\n",
    "            #        problems.append(f\"Non-standard character '{char}' found in line {line_number}\")\n",
    "            #        break\n",
    "\n",
    "    return problems\n",
    "\n",
    "#Use\n",
    "file_problems = detect_problems(file_path)\n",
    "for problem in file_problems:\n",
    "    print(problem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b1a8d79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'false_negative_terms_example20'"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bad.columns[23]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdf1cc25",
   "metadata": {},
   "outputs": [
    {
     "ename": "ParserError",
     "evalue": "Error tokenizing data. C error: Expected 23 fields in line 302, saw 31\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mParserError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[104], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mresults/processed_1234567891011121314151617181920212223.tsv\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;130;43;01m\\t\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlow_memory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m df\n",
      "File \u001b[0;32m~/Documents/VIMSS/ontology/LLMs/talisman-paper/venv/lib/python3.10/site-packages/pandas/io/parsers/readers.py:948\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m    935\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m    936\u001b[0m     dialect,\n\u001b[1;32m    937\u001b[0m     delimiter,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    944\u001b[0m     dtype_backend\u001b[38;5;241m=\u001b[39mdtype_backend,\n\u001b[1;32m    945\u001b[0m )\n\u001b[1;32m    946\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 948\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Documents/VIMSS/ontology/LLMs/talisman-paper/venv/lib/python3.10/site-packages/pandas/io/parsers/readers.py:617\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    614\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m parser\n\u001b[1;32m    616\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m parser:\n\u001b[0;32m--> 617\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Documents/VIMSS/ontology/LLMs/talisman-paper/venv/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1748\u001b[0m, in \u001b[0;36mTextFileReader.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1741\u001b[0m nrows \u001b[38;5;241m=\u001b[39m validate_integer(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnrows\u001b[39m\u001b[38;5;124m\"\u001b[39m, nrows)\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1743\u001b[0m     \u001b[38;5;66;03m# error: \"ParserBase\" has no attribute \"read\"\u001b[39;00m\n\u001b[1;32m   1744\u001b[0m     (\n\u001b[1;32m   1745\u001b[0m         index,\n\u001b[1;32m   1746\u001b[0m         columns,\n\u001b[1;32m   1747\u001b[0m         col_dict,\n\u001b[0;32m-> 1748\u001b[0m     ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[attr-defined]\u001b[39;49;00m\n\u001b[1;32m   1749\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnrows\u001b[49m\n\u001b[1;32m   1750\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1751\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m   1752\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/Documents/VIMSS/ontology/LLMs/talisman-paper/venv/lib/python3.10/site-packages/pandas/io/parsers/c_parser_wrapper.py:239\u001b[0m, in \u001b[0;36mCParserWrapper.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m    236\u001b[0m         data \u001b[38;5;241m=\u001b[39m _concatenate_chunks(chunks)\n\u001b[1;32m    238\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 239\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    240\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m    241\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_first_chunk:\n",
      "File \u001b[0;32mparsers.pyx:825\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mparsers.pyx:913\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mparsers.pyx:890\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._check_tokenize_status\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mparsers.pyx:2058\u001b[0m, in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mParserError\u001b[0m: Error tokenizing data. C error: Expected 23 fields in line 302, saw 31\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('results/processed_1234567891011121314151617181920212223.tsv', sep='\\t', header=0, low_memory=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b881ca-c5a1-4e39-85ab-d3c5ba53b8d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('results/processed.tsv', sep='\\t', header=0, low_memory=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a9308c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.iloc[45561,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0079859",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.tail(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70d710c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_non_ascii(input_file_path, output_file_path):\n",
    "    with open(input_file_path, 'r', encoding='utf-8') as file, \\\n",
    "         open(output_file_path, 'w', encoding='utf-8') as output_file:\n",
    "        for line in file:\n",
    "            modified_line = ''.join(char if ord(char) < 128 else '*' for char in line)\n",
    "            output_file.write(modified_line)\n",
    "\n",
    "# Specify your file paths\n",
    "input_file_path = 'results/processed.tsv'\n",
    "output_file_path = 'results/processed_clean.tsv'\n",
    "\n",
    "# Replace non-ASCII characters and write to a new file\n",
    "replace_non_ascii(input_file_path, output_file_path)\n",
    "\n",
    "# Load the modified file into a DataFrame\n",
    "df = pd.read_csv(output_file_path, sep='\\t')\n",
    "\n",
    "# Display the DataFrame (or perform further operations)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fa7ef20",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def process_line(line, max_field_length, list_delimiter):\n",
    "    # Replace non-ASCII characters with '*'\n",
    "    cleaned_line = ''.join(char if ord(char) < 128 else '*' for char in line)\n",
    "\n",
    "    # Truncate long fields\n",
    "    fields = cleaned_line.split('\\t')\n",
    "    truncated_fields = []\n",
    "    for field in fields:\n",
    "        if len(field) > max_field_length:\n",
    "            # Find the nearest list delimiter before the max length\n",
    "            truncation_index = field.rfind(list_delimiter, 0, max_field_length)\n",
    "            if truncation_index == -1:\n",
    "                # If no list delimiter found, truncate to max length\n",
    "                truncation_index = max_field_length\n",
    "            truncated_field = field[:truncation_index] + '}'\n",
    "            truncated_fields.append(truncated_field)\n",
    "        else:\n",
    "            truncated_fields.append(field)\n",
    "    \n",
    "    return '\\t'.join(truncated_fields)\n",
    "\n",
    "def parse_and_load_tsv(input_file_path, output_file_path, max_field_length=1000, list_delimiter=','):\n",
    "    with open(input_file_path, 'r', encoding='utf-8', newline='') as file, \\\n",
    "         open(output_file_path, 'w', encoding='utf-8', newline='') as output_file:\n",
    "\n",
    "        line_number = 0\n",
    "        for line in file:\n",
    "            line_number += 1\n",
    "\n",
    "            # Handle empty lines\n",
    "            if not line.strip():\n",
    "                output_file.write('\\n')\n",
    "                continue\n",
    "\n",
    "            try:\n",
    "                processed_line = process_line(line, max_field_length, list_delimiter)\n",
    "                output_file.write(processed_line + '\\n')\n",
    "            except Exception as e:\n",
    "                print(f\"Error processing line {line_number}: {e}\")\n",
    "\n",
    "            \n",
    "# Specify your file paths\n",
    "input_file_path = 'results/processed.tsv'\n",
    "output_file_path = 'results/processed_cleaned.tsv'\n",
    "\n",
    "# Process the input file and write the modified data to a new file\n",
    "parse_and_load_tsv(input_file_path, output_file_path)\n",
    "\n",
    "# Load the modified file into a DataFrame\n",
    "df = pd.read_csv(output_file_path, sep='\\t', lineterminator='\\n')\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e27c817f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}