parse_sst_html.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook to parse html files to produce cleaned text of SST decisions\n",
    "\n",
    "Author: Sean Rehaag\n",
    "\n",
    "License: Creative Commons Attribution-NonCommercial 4.0 International [(CC BY-NC 4.0)](https://creativecommons.org/licenses/by-nc/4.0/). \n",
    "\n",
    "Dataset & Code to be cited as: \n",
    "\n",
    "    Sean Rehaag, \"Social Security Tribunal Bulk Decisions Dataset\" (2023), online: Refugee Law Laboratory <https://refugeelab.ca/bulk-data/sst> (updated 2024).\n",
    "\n",
    "To load data, see [load_and_analyze_sst_cases.ipynb](https://github.com/Refugee-Law-Lab/sst_bulk_data/blob/master/load_and_analyze_sst_cases.ipynb)\n",
    "\n",
    "### Notes:\n",
    "\n",
    "Notes:\n",
    "\n",
    "(1) Data Source: [Social Security Tribunal of Canada](https://www.sst-tss.gc.ca).\n",
    "\n",
    "(2) Unofficial Data: The data are unofficial reproductions of materials on the Social Security Tribunal website. Links to official versions are included in the dataset.\n",
    "\n",
    "(3) Non-Affiliation / Endorsement: The data has been collected and reproduced without any affiliation or endorsement from the Social Security Tribunal of Canada.\n",
    "\n",
    "(4) Non-Commerical Use: As indicated in the license, data may be used for non-commercial use (with attribution) only. For commercial use, see the Social Security Tribunal of Canada website's [Terms of Use](https://www.sst-tss.gc.ca/en/terms-and-conditions).\n",
    "\n",
    "(5) Accuracy: Data was collected and processed programmatically for the purposes of academic research. While we make best efforts to ensure accuracy, data gathering of this kind inevitably involves errors. As such the data should be viewed as preliminary information aimed to prompt further research and discussion, rather than as definitive information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import re\n",
    "import pathlib\n",
    "import json\n",
    "\n",
    "# set up progress bar\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "# set paths\n",
    "in_path = pathlib.Path('d:/scraping/DATA/DECISIONS/SST/BULK/HTML/')\n",
    "out_path_raw = pathlib.Path('DATA/sst_raw.jsonl')\n",
    "out_path_parsed = pathlib.Path('DATA/sst_cases.jsonl')\n",
    "out_path_parquet = pathlib.Path('DATA/sst_cases.parquet')\n",
    "out_path_yearly = pathlib.Path('DATA/YEARLY/')\n",
    "out_path_HF = pathlib.Path('d:/AI-Projects/canadian-legal-data/SST/train.parquet')\n",
    "\n",
    "# set years sought\n",
    "start_year = 2001 \n",
    "end_year = 2024\n",
    "\n",
    "# set language sought\n",
    "language_sought = None  #set to 'en' if english only, set to 'fr' if French only, set to None if all languages\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Raw Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29173/29173 [00:49<00:00, 589.04it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case_citation</th>\n",
       "      <th>case_citation2</th>\n",
       "      <th>case_year</th>\n",
       "      <th>case_name</th>\n",
       "      <th>case_language</th>\n",
       "      <th>case_decision_date</th>\n",
       "      <th>scraped_link</th>\n",
       "      <th>scraped_timestamp</th>\n",
       "      <th>scraped_status_code</th>\n",
       "      <th>referrer_main_source</th>\n",
       "      <th>referrer_sub_source</th>\n",
       "      <th>referrer_file</th>\n",
       "      <th>referrer_timestamp</th>\n",
       "      <th>case_html</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013 SSTAD 1</td>\n",
       "      <td></td>\n",
       "      <td>2013</td>\n",
       "      <td>K.O. v. Minister of Human Resources and Skills...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-05-14</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26T07:53:08.592343</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>2023-07-24T17:29:23.497800</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"en\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013 SSTAD 10</td>\n",
       "      <td></td>\n",
       "      <td>2013</td>\n",
       "      <td>S. H. v. Minister of Human Resources and Skill...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-11-27</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26T07:52:25.832988</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>2023-07-24T17:29:23.497800</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"en\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013 SSTAD 11</td>\n",
       "      <td></td>\n",
       "      <td>2013</td>\n",
       "      <td>M. H. v. Minister of Human Resources and Skill...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-11-27</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26T07:52:20.338058</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>2023-07-24T17:29:23.497800</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"en\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013 SSTAD 12</td>\n",
       "      <td></td>\n",
       "      <td>2013</td>\n",
       "      <td>L. L. and Minister of Human Resources and Skil...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cpp-rp...</td>\n",
       "      <td>2023-07-27T08:33:30.119367</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_batch_other_cpp_jul23.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_batch_other_cpp_jul23.j...</td>\n",
       "      <td>2023-07-24T18:04:02.219239</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"en\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013 SSTAD 13</td>\n",
       "      <td></td>\n",
       "      <td>2013</td>\n",
       "      <td>R. S. v. Minister of Human Resources and Skill...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-12-17</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26T07:52:14.077767</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl</td>\n",
       "      <td>2023-07-24T17:29:23.497800</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"en\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29168</th>\n",
       "      <td>2024 TSS 94</td>\n",
       "      <td></td>\n",
       "      <td>2024</td>\n",
       "      <td>RO c Ministre de l’Emploi et du Développement ...</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-30</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2024-07-14T11:41:16.264817</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_cpp_update_jul24.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_cpp_update_jul24.jsonl</td>\n",
       "      <td>2024-07-14T08:24:10.963757</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"fr\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29169</th>\n",
       "      <td>2024 TSS 95</td>\n",
       "      <td></td>\n",
       "      <td>2024</td>\n",
       "      <td>TK c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-29</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14T07:35:15.783910</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_ei_update_jul24.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl</td>\n",
       "      <td>2024-07-13T21:51:32.856375</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"fr\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29170</th>\n",
       "      <td>2024 TSS 96</td>\n",
       "      <td></td>\n",
       "      <td>2024</td>\n",
       "      <td>IR c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14T07:31:28.988813</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_ei_update_jul24.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl</td>\n",
       "      <td>2024-07-13T21:51:32.856375</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"fr\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29171</th>\n",
       "      <td>2024 TSS 98</td>\n",
       "      <td></td>\n",
       "      <td>2024</td>\n",
       "      <td>JC c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14T07:31:35.472880</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_ei_update_jul24.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl</td>\n",
       "      <td>2024-07-13T21:51:32.856375</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"fr\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29172</th>\n",
       "      <td>2024 TSS 99</td>\n",
       "      <td></td>\n",
       "      <td>2024</td>\n",
       "      <td>DF c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-30</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14T07:32:27.881584</td>\n",
       "      <td>200</td>\n",
       "      <td>SR-Local</td>\n",
       "      <td>sst_ei_update_jul24.jsonl</td>\n",
       "      <td>C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl</td>\n",
       "      <td>2024-07-13T21:51:32.856375</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;\\n&lt;html lang=\"fr\"&gt;\\n\\n&lt;head&gt;\\n ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29173 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       case_citation case_citation2 case_year   \n",
       "0       2013 SSTAD 1                     2013  \\\n",
       "1      2013 SSTAD 10                     2013   \n",
       "2      2013 SSTAD 11                     2013   \n",
       "3      2013 SSTAD 12                     2013   \n",
       "4      2013 SSTAD 13                     2013   \n",
       "...              ...            ...       ...   \n",
       "29168    2024 TSS 94                     2024   \n",
       "29169    2024 TSS 95                     2024   \n",
       "29170    2024 TSS 96                     2024   \n",
       "29171    2024 TSS 98                     2024   \n",
       "29172    2024 TSS 99                     2024   \n",
       "\n",
       "                                               case_name case_language   \n",
       "0      K.O. v. Minister of Human Resources and Skills...            en  \\\n",
       "1      S. H. v. Minister of Human Resources and Skill...            en   \n",
       "2      M. H. v. Minister of Human Resources and Skill...            en   \n",
       "3      L. L. and Minister of Human Resources and Skil...            en   \n",
       "4      R. S. v. Minister of Human Resources and Skill...            en   \n",
       "...                                                  ...           ...   \n",
       "29168  RO c Ministre de l’Emploi et du Développement ...            fr   \n",
       "29169    TK c Commission de l’assurance-emploi du Canada            fr   \n",
       "29170    IR c Commission de l’assurance-emploi du Canada            fr   \n",
       "29171    JC c Commission de l’assurance-emploi du Canada            fr   \n",
       "29172    DF c Commission de l’assurance-emploi du Canada            fr   \n",
       "\n",
       "      case_decision_date                                       scraped_link   \n",
       "0             2013-05-14  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...  \\\n",
       "1             2013-11-27  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...   \n",
       "2             2013-11-27  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...   \n",
       "3             2013-12-31  https://decisions.sst-tss.gc.ca/sst-tss/cpp-rp...   \n",
       "4             2013-12-17  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...   \n",
       "...                  ...                                                ...   \n",
       "29168         2024-01-30  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...   \n",
       "29169         2024-01-29  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "29170         2024-01-31  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "29171         2024-01-31  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "29172         2024-01-30  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "\n",
       "                scraped_timestamp  scraped_status_code referrer_main_source   \n",
       "0      2023-07-26T07:53:08.592343                  200             SR-Local  \\\n",
       "1      2023-07-26T07:52:25.832988                  200             SR-Local   \n",
       "2      2023-07-26T07:52:20.338058                  200             SR-Local   \n",
       "3      2023-07-27T08:33:30.119367                  200             SR-Local   \n",
       "4      2023-07-26T07:52:14.077767                  200             SR-Local   \n",
       "...                           ...                  ...                  ...   \n",
       "29168  2024-07-14T11:41:16.264817                  200             SR-Local   \n",
       "29169  2024-07-14T07:35:15.783910                  200             SR-Local   \n",
       "29170  2024-07-14T07:31:28.988813                  200             SR-Local   \n",
       "29171  2024-07-14T07:31:35.472880                  200             SR-Local   \n",
       "29172  2024-07-14T07:32:27.881584                  200             SR-Local   \n",
       "\n",
       "                   referrer_sub_source   \n",
       "0            sst_batch_cpp_jul23.jsonl  \\\n",
       "1            sst_batch_cpp_jul23.jsonl   \n",
       "2            sst_batch_cpp_jul23.jsonl   \n",
       "3      sst_batch_other_cpp_jul23.jsonl   \n",
       "4            sst_batch_cpp_jul23.jsonl   \n",
       "...                                ...   \n",
       "29168       sst_cpp_update_jul24.jsonl   \n",
       "29169        sst_ei_update_jul24.jsonl   \n",
       "29170        sst_ei_update_jul24.jsonl   \n",
       "29171        sst_ei_update_jul24.jsonl   \n",
       "29172        sst_ei_update_jul24.jsonl   \n",
       "\n",
       "                                           referrer_file   \n",
       "0           C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl  \\\n",
       "1           C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl   \n",
       "2           C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl   \n",
       "3      C:/Scraping/QUEUES/sst_batch_other_cpp_jul23.j...   \n",
       "4           C:/Scraping/QUEUES/sst_batch_cpp_jul23.jsonl   \n",
       "...                                                  ...   \n",
       "29168      C:/Scraping/QUEUES/sst_cpp_update_jul24.jsonl   \n",
       "29169       C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl   \n",
       "29170       C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl   \n",
       "29171       C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl   \n",
       "29172       C:/Scraping/QUEUES/sst_ei_update_jul24.jsonl   \n",
       "\n",
       "               referrer_timestamp   \n",
       "0      2023-07-24T17:29:23.497800  \\\n",
       "1      2023-07-24T17:29:23.497800   \n",
       "2      2023-07-24T17:29:23.497800   \n",
       "3      2023-07-24T18:04:02.219239   \n",
       "4      2023-07-24T17:29:23.497800   \n",
       "...                           ...   \n",
       "29168  2024-07-14T08:24:10.963757   \n",
       "29169  2024-07-13T21:51:32.856375   \n",
       "29170  2024-07-13T21:51:32.856375   \n",
       "29171  2024-07-13T21:51:32.856375   \n",
       "29172  2024-07-13T21:51:32.856375   \n",
       "\n",
       "                                               case_html  \n",
       "0      <!DOCTYPE html>\\n<html lang=\"en\">\\n\\n<head>\\n ...  \n",
       "1      <!DOCTYPE html>\\n<html lang=\"en\">\\n\\n<head>\\n ...  \n",
       "2      <!DOCTYPE html>\\n<html lang=\"en\">\\n\\n<head>\\n ...  \n",
       "3      <!DOCTYPE html>\\n<html lang=\"en\">\\n\\n<head>\\n ...  \n",
       "4      <!DOCTYPE html>\\n<html lang=\"en\">\\n\\n<head>\\n ...  \n",
       "...                                                  ...  \n",
       "29168  <!DOCTYPE html>\\n<html lang=\"fr\">\\n\\n<head>\\n ...  \n",
       "29169  <!DOCTYPE html>\\n<html lang=\"fr\">\\n\\n<head>\\n ...  \n",
       "29170  <!DOCTYPE html>\\n<html lang=\"fr\">\\n\\n<head>\\n ...  \n",
       "29171  <!DOCTYPE html>\\n<html lang=\"fr\">\\n\\n<head>\\n ...  \n",
       "29172  <!DOCTYPE html>\\n<html lang=\"fr\">\\n\\n<head>\\n ...  \n",
       "\n",
       "[29173 rows x 14 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get list of files (including subdirectories) using pathlib\n",
    "files = list(in_path.glob('**/*.json'))\n",
    "print(len(files))\n",
    "\n",
    "# Load data from files\n",
    "results = []\n",
    "for file in tqdm(files):\n",
    "    with open(file) as f:\n",
    "        data = json.load(f)\n",
    "        results.append(data)\n",
    "\n",
    "# convert list of dictionaries to dataframe\n",
    "df = pd.DataFrame(results)\n",
    "\n",
    "results = []\n",
    "\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export raw df to jsonl\n",
    "df.to_json(out_path_raw, orient='records', lines=True)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean dataframe\n",
    "\n",
    "#remove ?iframe=true from scraped_link\n",
    "df['scraped_link'] = df['scraped_link'].str.replace('\\?iframe=true', '', regex=True)\n",
    "\n",
    "# remove T and everything after from scraped_timestamp & case_decision_date\n",
    "df['scraped_timestamp'] = df['scraped_timestamp'].str.replace('T.*', '', regex=True)\n",
    "df['case_decision_date'] = df['case_decision_date'].str.replace('T.*', '', regex=True)\n",
    "\n",
    "# remove scraped_status_code, referrer_main_source, referrer_sub_source, referrer_file, referrer_timestamp\n",
    "df = df.drop(columns=['scraped_status_code', 'referrer_main_source', 'referrer_sub_source', 'referrer_file', 'referrer_timestamp'])\n",
    "\n",
    "# convert case_year to int and filter for years sought\n",
    "df['case_year'] = df['case_year'].astype(int)\n",
    "df = df[df.case_year >= start_year]\n",
    "df = df[df.case_year <= end_year]\n",
    "\n",
    "# filter for langauge if desired\n",
    "if language_sought:\n",
    "    df = df[df.case_language == language_sought]\n",
    "\n",
    "# remove cases where no citation (typically orders or errors)\n",
    "# remove df where citation includes '=' or '-'\n",
    "df = df[df.case_citation.str.contains('=') == False]\n",
    "df = df[df.case_citation.str.contains('-') == False]\n",
    "\n",
    "# if citation2 = ciation 1, make citation2 = ''\n",
    "df['case_citation2'] = df['case_citation2'].where(df['case_citation2'] != df['case_citation'], '')\n",
    "\n",
    "# change 'scraped_link' column to 'source_url;\n",
    "df = df.rename(columns={'scraped_link': 'source_url'})\n",
    "\n",
    "# remove 'case_' from all column names\n",
    "df.columns = df.columns.str.replace('case_', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[                                        ] | 0% Completed | 2.96 ms"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\AI-Projects\\venv311\\Lib\\site-packages\\dask\\dataframe\\core.py:4378: UserWarning: \n",
      "You did not provide metadata, so Dask is running your function on a small dataset to guess output types. It is possible that Dask will guess incorrectly.\n",
      "To provide an explicit output types or to silence this message, please provide the `meta=` keyword, as described in the map or apply function that you are using.\n",
      "  Before: .apply(func)\n",
      "  After:  .apply(func, meta=('html', 'object'))\n",
      "\n",
      "  warnings.warn(meta_warning(meta))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed | 95.45 s\n"
     ]
    }
   ],
   "source": [
    "# Run this in parallel to speed up\n",
    "\n",
    "import dask\n",
    "import dask.dataframe as dd\n",
    "from dask.diagnostics import ProgressBar\n",
    "\n",
    "# Extract text of cases from html\n",
    "def get_text(html):\n",
    "\n",
    "    # if html is None, return None\n",
    "    if html is None:\n",
    "        return None\n",
    "    \n",
    "    # extract text from class 'entry-content' in page1 html using beautiful soup\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "    # convert <br> to new line to preserve paragraphs\n",
    "    for br in soup.find_all('br'):\n",
    "        br.replace_with('\\n')\n",
    "\n",
    "    # Insert newline characters after each <p> tag to preserve paragraphs\n",
    "    for p in soup.find_all('p'):\n",
    "        p.insert_after('\\n')\n",
    "\n",
    "    return soup.text\n",
    "\n",
    "# Convert the pandas DataFrame to a Dask DataFrame # HAD TO INCREASE PARTITIONS DUE TO MEMORY LIMITS\n",
    "ddf = dd.from_pandas(df, npartitions=50) \n",
    "\n",
    "# Apply the function in parallel\n",
    "ddf['text'] = ddf.html.apply(get_text)\n",
    "\n",
    "# Convert the Dask DataFrame back to a pandas DataFrame\n",
    "with dask.config.set(scheduler='processes'):\n",
    "    with ProgressBar():\n",
    "        df = ddf.compute()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29172/29172 [00:22<00:00, 1325.46it/s]\n"
     ]
    }
   ],
   "source": [
    "# Clean text of cases\n",
    "def clean_text(text):\n",
    "\n",
    "    # remove \\xa0\n",
    "    text = text.replace('\\xa0', ' ')\n",
    "\n",
    "    # Remove multiple whitespaces and preserve paragraphs\n",
    "    text = '\\n'.join([re.sub(r'\\s+', ' ', line.strip()) for line in text.split('\\n')])\n",
    "    \n",
    "    # Remove single newlines\n",
    "    text = re.sub(r'(?<!\\n)\\n(?!\\n)', ' ', text)\n",
    "\n",
    "    # Convert multiple newlines to single newlines\n",
    "    text = re.sub(r'\\n+', '\\n', text)\n",
    "\n",
    "    # Remove 'You are being directed to the most recent version...\n",
    "    if 'You are being directed to the most recent version of the statute which may not be' in text:\n",
    "        text = text.split('You are being directed to the most recent version of the statute which may not be')[0]\n",
    "\n",
    "    # Remove 'Vous allez être redirigé vers la version...'\n",
    "    if '\\nVous allez être redirigé vers la version' in text:\n",
    "        text = text.split('\\nVous allez être redirigé vers la version')[0]\n",
    "\n",
    "    # if '\\nDecision Information\\n' in text, remove everything before it\n",
    "    if '\\nDecision Information\\n' in text:\n",
    "        text = text.split('\\nDecision Information\\n')[1]\n",
    "\n",
    "    # if '\\nInformations sur la décision\\n' in text, remove everything before it\n",
    "    if '\\nInformations sur la décision\\n' in text:\n",
    "        text = text.split('\\nInformations sur la décision\\n')[1]\n",
    "\n",
    "    # Remove all strings '\\n[Page #]\\n' (with # being a number of up to 4 digits \n",
    "    text = re.sub(r'\\n\\[Page \\d{1,3}\\]\\n', ' ', text)\n",
    "    \n",
    "    return text\n",
    "\n",
    "df['unofficial_text'] = df.text.progress_apply(clean_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specific cleaning for SST:\n",
    "\n",
    "# fix the weird footnote references\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Return to footnote \\d+ referrer\\n', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Return to footnote \\d+ referrer', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Return to footnote \\d+\\n', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Return to footnote \\d+', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Retour à la référence de la note de bas de page \\d+ referrer\\n', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Retour à la référence de la note de bas de page \\d+ referrer', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Retour à la référence de la note de bas de page \\d+\\n', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'Retour à la référence de la note de bas de page \\d+', '', x))\n",
    "\n",
    "# remove the weird charater \"\\ufeff\\n\"\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'\\ufeff\\n', '', x))\n",
    "df['unofficial_text'] = df['unofficial_text'].apply(lambda x: re.sub(r'\\ufeff ', '', x))\n",
    "\n",
    "# Formatting of the section headings could use some work\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop unneeded columns\n",
    "df = df.drop(columns=['html'])\n",
    "df = df.drop(columns=['text'])\n",
    "\n",
    "# rename decision_date to document_date\n",
    "df = df.rename(columns={'decision_date': 'document_date'})\n",
    "\n",
    "# add dataset column\n",
    "df['dataset'] = 'SST'\n",
    "\n",
    "# add other column\n",
    "df['other'] = ''\n",
    "\n",
    "# reorder columns\n",
    "list_cols = ['citation',\n",
    "             'citation2', \n",
    "             'dataset', \n",
    "             'year', \n",
    "             'name', \n",
    "             'language', \n",
    "             'document_date', \n",
    "             'source_url', \n",
    "             'scraped_timestamp', \n",
    "             'unofficial_text',\n",
    "             'other']\n",
    "\n",
    "df = df[list_cols]\n",
    "\n",
    "# reindex\n",
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>citation</th>\n",
       "      <th>citation2</th>\n",
       "      <th>dataset</th>\n",
       "      <th>year</th>\n",
       "      <th>name</th>\n",
       "      <th>language</th>\n",
       "      <th>document_date</th>\n",
       "      <th>source_url</th>\n",
       "      <th>scraped_timestamp</th>\n",
       "      <th>unofficial_text</th>\n",
       "      <th>other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013 SSTAD 1</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2013</td>\n",
       "      <td>K.O. v. Minister of Human Resources and Skills...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-05-14</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26</td>\n",
       "      <td>K.O. v. Minister of Human Resources and Skills...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013 SSTAD 10</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2013</td>\n",
       "      <td>S. H. v. Minister of Human Resources and Skill...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-11-27</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26</td>\n",
       "      <td>S. H. v. Minister of Human Resources and Skill...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013 SSTAD 11</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2013</td>\n",
       "      <td>M. H. v. Minister of Human Resources and Skill...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-11-27</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26</td>\n",
       "      <td>M. H. v. Minister of Human Resources and Skill...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013 SSTAD 12</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2013</td>\n",
       "      <td>L. L. and Minister of Human Resources and Skil...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cpp-rp...</td>\n",
       "      <td>2023-07-27</td>\n",
       "      <td>L. L. and Minister of Human Resources and Skil...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013 SSTAD 13</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2013</td>\n",
       "      <td>R. S. v. Minister of Human Resources and Skill...</td>\n",
       "      <td>en</td>\n",
       "      <td>2013-12-17</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2023-07-26</td>\n",
       "      <td>R. S. v. Minister of Human Resources and Skill...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        citation citation2 dataset  year   \n",
       "0   2013 SSTAD 1               SST  2013  \\\n",
       "1  2013 SSTAD 10               SST  2013   \n",
       "2  2013 SSTAD 11               SST  2013   \n",
       "3  2013 SSTAD 12               SST  2013   \n",
       "4  2013 SSTAD 13               SST  2013   \n",
       "\n",
       "                                                name language document_date   \n",
       "0  K.O. v. Minister of Human Resources and Skills...       en    2013-05-14  \\\n",
       "1  S. H. v. Minister of Human Resources and Skill...       en    2013-11-27   \n",
       "2  M. H. v. Minister of Human Resources and Skill...       en    2013-11-27   \n",
       "3  L. L. and Minister of Human Resources and Skil...       en    2013-12-31   \n",
       "4  R. S. v. Minister of Human Resources and Skill...       en    2013-12-17   \n",
       "\n",
       "                                          source_url scraped_timestamp   \n",
       "0  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...        2023-07-26  \\\n",
       "1  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...        2023-07-26   \n",
       "2  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...        2023-07-26   \n",
       "3  https://decisions.sst-tss.gc.ca/sst-tss/cpp-rp...        2023-07-27   \n",
       "4  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...        2023-07-26   \n",
       "\n",
       "                                     unofficial_text other  \n",
       "0  K.O. v. Minister of Human Resources and Skills...        \n",
       "1  S. H. v. Minister of Human Resources and Skill...        \n",
       "2  M. H. v. Minister of Human Resources and Skill...        \n",
       "3  L. L. and Minister of Human Resources and Skil...        \n",
       "4  R. S. v. Minister of Human Resources and Skill...        "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>citation</th>\n",
       "      <th>citation2</th>\n",
       "      <th>dataset</th>\n",
       "      <th>year</th>\n",
       "      <th>name</th>\n",
       "      <th>language</th>\n",
       "      <th>document_date</th>\n",
       "      <th>source_url</th>\n",
       "      <th>scraped_timestamp</th>\n",
       "      <th>unofficial_text</th>\n",
       "      <th>other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29167</th>\n",
       "      <td>2024 TSS 94</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2024</td>\n",
       "      <td>RO c Ministre de l’Emploi et du Développement ...</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-30</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...</td>\n",
       "      <td>2024-07-14</td>\n",
       "      <td>RO c Ministre de l’Emploi et du Développement ...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29168</th>\n",
       "      <td>2024 TSS 95</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2024</td>\n",
       "      <td>TK c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-29</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14</td>\n",
       "      <td>TK c Commission de l’assurance-emploi du Canad...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29169</th>\n",
       "      <td>2024 TSS 96</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2024</td>\n",
       "      <td>IR c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14</td>\n",
       "      <td>IR c Commission de l’assurance-emploi du Canad...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29170</th>\n",
       "      <td>2024 TSS 98</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2024</td>\n",
       "      <td>JC c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14</td>\n",
       "      <td>JC c Commission de l’assurance-emploi du Canad...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29171</th>\n",
       "      <td>2024 TSS 99</td>\n",
       "      <td></td>\n",
       "      <td>SST</td>\n",
       "      <td>2024</td>\n",
       "      <td>DF c Commission de l’assurance-emploi du Canada</td>\n",
       "      <td>fr</td>\n",
       "      <td>2024-01-30</td>\n",
       "      <td>https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...</td>\n",
       "      <td>2024-07-14</td>\n",
       "      <td>DF c Commission de l’assurance-emploi du Canad...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          citation citation2 dataset  year   \n",
       "29167  2024 TSS 94               SST  2024  \\\n",
       "29168  2024 TSS 95               SST  2024   \n",
       "29169  2024 TSS 96               SST  2024   \n",
       "29170  2024 TSS 98               SST  2024   \n",
       "29171  2024 TSS 99               SST  2024   \n",
       "\n",
       "                                                    name language   \n",
       "29167  RO c Ministre de l’Emploi et du Développement ...       fr  \\\n",
       "29168    TK c Commission de l’assurance-emploi du Canada       fr   \n",
       "29169    IR c Commission de l’assurance-emploi du Canada       fr   \n",
       "29170    JC c Commission de l’assurance-emploi du Canada       fr   \n",
       "29171    DF c Commission de l’assurance-emploi du Canada       fr   \n",
       "\n",
       "      document_date                                         source_url   \n",
       "29167    2024-01-30  https://decisions.sst-tss.gc.ca/sst-tss/cppd-r...  \\\n",
       "29168    2024-01-29  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "29169    2024-01-31  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "29170    2024-01-31  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "29171    2024-01-30  https://decisions.sst-tss.gc.ca/sst-tss/ei-ae/...   \n",
       "\n",
       "      scraped_timestamp                                    unofficial_text   \n",
       "29167        2024-07-14  RO c Ministre de l’Emploi et du Développement ...  \\\n",
       "29168        2024-07-14  TK c Commission de l’assurance-emploi du Canad...   \n",
       "29169        2024-07-14  IR c Commission de l’assurance-emploi du Canad...   \n",
       "29170        2024-07-14  JC c Commission de l’assurance-emploi du Canad...   \n",
       "29171        2024-07-14  DF c Commission de l’assurance-emploi du Canad...   \n",
       "\n",
       "      other  \n",
       "29167        \n",
       "29168        \n",
       "29169        \n",
       "29170        \n",
       "29171        "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export cleaned df to jsonl\n",
    "df.to_json(out_path_parsed, orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export cleaned df to parquet\n",
    "\n",
    "df.to_parquet(out_path_parquet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:00<00:00, 25.90it/s] \n"
     ]
    }
   ],
   "source": [
    "# export cleaned df to yearly json files\n",
    "for year in tqdm(range(start_year, end_year+1)):\n",
    "    df[df.year == year].to_json(out_path_yearly / f'{year}.json', orient='records', indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Direct to HF\n",
    "df.to_parquet(out_path_HF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}