fetch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# Basic notebook to fetch the mhtml of a page and add it to example data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import urllib.parse\n",
    "import uuid\n",
    "\n",
    "from playwright.async_api import async_playwright"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup a playwright instance. We need to do this to access the dev tool functions\n",
    "p = await async_playwright().__aenter__()\n",
    "browser = await p.chromium.launch(headless=False)\n",
    "context = await browser.new_context(viewport={\"width\": 1280, \"height\": 1280})\n",
    "page = await context.new_page()\n",
    "client = await context.new_cdp_session(page)\n",
    "page.set_default_navigation_timeout(50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response url='https://gpssa.gov.ae/en/Pages/LawsandRegulations.aspx' request=<Request url='https://gpssa.gov.ae/en/Pages/LawsandRegulations.aspx' method='GET'>>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Go to your desired URL\n",
    "desired_url = \"https://gpssa.gov.ae/en/Pages/LawsandRegulations.aspx#/\"\n",
    "await page.goto(desired_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_path(url: str) -> str:\n",
    "    parsed_url = urllib.parse.urlparse(url)\n",
    "    domain = parsed_url.netloc\n",
    "    domain = domain.replace(\"www.\", \"\").replace(\".com\", \"\")\n",
    "    return f\"./{domain}.mhtml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page saved as MHTML at: ./static/caa0a351-24a8-40c3-9c62-134d52003f49/index.mhtml\n",
      "Page saved as MHTML at: ./static/caa0a351-24a8-40c3-9c62-134d52003f49/index.mhtml\n"
     ]
    }
   ],
   "source": [
    "# Capture page content as MHTML\n",
    "result = await client.send(\"Page.captureSnapshot\", {\"format\": \"mhtml\"})\n",
    "mhtml = result[\"data\"]\n",
    "\n",
    "# Create example values\n",
    "# values = {\n",
    "#     \"id\": str(uuid.uuid4()),\n",
    "#     \"url\": desired_url,\n",
    "#     \"source\": \"mhtml\",\n",
    "#     \"domain\": \"UPDATE\",\n",
    "#     \"subdomain\": \"UPDATE\",\n",
    "#     \"type\": \"fetch\",\n",
    "#     \"goal\": {},\n",
    "#     \"evals\": [\n",
    "#       {\n",
    "#         \"type\": \"json_match\",\n",
    "#         \"expected\": {}\n",
    "#       }\n",
    "#     ]\n",
    "#   }\n",
    "values = {\n",
    "    \"id\": str(uuid.uuid4()),\n",
    "    \"url\": page.url,\n",
    "    \"source\": \"mhtml\",\n",
    "    \"category\": \"government\",\n",
    "    \"subcategory\": \"download\",\n",
    "    \"type\": \"links_fetch\",\n",
    "    \"goal\": 'Fetch all of the links to the documents on the current page. Return as a list of objects of the following schema:\\n{\"name\":\\n\\t{\"type\": \"string\", \"description\": \"The name of the document. Note that may not always be the link text. Use the element that best captures the name\"}\\n, \"url\":\\n\\t{\"type\": \"string\"}\\n}',\n",
    "    \"evals\": [{\"type\": \"json_match\", \"expected\": []}],\n",
    "}\n",
    "\n",
    "# Write MHTML content to the specified file\n",
    "folder_path = f\"./static/{values['id']}\"\n",
    "os.makedirs(folder_path, exist_ok=True)\n",
    "file_path = os.path.join(folder_path, \"index.mhtml\")\n",
    "with open(file_path, \"w\") as f:\n",
    "    f.write(mhtml)\n",
    "\n",
    "# Print the path to the console\n",
    "print(f\"Page saved as MHTML at: {file_path}\")\n",
    "file_path = os.path.join(folder_path, \"index.mhtml\")\n",
    "with open(file_path, \"w\") as f:\n",
    "    f.write(mhtml)\n",
    "\n",
    "# Print the path to the console\n",
    "print(f\"Page saved as MHTML at: {file_path}\")\n",
    "\n",
    "json_file_path = \"./static/examples.json\"\n",
    "with open(json_file_path, \"r\") as json_file:\n",
    "    data = json.load(json_file)\n",
    "\n",
    "data.append(values)\n",
    "\n",
    "with open(json_file_path, \"w\") as json_file:\n",
    "    json.dump(data, json_file, indent=4)\n",
    "    json_file.write(\"\\n\")\n",
    "# print(json.dumps(data, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['c5e7ec4b-ad83-49a5-b056-a37ddf974a64', '8366209e-e5fb-4f64-85e0-79cd90985b59', '6152ba56-e62b-42d1-beec-6ecac1dfec35', 'd86e052f-3c14-4b4d-95e7-c26d59b771f7', '4310caaa-bf1b-4704-9362-dbf5b961972d', 'ad142573-4dcb-4c1e-9a4d-6fa1fa818026', '69a755f1-33db-4f77-9b8e-0e90cf7c7188', 'f8e6ac9d-584d-4321-9451-7ebe0af3b6cb', '766abf53-e055-4e4d-ac42-531e74606846', '9a10af79-08e2-4d73-a2a1-675bb97126bb', '77d6efb9-2581-4f5f-a681-1b94696d9044', '5db440aa-f155-4297-bae4-769c8c808acd', 'cc29229c-30f6-4a03-a4b6-2755ed0cca14', '6535b75f-be1e-4f68-aedd-f3ff167d98ca', '0880ea91-7c5e-415c-9d2f-00457f110ee8', 'a266e560-aa88-49fe-b3c0-2731163b1921', 'db0872df-fd27-4fff-bce8-3513f8e1164d', '574db794-8055-4fe6-bd16-5f9ee6eccbb4', 'a4e1ac68-35bc-44a6-9e5f-53785df53bf2', 'c0028644-9bfa-470e-9efd-d8473b42e313', '111ebd24-8617-48cb-b963-eb692bf1e53a', '270c2ebd-96c0-4bab-a24c-2c38a2df20aa', 'ed7b1cf6-5be4-40e7-ad00-89a67d1c8e40', '5df3d301-09f0-4ece-946d-c24952069ab7', '18b0b51e-196c-4765-bd70-aecff53f7e4d', 'db3fb266-a2cc-41bd-8995-1bdf2dd647fc', 'dcad6730-1eb2-4df7-afb7-9b034afb2f04', '174b1151-ea9a-4745-bc88-8c56ffc23131', '9aa944d7-257b-4451-9958-feddfb7bd250', '18c29ec8-de1d-4015-aeb5-92129b9bba6c', 'a10f13b0-e4f1-490a-9fa1-64082d15e087', '97022d6c-f9d6-4481-85c8-2a9d9779e7b9', '41147504-625d-45d5-9689-92218fdd32b5', 'c713976a-fd1d-4577-9956-c2a3ab295ead', '47b45f41-2b7e-4295-ac0c-93f02354cd93', '97dc7166-7dcb-4c9f-941f-342996bf1ca8', '29cc4cad-f85c-4d8f-952e-f41f7eb2bc73', 'bb50f4e4-7d66-4ba6-b613-1065ea58a6fc', '1e4d7ca6-5292-4857-aca6-bd0fff337435', 'c45a5a21-964c-422c-96a9-acbfdebc64bc', 'f9e89327-3556-492d-963d-3765eebdbb1d', '3724f66f-9743-499c-a4b5-7b7f32d9e2a4', '3ee0e369-1cd7-41ac-940c-bee1afc1ef26', 'ce4dede3-9fee-44ea-99a6-0137a9aa2db8', '998ed1f9-a3a8-4e85-9863-ccfa8d21d9a6', '18f93ec4-086c-467c-a952-8565d8005742', 'cd1ba388-8732-49e0-a5b1-09691aad999d', '6ad8593d-547d-4591-b4d5-a7cc42feeed4', '974d2768-776c-4157-8e6d-aea8bb22f832', 'bdf05e0f-fe5f-4b0c-b750-a176d274d281', '8bf5db6d-3df6-464b-92b0-c0a04317feec', 'a2a07987-e9da-434d-9190-07a2e74c9600', '6d744ee4-bfb8-4061-badd-8ffa7e954e0f', 'fe899480-b3d1-4c28-a91f-a7c63b05c940', '62bca094-6cff-4cc5-8592-71d283babfa6', '361cdcca-41ed-4afd-b9fb-d7f98289d4fb', 'd5df56b5-f0bd-4d69-8cb5-11edfbf4568e', 'a4593517-4ff7-40dd-b100-e24b90c72819', '3692abd6-a2d1-4148-bd85-8993e9e2db81', '3b13610f-9bf2-44ad-b223-f5db5a6e259a', '4e974283-6ad8-4c6d-9788-0aff49ab0a57', '3c99030a-088b-42fc-bab6-84972afb65b5', 'afc8133f-a641-46fe-bf93-5b98d8e6cea3', '17d5678f-05ac-4bd4-b52b-79e6a710f8f1', '180a3e5f-1cd7-4073-bf02-3dc663a6d41e', '5b3eb94e-3e5c-4c39-832c-48c25f38a8b8', 'f6c00da0-3976-457b-982c-3efdea0ff7bb', '583c1f78-16aa-4eee-ae29-4277f4495ea1', '88ac8af0-d461-4c7f-8498-d7d53d65e3c0', 'b81f0252-ecca-4dda-a7e1-dcdc48d0db2c', 'c8ebee22-eeac-4479-9ca6-aebb1efc3a2f', '9d2df10f-4f04-4968-8227-c8a12f74b63d', 'ae5045c0-b6c2-40a3-9dab-bfafadb1f1ed', 'f436cee3-06d1-472e-b98b-615905e41c52', '0fe1cf15-d5fb-416c-9e1b-f5cae188d088', '4d167753-d180-497d-8510-02ab26f3a0f7', 'd401233f-2d50-426f-bf77-004411a19d6d', 'e0ce7070-8cb3-416a-8090-7a6a186b4e8a', 'c95632a7-31a8-4177-9361-d07bdd809a7a', 'a33c17c6-b2cd-433f-8972-09b969667866', 'cb25b8d2-1ac6-43d5-930f-70edeccf34f9', '2ac79ff2-a923-49df-9e66-5ce7f9d5d0e4', 'd76159ad-f62a-49e1-9896-df5ef91dcb5f', '17c411ab-5586-4cd7-940c-514959796b4f', '6d85220a-b86e-4b85-96c8-0fb94447cf29', '6b9054b1-c67c-46cc-bd67-7dfd29a9ebf1', 'a316a94d-7ad6-4385-a2e0-4b47e1f1a661', '8fc4b97d-f823-4c31-b50f-ad42671fe2b1', 'b572a5e4-717b-41ab-86e3-ec41cac368b3', 'b2a98bed-bce2-404a-9b6b-ff27d28a6c1f', '798be63a-06df-494e-a1b1-a021bec4882e', '6adb9aad-6d42-4366-a6a9-f5ae958889d7', '75ce3f5e-1dd4-42bf-8620-44a6b5bc0f78', '1d7f793c-5c51-487a-86e7-dd68a478b16a', '7bc386a7-0d5b-45dc-ab10-8189451379a6', '9e1761b8-e5bd-4e96-b47e-8ff11bfadbc0', '4a7dac8a-ff74-4e36-8edf-41ebaa30790d', '01d6fabd-adaf-4a4e-9990-0db26ea206ed', '17c261ff-3f04-4fcf-bccd-90df4d043e61', 'd0164f97-f6a3-403f-acbb-a9784194ef17', 'ad07bc26-794c-4264-8122-2aae31c383ef', '47a75c05-fc65-4a7c-b98c-d9438a5f8988', '03a77421-5216-4862-bc1d-02a8012d36cd', '854eb702-93f1-42c2-8d07-16d7b776cc38', 'a72534ac-79ea-4e94-9a76-2928ae2342b8', '1534f6a2-81ed-48ff-b118-a4d752dae524', 'fb3e1776-3a8a-41da-b077-bff601723a70', '7614e6a7-7d69-4f59-879a-4288ca319bee', '0f8d967e-07a9-450a-b6b9-c60ad4c07ec7', 'ab4689a7-cf62-473d-bfa0-5d8aa1b78d87', 'd2bffe5e-ccda-4ec7-bbab-4827da462bf9', '75d8750c-7042-4079-b782-1448475d724e', 'e99a7fb9-360e-4949-9d04-73b655502d3d', '2d12563a-feef-4049-9e56-73e6aed55dfb', '46d1069c-b509-4379-8a8b-8eed873f04e0', '2bf1dfc6-e448-4e81-90c5-70d03f6e9e87', 'a9e86456-905b-4db8-a2c6-92024869254d', 'a04ef9cc-cc51-4d28-88ad-8e153f6fbe9b', '52cafd36-198d-443f-9c7e-482970896695', 'a4c8292a-079c-4e49-bca1-cf7c9da205ec', '11332f64-75ff-42d1-a301-60aa68f9dfef', 'e01032c3-8880-4e29-990e-554d5b56a870', '79be8341-ea51-441f-ace8-21804bf0258d', 'd4d5244e-4e95-4720-900f-f809c8df320b', '755cb11a-056a-4921-9af1-f26ca5f1ddb7', 'ba7888c3-1743-44e9-90c4-d380b3fa779e', 'cbcaee75-1120-436e-8999-c2b37cf17590', '54fca906-3c08-41a4-aeeb-9b7d16dbf101', '94fdd18b-5236-4747-beca-e6fbebec10dc', 'edf9a51f-d6ac-4c44-84bf-489ce113633a', '5157d20b-011d-4a4e-9454-6453e17e5270', '7c46eb72-4688-4779-b6a1-85b580505c07', '5105d321-4c6c-4dd0-871f-20a223987279', 'b8350bd1-e085-46ce-8d35-c028cd1c8b2b', 'a82d4f15-8eba-4416-b4c3-b13d63473405', 'c16e0a60-eed3-4bb5-afe2-443377eb098c', '7c7e20e0-0a85-4c16-9f9b-dae127169cf4', '37a6d131-ff57-4fc3-b08e-f148b40e273d', '1181febe-214c-44a6-a046-8326ca87e992', '289b7d7e-c374-4856-8e09-7a1fdf388382', 'd9a539e7-d764-4cca-a300-1c1847bd470a', 'afafdb72-2e9f-4c31-a59d-b669955e047d', '6af1b0d4-2cae-4b83-8473-6aef7c4b6524', 'ea841ddd-6a8f-4364-9592-6e120a924755', 'a144f1c6-2457-42fb-b7a8-746d83cf43ba', '06613871-5393-47cd-89c9-51282dde0f41', '76cfc5d1-5fd8-4d33-b78a-4bad590f7920', '9a018abb-2554-41d9-8431-2d1512c8f0de', '90721e15-35b6-4d81-a7c5-c98296079db1', '4b874d84-267e-4c67-9b4d-3f825d967d5c', 'f270716d-1418-4654-a038-7abc1535d8bf', '35611aa7-2696-47dd-ad9a-2959248e0267', '84599be1-4fba-4d37-803d-bbee40368b3e', '77fa7108-2e1b-4d41-9dbd-8bf96398c32c', '19c03b27-1aab-41df-a3a5-04ebf601d203', '23b28508-b8be-4237-8a6d-d9901b4ae27f', '38426cb9-f372-418b-8982-a726ee8779d8', '7427dec7-7aaf-4f5f-b444-a4f020385d6e', 'fe44a879-051a-4620-a7f1-8a03a26a5107', '84b8ec9c-7352-43c2-b3b0-1c1d9ac1efa3', '83e3aff0-8447-4c67-9d4c-18ce8954585a', 'db2c9d7a-7c76-441a-a21b-c2bfe200407d', '65e9b04a-7833-43f3-877d-ae27a02eed70', 'c6bee4be-3394-4867-b5a8-52d6e9901e62', '135502cc-bdaf-4f36-a2d5-4023a331aa6a', '36601c8a-255c-4f64-b723-3694d1c9dcc1', '073862e2-e519-42cb-8cb0-d9e419a33a1b', '953ff6ad-d115-499f-9e51-bed7b638ad41', '5cc195cf-ac77-46ef-a9ac-f96887c8a50f', 'e6067e2c-efd3-4ae4-a766-a1a846c678ad', 'ede570fd-c859-47bc-8432-7d0e7a25139e', '238f430b-824a-46e9-8f2e-5fa08176a0e0', 'd0f2dd50-db96-475a-ae1f-57acba811fae', 'ef217309-7c87-45f9-931c-fc662d55646a', '6088d589-a01d-4360-a3e5-c22e7a9baed4', '0de6f7ca-3eba-45d9-990d-3255955c38e6', '4cd7de7b-5cf9-48aa-b1c9-be1858eb0a26', 'bc7d1f13-811d-42a5-a3f7-49175675a54e', 'ca386162-ee75-485e-8f9d-13d488fc477a']\n",
      "179\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import shutil\n",
    "\n",
    "\"\"\"\n",
    "Utility to delete extra mhtml folders\n",
    "\"\"\"\n",
    "# Read in the examples.json and fetch all of the ids\n",
    "ids = []\n",
    "path = \"./static/examples.json\"\n",
    "with open(path, \"r\") as json_file:\n",
    "    data = json.load(json_file)\n",
    "    ids += [item[\"id\"] for item in data]\n",
    "path = \"./static/test_examples.json\"\n",
    "with open(path, \"r\") as json_file:\n",
    "    data = json.load(json_file)\n",
    "    ids += [item[\"id\"] for item in data]\n",
    "\n",
    "print(ids)\n",
    "print(len(ids))\n",
    "\n",
    "# Iterate through the folders\n",
    "base_folder = \"./static/\"\n",
    "for folder in os.listdir(base_folder):\n",
    "    folder_path = os.path.join(base_folder, folder)\n",
    "    if folder not in ids and os.path.isdir(folder_path):\n",
    "        # If there is an id that is not present and the path is a directory, delete that folder\n",
    "        shutil.rmtree(folder_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "  {\n",
      "    \"name\": \"Link Assembleia como \\u00f3rg\\u00e3o de soberania\",\n",
      "    \"url\": \"/Parlamento/Paginas/assembleia-como-orgao-soberania.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Funcionamento\",\n",
      "    \"url\": \"/Parlamento/Paginas/funcionamento.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Compet\\u00eancia\",\n",
      "    \"url\": \"/Parlamento/Paginas/Competencias.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Organiza\\u00e7\\u00e3o\",\n",
      "    \"url\": \"/Parlamento/Paginas/Organizacao.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Plen\\u00e1rio\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/Plenario.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Presidente\",\n",
      "    \"url\": \"/sites/PARXVL/Presidente/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Mesa\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/Mesa.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Confer\\u00eancia de L\\u00edderes\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/ConferenciaLideres.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Comiss\\u00f5es\",\n",
      "    \"url\": \"/sites/COM\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Comiss\\u00e3o Permanente\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/ComissaoPermanente.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Confer\\u00eancia dos Presidentes das Comiss\\u00f5es Parlamentares\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/ConferenciasPresidentesComissoes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Resultados eleitorais\",\n",
      "    \"url\": \"/Parlamento/Paginas/ResultadosEleitorais.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Composi\\u00e7\\u00e3o dos Grupos Parlamentares/Partidos\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/GruposParlamentaresI.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Sistema eleitoral\",\n",
      "    \"url\": \"/Parlamento/Paginas/SistemaEleitoral.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Legislaturas anteriores - composi\\u00e7\\u00e3o e resultados\",\n",
      "    \"url\": \"/Parlamento/Paginas/ResultadosEleitoraisAnteriores.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Contactos\",\n",
      "    \"url\": \"/Paginas/contactos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Hist\\u00f3ria do parlamentarismo\",\n",
      "    \"url\": \"/Parlamento/Paginas/historia-do-parlamentarismo.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Imagens e apontamentos\",\n",
      "    \"url\": \"/Parlamento/Paginas/Imagens-e-apontamentos-historicos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Visita virtual\",\n",
      "    \"url\": \"https://app.parlamento.pt:443/visita360/pt/\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Visitas ao Pal\\u00e1cio\",\n",
      "    \"url\": \"/EspacoCidadao/Paginas/VisitasPalacioSBento.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Espa\\u00e7o Jovem\",\n",
      "    \"url\": \"/espacojovem/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Arquivo\",\n",
      "    \"url\": \"/Parlamento/Paginas/Arquivo.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Biblioteca\",\n",
      "    \"url\": \"/Parlamento/Paginas/BibliotecaIntroducao.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Patrim\\u00f3nio museol\\u00f3gico\",\n",
      "    \"url\": \"https://museu.parlamento.pt:443/\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Livraria Parlamentar\",\n",
      "    \"url\": \"/LivrariaParlamentar/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Conselho de Administra\\u00e7\\u00e3o\",\n",
      "    \"url\": \"/GestaoAR/Paginas/ConselhoAdministracao.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Grupo de Trabalho para os Assuntos Culturais\",\n",
      "    \"url\": \"/GestaoAR/Paginas/GrupoTrabalhoAssuntosCulturais.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Conselho de Dire\\u00e7\\u00e3o do Canal Parlamento\",\n",
      "    \"url\": \"/GestaoAR/Paginas/ConselhoDiretivodoCanalParlamento.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Secret\\u00e1rio-Geral\",\n",
      "    \"url\": \"/GestaoAR/Paginas/SecretarioGeral.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Servi\\u00e7os\",\n",
      "    \"url\": \"/GestaoAR/Paginas/Servicos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Or\\u00e7amento e Conta de Ger\\u00eancia\",\n",
      "    \"url\": \"/GestaoAR/Paginas/oar.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Contrata\\u00e7\\u00e3o p\\u00fablica\",\n",
      "    \"url\": \"/GestaoAR/Paginas/ContratacaoPublica.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Plano de Preven\\u00e7\\u00e3o de Riscos de Corrup\\u00e7\\u00e3o e Infra\\u00e7\\u00f5es Conexas\",\n",
      "    \"url\": \"/GestaoAR/Documents/PlanoPrevencaoCorrupcao.pdf\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Balan\\u00e7o Social\",\n",
      "    \"url\": \"/GestaoAR/Paginas/balanco.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Recrutamento de pessoal\",\n",
      "    \"url\": \"/GestaoAR/Paginas/RecrutamentodePessoal.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Constitui\\u00e7\\u00e3o da Rep\\u00fablica Portuguesa\",\n",
      "    \"url\": \"/Legislacao/Paginas/ConstituicaoRepublicaPortuguesa.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Revis\\u00f5es constitucionais\",\n",
      "    \"url\": \"/RevisoesConstitucionais/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link \\u00c1rea da atividade parlamentar\",\n",
      "    \"url\": \"/Legislacao/Paginas/LegislacaoAtividadeParlamentar.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link \\u00c1reas tem\\u00e1ticas das comiss\\u00f5es\",\n",
      "    \"url\": \"/Legislacao/Paginas/Legislacao_AreaComissoes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Di\\u00e1rio da Rep\\u00fablica eletr\\u00f3nico\",\n",
      "    \"url\": \"/Parlamento/Paginas/Diario-da-Republica-eletronico.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Legisla\\u00e7\\u00e3o europeia\",\n",
      "    \"url\": \"/Legislacao/Paginas/Legislacao-Europeia.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Declara\\u00e7\\u00f5es e tratados internacionais\",\n",
      "    \"url\": \"/Legislacao/Paginas/Declaracao-e-tratados-internacionais.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Elei\\u00e7\\u00f5es e mandatos de \\u00f3rg\\u00e3os externos\",\n",
      "    \"url\": \"/Parlamento/Paginas/Eleicoes-e-mandatos-de-orgaos-externos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Entidades administrativas independentes\",\n",
      "    \"url\": \"/Parlamento/Paginas/entidades-administrativas-independentes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Comiss\\u00e3o Independente de acompanhamento e fiscaliza\\u00e7\\u00e3o das Medidas Especiais de Contrata\\u00e7\\u00e3o P\\u00fablica (CIMEC)\",\n",
      "    \"url\": \"/Parlamento/Paginas/cimec.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Deputados\",\n",
      "    \"url\": \"/RegistoInteresses/Paginas/RegistoInteressesDeputados.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Membros do Governo\",\n",
      "    \"url\": \"/RegistoInteresses/Paginas/MembrosGoverno.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Entidades administrativas independentes\",\n",
      "    \"url\": \"/RegistoInteresses/Paginas/EntidadesAdministrativasIndependentes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Presidente\",\n",
      "    \"url\": \"/sites/PARXVL/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Vice-Presidentes\",\n",
      "    \"url\": \"/sites/PARXVL/VicePar/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Secret\\u00e1rios da Mesa\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/Secretarios-da-Mesa.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Confer\\u00eancia de L\\u00edderes\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/ConferenciaLideresII.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Comiss\\u00e3o Permanente\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/ComissaoPermanenteII.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Confer\\u00eancia dos Presidentes das Comiss\\u00f5es Parlamentares\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/ConferenciasPresidentesComissoesII.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Deputados em fun\\u00e7\\u00f5es\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/Deputados_ef.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Registo de interesses\",\n",
      "    \"url\": \"/RegistoInteresses/Paginas/RegistoInteressesDeputadosII.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link C\\u00f3digo de Conduta\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/CodigoConduta.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Registo de ofertas, desloca\\u00e7\\u00f5es e hospitalidades\",\n",
      "    \"url\": \"/RegistoDeslocacoesHospitalidades/Paginas/ROfertasDHList.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Presen\\u00e7as \\u00e0s Reuni\\u00f5es Plen\\u00e1rias\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/reunioesplenarias.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Poderes, direitos e deveres dos Deputados\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/PoderesDireitosDeveresDeputados.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Estatuto remunerat\\u00f3rio\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/EstatutoRemuneratorioDeputados.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Altera\\u00e7\\u00f5es \\u00e0 composi\\u00e7\\u00e3o\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/AlteracoesComposicaoPlenario.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Acolhimento aos Deputados - XV Legislatura\",\n",
      "    \"url\": \"/DeputadoGP/Paginas/acolhimento-xvleg.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Agenda\",\n",
      "    \"url\": \"http://app.parlamento.pt/BI2/\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Vota\\u00e7\\u00f5es\",\n",
      "    \"url\": \"/ArquivoDocumentacao/Paginas/Arquivodevotacoes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Iniciativas\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/IniciativasLegislativas.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Diplomas aprovados\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/DiplomasAprovados.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Interven\\u00e7\\u00f5es em Plen\\u00e1rio\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/IntervencoesPlenario.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Tipos de debate\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/TiposdeDebates.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Aprecia\\u00e7\\u00e3o de Decretos-Lei\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/ApreciacaoDecretosLei.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Interpela\\u00e7\\u00f5es ao Governo\",\n",
      "    \"url\": \"/Fiscalizacao/Paginas/Interpelacoes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Inqu\\u00e9ritos Parlamentares\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/InqueritosParlamentares.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Perguntas ao Governo e Requerimentos\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/PerguntasRequerimentos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Programas do Governo\",\n",
      "    \"url\": \"/Fiscalizacao/Paginas/ProgramaDoGoverno.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Mo\\u00e7\\u00f5es\",\n",
      "    \"url\": \"/Fiscalizacao/Paginas/Mocoes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Votos\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/Votos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Elei\\u00e7\\u00f5es e composi\\u00e7\\u00e3o de \\u00f3rg\\u00e3os\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/EleicaoEComposicaoDosOrgaos.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Peti\\u00e7\\u00f5es\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/Peticoes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Cerim\\u00f3nias e atos oficiais\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/Cerimonias.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Relat\\u00f3rios e estat\\u00edsticas da atividade parlamentar\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/Relatorios-e-estatisticas.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Relat\\u00f3rios de fiscaliza\\u00e7\\u00e3o da atividade do Governo\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/Relatorios-de-fiscalizacao-da-atividade-do-Governo.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Relat\\u00f3rios sujeitos a aprecia\\u00e7\\u00e3o parlamentar\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/Relatorios.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Ordens de trabalho e s\\u00famulas da Confer\\u00eancia de L\\u00edderes\",\n",
      "    \"url\": \"/ActividadeParlamentar/Paginas/conferencia-lideres.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Or\\u00e7amento do Estado\",\n",
      "    \"url\": \"/OrcamentoEstado/Paginas/oe.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Grandes Op\\u00e7\\u00f5es\",\n",
      "    \"url\": \"/OrcamentoEstado/Paginas/gop.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Conta Geral do Estado\",\n",
      "    \"url\": \"/OrcamentoEstado/Paginas/ContaGeraldoEstado.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Programa de Estabilidade\",\n",
      "    \"url\": \"/OrcamentoEstado/Paginas/ProgramaEstabilidadeCrescimento.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Semestre Europeu\",\n",
      "    \"url\": \"/OrcamentoEstado/Paginas/Semestre-Europeu.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link UTAO - Unidade T\\u00e9cnica de Apoio Or\\u00e7amental\",\n",
      "    \"url\": \"/OrcamentoEstado/Paginas/UTAO_UnidadeTecnicadeApoioOrcamental.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link I S\\u00e9rie\",\n",
      "    \"url\": \"/DAR/Paginas/DAR1Serie.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link II S\\u00e9rie\",\n",
      "    \"url\": \"/DAR/Paginas/DAR2Serie.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Separatas\",\n",
      "    \"url\": \"/DAR/Paginas/Separatas.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Debates parlamentares desde 1821\",\n",
      "    \"url\": \"http://debates.parlamento.pt/\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 1.\\u00aa \\u2013 Assuntos Constitucionais, Direitos, Liberdades e Garantias\",\n",
      "    \"url\": \"/sites/com/XVLeg/1CACDLG/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 2.\\u00aa \\u2013 Neg\\u00f3cios Estrangeiros e Comunidades Portuguesas\",\n",
      "    \"url\": \"/sites/com/XVLeg/2CNECP\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 3.\\u00aa \\u2013 Defesa Nacional\",\n",
      "    \"url\": \"/sites/com/XVLeg/3CDN\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 4.\\u00aa \\u2013 Assuntos Europeus\",\n",
      "    \"url\": \"/sites/com/XVLeg/4CAE\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 5.\\u00aa \\u2013 Or\\u00e7amento e Finan\\u00e7as\",\n",
      "    \"url\": \"/sites/com/XVLeg/5COF\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 6.\\u00aa \\u2013 Economia, Obras P\\u00fablicas, Planeamento e Habita\\u00e7\\u00e3o\",\n",
      "    \"url\": \"/sites/com/XVLeg/6CEOPPH\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 7.\\u00aa \\u2013 Agricultura e Pescas\",\n",
      "    \"url\": \"/sites/com/XVLeg/7CAPES\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 8.\\u00aa \\u2013 Educa\\u00e7\\u00e3o e Ci\\u00eancia\",\n",
      "    \"url\": \"/sites/com/XVLeg/8CEC\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 9.\\u00aa \\u2013 Sa\\u00fade\",\n",
      "    \"url\": \"/sites/com/XVLeg/9CS\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 10.\\u00aa \\u2013 Trabalho, Seguran\\u00e7a Social e Inclus\\u00e3o\",\n",
      "    \"url\": \"/sites/com/XVLeg/10CTSSI\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 11.\\u00aa \\u2013 Ambiente e Energia\",\n",
      "    \"url\": \"/sites/com/XVLeg/11CAENE\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 12.\\u00aa \\u2013 Cultura, Comunica\\u00e7\\u00e3o, Juventude e Desporto\",\n",
      "    \"url\": \"/sites/com/XVLeg/12CCCJD\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 13.\\u00aa \\u2013 Administra\\u00e7\\u00e3o P\\u00fablica, Ordenamento do Territ\\u00f3rio e Poder Local\",\n",
      "    \"url\": \"/sites/com/XVLeg/13CAPOTPL\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link 14.\\u00aa \\u2013 Transpar\\u00eancia e Estatuto dos Deputados\",\n",
      "    \"url\": \"/sites/com/XVLeg/14CTED\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Comiss\\u00e3o Eventual para a Revis\\u00e3o Constitucional\",\n",
      "    \"url\": \"/sites/COM/XVLeg/CERC/Paginas/Default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Comiss\\u00e3o Parlamentar de Inqu\\u00e9rito \\u00e0 Tutela Pol\\u00edtica da Gest\\u00e3o da TAP\",\n",
      "    \"url\": \"/sites/com/XVLeg/CPITAP/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Tratado de Lisboa\",\n",
      "    \"url\": \"/europa/Paginas/TratadodeLisboa.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Escrut\\u00ednio de iniciativas europeias\",\n",
      "    \"url\": \"/europa/Paginas/iniciativaseuropeias.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Acompanhamento dos assuntos europeus pela Assembleia da Rep\\u00fablica\",\n",
      "    \"url\": \"/europa/Paginas/AcompanhamentoIniciativasEuropeiasAR.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Iniciativas europeias de acompanhamento priorit\\u00e1rio\",\n",
      "    \"url\": \"/europa/Paginas/IniciativasEuropeiasacompanahmentoprioritario.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Parlamentos nacionais da Uni\\u00e3o\",\n",
      "    \"url\": \"/europa/Paginas/Parlamentos_Nacionais_UE.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Institui\\u00e7\\u00f5es europeias e \\u00f3rg\\u00e3os consultivos da Uni\\u00e3o\",\n",
      "    \"url\": \"/europa/Paginas/InstituicoesEuropeias.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Coopera\\u00e7\\u00e3o interparlamentar e rela\\u00e7\\u00f5es entre os Parlamentos e as institui\\u00e7\\u00f5es europeias\",\n",
      "    \"url\": \"/europa/Paginas/ParlamentosNacionaiseInstituicoesUE.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Iniciativa de Cidadania Europeia\",\n",
      "    \"url\": \"/europa/Paginas/IniciativadeCidadaniaEuropeia.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Perguntas frequentes\",\n",
      "    \"url\": \"/europa/Paginas/PerguntasFrequentes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Atividades\",\n",
      "    \"url\": \"/RelacoesInternacionais/Paginas/default.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Delega\\u00e7\\u00f5es permanentes\",\n",
      "    \"url\": \"/RelacoesInternacionais/Paginas/DelegacoesPermanentes.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Grupos Parlamentares de Amizade\",\n",
      "    \"url\": \"/RelacoesInternacionais/Paginas/GruposParlamentaresdeAmizade.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Coopera\\u00e7\\u00e3o interparlamentar\",\n",
      "    \"url\": \"/RelacoesInternacionais/Paginas/CooperacaoInterparlamentar.aspx\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Link Visitas oficiais\",\n",
      "    \"url\": \"/RelacoesInternacionais/Paginas/VisitasOficias.aspx\"\n",
      "  }\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "from playwright.async_api import Page\n",
    "from typing import List, Dict, Any\n",
    "\n",
    "\n",
    "async def scrape(page: Page) -> List[Dict[str, Any]]:\n",
    "    link_elements = await page.query_selector_all(\"ul.nav.navbar-nav li a.subSubButton\")\n",
    "\n",
    "    document_links: List[Dict[str, Any]] = []\n",
    "    for link in link_elements:\n",
    "        name = await link.get_attribute(\"title\")\n",
    "        url = await link.get_attribute(\"href\")\n",
    "\n",
    "        if name and url:\n",
    "            document_links.append({\"name\": name, \"url\": url})\n",
    "\n",
    "    return document_links\n",
    "\n",
    "\n",
    "print(json.dumps(await scrape(page), indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "await page.evaluate(\"window.scrollBy(0, window.innerHeight);\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "await page.mouse.wheel(0, 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "735"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page.viewport_size[\"height\"]\n",
    "await page.evaluate(\n",
    "    \"Math.max(document.body.scrollHeight, document.documentElement.scrollHeight,document.body.offsetHeight, document.documentElement.offsetHeight,document.body.clientHeight, document.documentElement.clientHeight);\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "await page.evaluate(\"async () => {window.scrollBy(0, document.body.scrollHeight);}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}