scripts/02a_query_virustotal_proxy.nbconvert.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "id": "8366b127-7e97-4dfe-87ee-0023407f1f12",
   "metadata": {},
   "source": [
    "!pip install free-proxy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c80a16c5-6e2b-4e42-b911-bd20c93f66a2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:03:47.610672Z",
     "iopub.status.busy": "2023-07-18T06:03:47.610009Z",
     "iopub.status.idle": "2023-07-18T06:03:48.046209Z",
     "shell.execute_reply": "2023-07-18T06:03:48.045038Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import time\n",
    "from datetime import datetime\n",
    "import json\n",
    "import logging\n",
    "import os\n",
    "\n",
    "logging.basicConfig(filename='12_nbconvert_proxy.log', level=logging.INFO)\n",
    "\n",
    "# Request rate\t4 lookups / min\n",
    "# Daily quota\t500 lookups / day\n",
    "# Monthly quota\t15.5 K lookups / month\n",
    "\n",
    "sleep = 200  # sleep 200 seconds after each call\n",
    "\n",
    "headers = {\n",
    "    \"accept\": \"application/json\",\n",
    "    # https://www.virustotal.com/gui/user/lsys/apikey\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "74de0d0a-50cd-4fb7-9170-c6a599a524a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:03:48.051975Z",
     "iopub.status.busy": "2023-07-18T06:03:48.050785Z",
     "iopub.status.idle": "2023-07-18T06:03:48.080580Z",
     "shell.execute_reply": "2023-07-18T06:03:48.079189Z"
    }
   },
   "outputs": [],
   "source": [
    "from requests.exceptions import ProxyError, ReadTimeout, ConnectTimeout\n",
    "from fp.fp import FreeProxy\n",
    "\n",
    "def get_proxy():\n",
    "    try:\n",
    "        proxy = FreeProxy(rand=True).get()\n",
    "        scheme_proxy_map = {\n",
    "            'http': proxy,\n",
    "            'https': proxy,\n",
    "        }\n",
    "        response = requests.get('https://ip.oxylabs.io/ip', proxies=scheme_proxy_map, timeout=10, verify=False)\n",
    "        #print(response.text)\n",
    "        if len(response.text.split()) == 1:\n",
    "            return proxy\n",
    "        else:\n",
    "            return None\n",
    "    #except (ProxyError, ReadTimeout, ConnectTimeout) as error:\n",
    "    except Exception as error:\n",
    "        print('Unable to connect to the proxy: ', error)\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5d47f4d7-6097-479a-9e73-46b45a9c40f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:03:48.085007Z",
     "iopub.status.busy": "2023-07-18T06:03:48.084376Z",
     "iopub.status.idle": "2023-07-18T06:04:28.302995Z",
     "shell.execute_reply": "2023-07-18T06:04:28.302366Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get proxy...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/venv/lib/python3.8/site-packages/urllib3/connectionpool.py:1056: InsecureRequestWarning: Unverified HTTPS request is being made to host '3.96.207.125'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get proxy...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/venv/lib/python3.8/site-packages/urllib3/connectionpool.py:1056: InsecureRequestWarning: Unverified HTTPS request is being made to host '13.125.218.203'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get proxy...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/venv/lib/python3.8/site-packages/urllib3/connectionpool.py:1056: InsecureRequestWarning: Unverified HTTPS request is being made to host '13.125.218.203'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get proxy...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/venv/lib/python3.8/site-packages/urllib3/connectionpool.py:1056: InsecureRequestWarning: Unverified HTTPS request is being made to host '186.121.235.222'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://186.121.235.222:8080\n"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    print('Get proxy...')\n",
    "    proxy = get_proxy()\n",
    "    if proxy:\n",
    "        print(proxy)\n",
    "        break\n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eda8df6f-9503-42e8-a44a-45bc095bf2c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:04:28.306130Z",
     "iopub.status.busy": "2023-07-18T06:04:28.305675Z",
     "iopub.status.idle": "2023-07-18T06:04:28.308676Z",
     "shell.execute_reply": "2023-07-18T06:04:28.308181Z"
    }
   },
   "outputs": [],
   "source": [
    "scheme_proxy_map = {\n",
    "    'http': proxy,\n",
    "    'https': proxy,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c70f131e-74f1-47dc-9869-0d7d1f0eaaa2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:04:28.312361Z",
     "iopub.status.busy": "2023-07-18T06:04:28.311673Z",
     "iopub.status.idle": "2023-07-18T06:07:52.159068Z",
     "shell.execute_reply": "2023-07-18T06:07:52.157892Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/venv/lib/python3.8/site-packages/urllib3/connectionpool.py:1056: InsecureRequestWarning: Unverified HTTPS request is being made to host '186.121.235.222'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "chunk = \"data/chunk2.csv\"\n",
    "df = pd.read_csv(chunk)\n",
    "\n",
    "for ix, row in df.iterrows():\n",
    "    now = datetime.now().strftime(\"%d %b %Y %H:%M:%S\")\n",
    "    domain = row[\"private_domain\"]\n",
    "    return_code = row[\"return_code\"]\n",
    "    \n",
    "    ## Move on if already queried\n",
    "    if return_code == 200:\n",
    "        logging.info(f\"({1+ix}/{len(df)}, {now}): Return code: {int(return_code)} - {domain}\")\n",
    "        continue\n",
    "\n",
    "    if os.path.exists(f\"payloads_json/{domain}.json\"):\n",
    "        logging.info(f\"({1+ix}/{len(df)}, {now}): Return code: {return_code} - {domain}\")\n",
    "        continue\n",
    "\n",
    "    while True:\n",
    "        try:\n",
    "            ## Query\n",
    "            # https://developers.virustotal.com/reference/domain-info\n",
    "            vt_url = f\"https://www.virustotal.com/api/v3/domains/{domain}\"\n",
    "            response = requests.get(vt_url, headers=headers, proxies=scheme_proxy_map, timeout=15, verify=False)\n",
    "            break\n",
    "        except Exception as e:\n",
    "            logging.error(f'Error: {e}')\n",
    "            # Get new proxy\n",
    "            while True:\n",
    "                print('Get proxy...')\n",
    "                proxy = get_proxy()\n",
    "                if proxy:\n",
    "                    print(proxy)\n",
    "                    break\n",
    "                time.sleep(5)\n",
    "            scheme_proxy_map = {\n",
    "                'http': proxy,\n",
    "                'https': proxy,\n",
    "            }\n",
    "    ## Save payload\n",
    "    return_code = response.status_code \n",
    "    if return_code == 200:\n",
    "        with open(f\"payloads_json/{domain}.json\", \"wb\") as f:\n",
    "            f.write(response.content)\n",
    "\n",
    "    ## Update csv to indicate row is already queried\n",
    "    df.at[ix, \"return_code\"] = return_code\n",
    "\n",
    "    logging.info(f\"({1+ix}/{len(df)}, {now}): Return code: {return_code} - {domain}\")\n",
    "\n",
    "    time.sleep(sleep)\n",
    "\n",
    "df.to_csv(chunk, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4eb96b08-e85a-4eed-b2c5-6b294dd30169",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:07:52.165617Z",
     "iopub.status.busy": "2023-07-18T06:07:52.164601Z",
     "iopub.status.idle": "2023-07-18T06:07:52.184457Z",
     "shell.execute_reply": "2023-07-18T06:07:52.183646Z"
    }
   },
   "outputs": [],
   "source": [
    "df.to_csv(chunk, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4b2acec6-7bee-480e-b126-bdfa9939766b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T06:07:52.187860Z",
     "iopub.status.busy": "2023-07-18T06:07:52.187552Z",
     "iopub.status.idle": "2023-07-18T06:07:52.210521Z",
     "shell.execute_reply": "2023-07-18T06:07:52.209339Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>private_domain</th>\n",
       "      <th>return_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>evanced.info</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>pinata.ai</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>rmine.co</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>replaymod.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>rhoadsenergy.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6993</th>\n",
       "      <td>knfilters.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6994</th>\n",
       "      <td>ambankwacoonline.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6995</th>\n",
       "      <td>rewasd.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6996</th>\n",
       "      <td>finduslocal.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6997</th>\n",
       "      <td>shellride.com</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1215 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            private_domain  return_code\n",
       "251           evanced.info        200.0\n",
       "252              pinata.ai        200.0\n",
       "253               rmine.co        200.0\n",
       "254          replaymod.com        200.0\n",
       "255       rhoadsenergy.com        200.0\n",
       "...                    ...          ...\n",
       "6993         knfilters.com        200.0\n",
       "6994  ambankwacoonline.com        200.0\n",
       "6995            rewasd.com        200.0\n",
       "6996       finduslocal.com        200.0\n",
       "6997         shellride.com        200.0\n",
       "\n",
       "[1215 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.query(\"return_code==200\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dcb5cca-4fb4-4932-bf2e-f9d3029fbd48",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}