Movie_review_rotten_tomatoes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing necessary Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting textblob\n",
      "  Downloading textblob-0.18.0.post0-py3-none-any.whl.metadata (4.5 kB)\n",
      "Collecting nltk>=3.8 (from textblob)\n",
      "  Downloading nltk-3.8.1-py3-none-any.whl.metadata (2.8 kB)\n",
      "Requirement already satisfied: click in c:\\users\\kiran\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from nltk>=3.8->textblob) (8.1.7)\n",
      "Requirement already satisfied: joblib in c:\\users\\kiran\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from nltk>=3.8->textblob) (1.3.2)\n",
      "Collecting regex>=2021.8.3 (from nltk>=3.8->textblob)\n",
      "  Downloading regex-2024.5.15-cp312-cp312-win_amd64.whl.metadata (41 kB)\n",
      "     ---------------------------------------- 0.0/42.0 kB ? eta -:--:--\n",
      "     ---------------------------------------  41.0/42.0 kB ? eta -:--:--\n",
      "     ---------------------------------------  41.0/42.0 kB ? eta -:--:--\n",
      "     -------------------------------------- 42.0/42.0 kB 406.8 kB/s eta 0:00:00\n",
      "Collecting tqdm (from nltk>=3.8->textblob)\n",
      "  Downloading tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)\n",
      "     ---------------------------------------- 0.0/57.6 kB ? eta -:--:--\n",
      "     ----------------------------------- ---- 51.2/57.6 kB ? eta -:--:--\n",
      "     -------------------------------------- 57.6/57.6 kB 606.6 kB/s eta 0:00:00\n",
      "Requirement already satisfied: colorama in c:\\users\\kiran\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from click->nltk>=3.8->textblob) (0.4.6)\n",
      "Downloading textblob-0.18.0.post0-py3-none-any.whl (626 kB)\n",
      "   ---------------------------------------- 0.0/626.3 kB ? eta -:--:--\n",
      "   --------------- ----------------------- 256.0/626.3 kB 15.4 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   --------------------------------------  624.6/626.3 kB 13.1 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 626.3/626.3 kB 1.2 MB/s eta 0:00:00\n",
      "Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "   ---------------------------------------- 0.0/1.5 MB ? eta -:--:--\n",
      "   ----------------- ---------------------- 0.6/1.5 MB 13.5 MB/s eta 0:00:01\n",
      "   ----------------------- ---------------- 0.9/1.5 MB 8.1 MB/s eta 0:00:01\n",
      "   --------------------------- ------------ 1.0/1.5 MB 6.5 MB/s eta 0:00:01\n",
      "   --------------------------------- ------ 1.3/1.5 MB 6.2 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------  1.5/1.5 MB 5.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 1.5/1.5 MB 2.3 MB/s eta 0:00:00\n",
      "Downloading regex-2024.5.15-cp312-cp312-win_amd64.whl (268 kB)\n",
      "   ---------------------------------------- 0.0/268.5 kB ? eta -:--:--\n",
      "   --------------------------------------  266.2/268.5 kB 17.1 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 268.5/268.5 kB 2.8 MB/s eta 0:00:00\n",
      "Downloading tqdm-4.66.4-py3-none-any.whl (78 kB)\n",
      "   ---------------------------------------- 0.0/78.3 kB ? eta -:--:--\n",
      "   ------------------------------------ --- 71.7/78.3 kB ? eta -:--:--\n",
      "   ------------------------------------ --- 71.7/78.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 78.3/78.3 kB 544.3 kB/s eta 0:00:00\n",
      "Installing collected packages: tqdm, regex, nltk, textblob\n",
      "Successfully installed nltk-3.8.1 regex-2024.5.15 textblob-0.18.0.post0 tqdm-4.66.4\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip is available: 23.3.1 -> 24.1.2\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    }
   ],
   "source": [
    "pip install textblob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools #to create efficent looping to fetch more data in a go\n",
    "import re \n",
    "import random \n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Movie Urls\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled~critics:fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled,upright~critics:fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:certified_fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled~critics:certified_fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled,upright~critics:certified_fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:rotten?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled~critics:rotten?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled,upright~critics:rotten?page=5\n",
    "\n",
    "Here we use page=5 as rottentomatoes will only allow us to check 140 movies at a time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:fresh?page=5\"\n",
    "\n",
    "def getSoup(url):\n",
    "    \"\"\"\n",
    "    Utility function this get soup function will fetch the above url which stored in url var.\n",
    "    \"\"\"\n",
    "    headers = {\n",
    "        'User-Agent': 'Your User-Agent String',\n",
    "        'Authorization': 'Bearer Your_Authentication_Token'  # Include this if authentication is required\n",
    "    }\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    return soup\n",
    "\n",
    "def getReviewText(review_url):\n",
    "    '''Returns the user review text given the review soup.'''\n",
    "    tag = review_url.find('p', attrs={'class': 'review-text'})  # Use select_one for efficient CSS selector\n",
    "    if tag:\n",
    "        return tag.get_text(strip=True)  # Use strip=True to remove extra whitespace\n",
    "    return None  # Handle case where review text is not found\n",
    "\n",
    "def getMovieTitle(review_url):\n",
    "    '''Returns the movie title from the review soup.'''\n",
    "    tag = review_url.find('title')\n",
    "    if tag:\n",
    "        title_tag = list(tag.children)[0].get_text()\n",
    "        movie_title = title_tag.split(' - Movie Reviews | Rotten Tomatoes')[0]\n",
    "        return movie_title\n",
    "    return None  # Handle case where title is not found\n",
    "\n",
    "\n",
    "def getNounChunks(user_review):\n",
    "    # create the doc object\n",
    "    doc = nlp(user_review)\n",
    "    # get a list of noun_chunks\n",
    "    noun_chunks = list(doc.noun_chunks)\n",
    "    # convert noun_chunks from span objects to strings, otherwise it won't pick\n",
    "    noun_chunks_strlist = [chunk.text for chunk in noun_chunks]\n",
    "    return noun_chunks_strlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filtering the movie tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are a total of 140 movie titles\n",
      "Displaying 10 titles\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/m/the_boy_and_the_heron',\n",
       " '/m/ferrari_2023',\n",
       " '/m/civil_war_2024',\n",
       " '/m/furiosa_a_mad_max_saga',\n",
       " '/m/the_witch_2016',\n",
       " '/m/may_december',\n",
       " '/m/dream_scenario',\n",
       " '/m/a_simple_favor',\n",
       " '/m/the_beast_2023',\n",
       " '/m/brats_2024']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_soup = getSoup(url)\n",
    "movie_tags = movies_soup.find_all('a', attrs={'data-qa': \"discovery-media-list-item-caption\"}) + movies_soup.find_all('a', attrs={'class': \"js-tile-link\"})\n",
    "\n",
    "# filter the a-tags to get just the titles\n",
    "movie_links = [tag['href'] for tag in movie_tags]\n",
    "# remove duplicate links\n",
    "unique_movie_links = list(set(tag['href'] for tag in movie_tags))\n",
    "\n",
    "print(\"There are a total of \" + str(len(unique_movie_links)) + \" movie titles\")\n",
    "print(\"Displaying 10 titles\")\n",
    "unique_movie_links[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filtering the movie URL's"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are a total of 140 movie user reviews\n",
      "Displaying 20 user reviews links\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['https://www.rottentomatoes.com/m/the_bikeriders/reviews',\n",
       " 'https://www.rottentomatoes.com/m/kingdom_of_the_planet_of_the_apes/reviews',\n",
       " 'https://www.rottentomatoes.com/m/dragonkeeper/reviews',\n",
       " 'https://www.rottentomatoes.com/m/cora_bora/reviews',\n",
       " 'https://www.rottentomatoes.com/m/beverly_hills_cop_axel_f/reviews',\n",
       " 'https://www.rottentomatoes.com/m/hit_man_2023/reviews',\n",
       " 'https://www.rottentomatoes.com/m/the_imaginary/reviews',\n",
       " 'https://www.rottentomatoes.com/m/furiosa_a_mad_max_saga/reviews',\n",
       " 'https://www.rottentomatoes.com/m/a_quiet_place_2018/reviews',\n",
       " 'https://www.rottentomatoes.com/m/the_beekeeper_2024/reviews',\n",
       " 'https://www.rottentomatoes.com/m/the_fall_guy_2024/reviews',\n",
       " 'https://www.rottentomatoes.com/m/civil_war_2024/reviews',\n",
       " 'https://www.rottentomatoes.com/m/pearl_2022/reviews',\n",
       " 'https://www.rottentomatoes.com/m/late_night_with_the_devil/reviews',\n",
       " 'https://www.rottentomatoes.com/m/x_2022/reviews',\n",
       " 'https://www.rottentomatoes.com/m/new_life_2023/reviews',\n",
       " 'https://www.rottentomatoes.com/m/challengers_2024/reviews',\n",
       " 'https://www.rottentomatoes.com/m/monkey_man/reviews',\n",
       " 'https://www.rottentomatoes.com/m/i_saw_the_tv_glow/reviews',\n",
       " 'https://www.rottentomatoes.com/m/the_ministry_of_ungentlemanly_warfare/reviews']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "base_url = \"https://www.rottentomatoes.com\"\n",
    "movie_links = [base_url + tag['href'] + '/reviews' for tag in movie_tags]\n",
    "print(\"There are a total of \" + str(len(movie_links)) + \" movie user reviews\")\n",
    "print(\"Displaying 20 user reviews links\")\n",
    "movie_links[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_soups = [getSoup(link) for link in movie_links]\n",
    "# get all movie review links from the 140 listing\n",
    "movie_review_list = [getReviewText(movie_soup) for movie_soup in movie_soups]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are a total of 22011 individual movie reviews\n",
      "Displaying 10 reviews\n",
      "['M', 'o', 's', 't', ' ', 'c', 'o', 'n', 't', 'e']\n"
     ]
    }
   ],
   "source": [
    "#Checking how many movie review were able to filter.\n",
    "movie_review_list = list(itertools.chain(*movie_review_list))\n",
    "\n",
    "print(\"There are a total of \" + str(len(movie_review_list)) + \" individual movie reviews\")\n",
    "print(\"Displaying 10 reviews\")\n",
    "print(movie_review_list[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting into the Pandas Data Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The Bikeriders', 'Kingdom of the Planet of the Apes', 'Dragonkeeper', 'Cora Bora', 'Beverly Hills Cop: Axel F', 'Hit Man', 'The Imaginary', 'Furiosa: A Mad Max Saga', 'A Quiet Place', 'The Beekeeper', 'The Fall Guy', 'Civil War', 'Pearl', 'Late Night with the Devil', 'X', 'New Life', 'Challengers', 'Monkey Man', 'I Saw the TV Glow', 'The Ministry of Ungentlemanly Warfare', 'The Long Game', 'Inside Out', 'Fancy Dance', \"The Devil's Bath\", 'A Quiet Place Part II', 'Abigail', 'Godzilla Minus One', 'The First Omen', 'Beverly Hills Cop', 'Talk to Me', 'Babes', 'Aftersun', 'Fresh Kills', 'Poor Things', 'Gladiator', 'The Idea of You', 'Infested', 'Smile', 'Dune: Part Two', 'Ultraman: Rising', 'The Killer', 'Immaculate', 'Barbarian', 'The Iron Claw', 'Princess Mononoke', 'Kung Fu Panda 4', 'Oppenheimer', 'Land of Bad', 'The Beast', 'Dark Waters', 'All of Us Strangers', 'The Last Stop in Yuma County', 'His House', 'The Boy and the Heron', 'Hereditary', 'Minions: The Rise of Gru', 'Love Lies Bleeding', 'In the Land of Saints and Sinners', 'When Evil Lurks', 'How to Rob a Bank', 'Anatomy of a Fall', 'The Gentlemen', 'Pig', 'The Holdovers', 'LaRoy, Texas', 'Interstellar', 'The Batman', 'The Hunger Games: The Ballad of Songbirds & Snakes', 'Dream Scenario', 'Killers of the Flower Moon', 'Mad Max: Fury Road', 'War for the Planet of the Apes', 'Ferrari', 'American Psycho', 'Seven Samurai', 'One Life', 'Get Out', 'The Peanut Butter Falcon', 'Am I OK?', 'The Marvels', 'Wicked Little Letters', 'The Zone of Interest', 'Perfect Days', 'Memory', 'Ambulance', 'John Wick: Chapter 4', 'Problemista', 'Brats', 'Deadpool', 'Exhuma', 'Hunt for the Wilderpeople', 'Nope', 'The Gift', 'Midsommar', 'Top Gun: Maverick', 'May December', 'Saltburn', 'Parasite', 'The Northman', 'Mission: Impossible - Dead Reckoning Part One', 'Wonka', 'Remembering Gene Wilder', 'American Fiction', 'No Hard Feelings', 'Arthur the King', 'The Killing of a Sacred Deer', 'Past Lives', 'A Simple Favor', 'The Truman Show', 'Avengers: Endgame', 'Spider-Man: Across the Spider-Verse', 'Barbie', 'Shrek', 'Indiana Jones and the Dial of Destiny', 'Bad Boys for Life', 'Dawn of the Planet of the Apes', 'Saw X', 'The Witch', \"Marvel's the Avengers\", 'The Promised Land', 'Godzilla vs. Kong', 'Thanksgiving', 'Goyo', 'Tell Them You Love Me', 'It Follows', 'Despicable Me', 'I Am: Celine Dion', 'Despicable Me 2', 'Presumed Innocent', 'L.A. Confidential', 'The Grab', 'Incendies', 'Under the Shadow', 'A Most Wanted Man', 'Serial Mom', 'Rise of the Planet of the Apes', 'The Paper', 'The Silence of the Lambs', 'The Girl With the Dragon Tattoo', 'The Adjustment Bureau']\n"
     ]
    }
   ],
   "source": [
    "review_texts = [getReviewText(url) for url in movie_soups]\n",
    "\n",
    "# get movie name from the review link\n",
    "movie_titles = [getMovieTitle(url) for url in movie_soups]\n",
    "print(movie_titles)\n",
    "\n",
    "# Filtering the dataframe with only User_reviews by avoiding links and title\n",
    "\n",
    "# construct a dataframe\n",
    "df = pd.DataFrame({'user_review': review_texts })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Most contemporary westerns end up mourning a v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You can nitpick some of Kingdom of the Planet ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Perhaps not surprisingly, given that the book ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Statler, in a turn that lends a darker... is a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alas, forty years have passed, but the series ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         user_review\n",
       "0  Most contemporary westerns end up mourning a v...\n",
       "1  You can nitpick some of Kingdom of the Planet ...\n",
       "2  Perhaps not surprisingly, given that the book ...\n",
       "3  Statler, in a turn that lends a darker... is a...\n",
       "4  Alas, forty years have passed, but the series ..."
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5) #displaying the resultant data frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The data frame need to remove index and filter the limit review length by 50 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_list = [m for m in df['user_review']]\n",
    "#text_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_review</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Most contemporary westerns end up mourning a v...</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You can nitpick some of Kingdom of the Planet ...</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Perhaps not surprisingly, given that the book ...</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Statler, in a turn that lends a darker... is a...</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alas, forty years have passed, but the series ...</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Rise of the Planet of the Apes has everything ...</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>A smart and entertaining movie, not an importa...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>The crux of this excellent thriller is the bat...</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>It is a beautiful example of what happens when...</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>What sustains the film is the handsome locatio...</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           user_review  length\n",
       "0    Most contemporary westerns end up mourning a v...      37\n",
       "1    You can nitpick some of Kingdom of the Planet ...      26\n",
       "2    Perhaps not surprisingly, given that the book ...      26\n",
       "3    Statler, in a turn that lends a darker... is a...      37\n",
       "4    Alas, forty years have passed, but the series ...      20\n",
       "..                                                 ...     ...\n",
       "135  Rise of the Planet of the Apes has everything ...      19\n",
       "136  A smart and entertaining movie, not an importa...       9\n",
       "137  The crux of this excellent thriller is the bat...      21\n",
       "138  It is a beautiful example of what happens when...      21\n",
       "139  What sustains the film is the handsome locatio...      17\n",
       "\n",
       "[140 rows x 2 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calculating the length of the text\n",
    "text_list_length = [len(m.split()) for m in text_list]     \n",
    "df['length'] = text_list_length\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_review</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Most contemporary westerns end up mourning a v...</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You can nitpick some of Kingdom of the Planet ...</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Perhaps not surprisingly, given that the book ...</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Statler, in a turn that lends a darker... is a...</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alas, forty years have passed, but the series ...</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Rise of the Planet of the Apes has everything ...</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>A smart and entertaining movie, not an importa...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>The crux of this excellent thriller is the bat...</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>It is a beautiful example of what happens when...</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>What sustains the film is the handsome locatio...</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           user_review  length\n",
       "0    Most contemporary westerns end up mourning a v...      37\n",
       "1    You can nitpick some of Kingdom of the Planet ...      26\n",
       "2    Perhaps not surprisingly, given that the book ...      26\n",
       "3    Statler, in a turn that lends a darker... is a...      37\n",
       "4    Alas, forty years have passed, but the series ...      20\n",
       "..                                                 ...     ...\n",
       "135  Rise of the Planet of the Apes has everything ...      19\n",
       "136  A smart and entertaining movie, not an importa...       9\n",
       "137  The crux of this excellent thriller is the bat...      21\n",
       "138  It is a beautiful example of what happens when...      21\n",
       "139  What sustains the film is the handsome locatio...      17\n",
       "\n",
       "[140 rows x 2 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df['length'] < 50]  #limiting the df by 50 in length\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Most contemporary westerns end up mourning a v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You can nitpick some of Kingdom of the Planet ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Perhaps not surprisingly, given that the book ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Statler, in a turn that lends a darker... is a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alas, forty years have passed, but the series ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Rise of the Planet of the Apes has everything ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>A smart and entertaining movie, not an importa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>The crux of this excellent thriller is the bat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>It is a beautiful example of what happens when...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>What sustains the film is the handsome locatio...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           user_review\n",
       "0    Most contemporary westerns end up mourning a v...\n",
       "1    You can nitpick some of Kingdom of the Planet ...\n",
       "2    Perhaps not surprisingly, given that the book ...\n",
       "3    Statler, in a turn that lends a darker... is a...\n",
       "4    Alas, forty years have passed, but the series ...\n",
       "..                                                 ...\n",
       "135  Rise of the Planet of the Apes has everything ...\n",
       "136  A smart and entertaining movie, not an importa...\n",
       "137  The crux of this excellent thriller is the bat...\n",
       "138  It is a beautiful example of what happens when...\n",
       "139  What sustains the film is the handsome locatio...\n",
       "\n",
       "[140 rows x 1 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop('length', axis=1, inplace=True)\n",
    "df\n",
    "#dropping the len row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Ensure the directory exists\n",
    "os.makedirs('data_scrapped', exist_ok=True)\n",
    "\n",
    "# Save the DataFrame to CSV\n",
    "df.to_csv('data_scrapped/data_rotten_tomatoes.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting the csv file to the indivitual text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "with open(\"data_scrapped/data_rotten_tomatoes.csv\", \"r\",encoding=\"utf-8\") as f:\n",
    "        reader = csv.reader(f)\n",
    "        rownumber = 2639    # used to start the naming of the file , change it accordingly \n",
    "        for row in reader:\n",
    "             g=open(str(rownumber)+\".txt\",\"w\")\n",
    "             g.write(str(row))\n",
    "             rownumber = rownumber + 1\n",
    "             g.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_sentiment(text):\n",
    "    \"\"\"\n",
    "    Analyzes the sentiment of the input text.\n",
    "    \n",
    "    Returns:\n",
    "    - 'positive' if sentiment polarity > 0\n",
    "    - 'negative' if sentiment polarity < 0\n",
    "    - 'neutral' if sentiment polarity == 0\n",
    "    \"\"\"\n",
    "    blob = TextBlob(text)\n",
    "    polarity = blob.sentiment.polarity\n",
    "    \n",
    "    if polarity > 0:\n",
    "        return 'positive'\n",
    "    elif polarity < 0:\n",
    "        return 'negative'\n",
    "    else:\n",
    "        return 'neutral'\n",
    "\n",
    "# Assuming df is your DataFrame containing the reviews\n",
    "df['sentiment'] = df['user_review'].apply(analyze_sentiment)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Most contemporary westerns end up mourning a v...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You can nitpick some of Kingdom of the Planet ...</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Perhaps not surprisingly, given that the book ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Statler, in a turn that lends a darker... is a...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alas, forty years have passed, but the series ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Rise of the Planet of the Apes has everything ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>A smart and entertaining movie, not an importa...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>The crux of this excellent thriller is the bat...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>It is a beautiful example of what happens when...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>What sustains the film is the handsome locatio...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           user_review sentiment\n",
       "0    Most contemporary westerns end up mourning a v...  positive\n",
       "1    You can nitpick some of Kingdom of the Planet ...   neutral\n",
       "2    Perhaps not surprisingly, given that the book ...  negative\n",
       "3    Statler, in a turn that lends a darker... is a...  positive\n",
       "4    Alas, forty years have passed, but the series ...  negative\n",
       "..                                                 ...       ...\n",
       "135  Rise of the Planet of the Apes has everything ...  positive\n",
       "136  A smart and entertaining movie, not an importa...  positive\n",
       "137  The crux of this excellent thriller is the bat...  positive\n",
       "138  It is a beautiful example of what happens when...  positive\n",
       "139  What sustains the film is the handsome locatio...  positive\n",
       "\n",
       "[140 rows x 2 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}