02_SparkDataframe_BasicOperations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init('/home/raj/spark-2.1.0-bin-hadoop2.7')\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "spark=SparkSession.builder.appName('ops').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df=spark.read.csv('/home/raj/Documents/Udemy-Spark/Python-and-Spark-for-Big-Data-master/Spark_DataFrames/appl_stock.csv',\n",
    "                 inferSchema=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Date=datetime.datetime(2010, 1, 4, 0, 0), Open=213.429998, High=214.499996, Low=212.38000099999996, Close=214.009998, Volume=123432400, Adj Close=27.727039),\n",
       " Row(Date=datetime.datetime(2010, 1, 5, 0, 0), Open=214.599998, High=215.589994, Low=213.249994, Close=214.379993, Volume=150476200, Adj Close=27.774976000000002),\n",
       " Row(Date=datetime.datetime(2010, 1, 6, 0, 0), Open=214.379993, High=215.23, Low=210.750004, Close=210.969995, Volume=138040000, Adj Close=27.333178000000004),\n",
       " Row(Date=datetime.datetime(2010, 1, 7, 0, 0), Open=211.75, High=212.000006, Low=209.050005, Close=210.58, Volume=119282800, Adj Close=27.28265),\n",
       " Row(Date=datetime.datetime(2010, 1, 8, 0, 0), Open=210.299994, High=212.000006, Low=209.06000500000002, Close=211.98000499999998, Volume=111902700, Adj Close=27.464034)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Date: timestamp (nullable = true)\n",
      " |-- Open: double (nullable = true)\n",
      " |-- High: double (nullable = true)\n",
      " |-- Low: double (nullable = true)\n",
      " |-- Close: double (nullable = true)\n",
      " |-- Volume: integer (nullable = true)\n",
      " |-- Adj Close: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+----------+----------+------------------+------------------+---------+------------------+\n",
      "|                Date|      Open|      High|               Low|             Close|   Volume|         Adj Close|\n",
      "+--------------------+----------+----------+------------------+------------------+---------+------------------+\n",
      "|2010-01-04 00:00:...|213.429998|214.499996|212.38000099999996|        214.009998|123432400|         27.727039|\n",
      "|2010-01-05 00:00:...|214.599998|215.589994|        213.249994|        214.379993|150476200|27.774976000000002|\n",
      "|2010-01-06 00:00:...|214.379993|    215.23|        210.750004|        210.969995|138040000|27.333178000000004|\n",
      "|2010-01-07 00:00:...|    211.75|212.000006|        209.050005|            210.58|119282800|          27.28265|\n",
      "|2010-01-08 00:00:...|210.299994|212.000006|209.06000500000002|211.98000499999998|111902700|         27.464034|\n",
      "+--------------------+----------+----------+------------------+------------------+---------+------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+------------------+------------------+---------+------------------+\n",
      "|                Date|              Open|      High|               Low|             Close|   Volume|         Adj Close|\n",
      "+--------------------+------------------+----------+------------------+------------------+---------+------------------+\n",
      "|2010-01-04 00:00:...|        213.429998|214.499996|212.38000099999996|        214.009998|123432400|         27.727039|\n",
      "|2010-01-05 00:00:...|        214.599998|215.589994|        213.249994|        214.379993|150476200|27.774976000000002|\n",
      "|2010-01-06 00:00:...|        214.379993|    215.23|        210.750004|        210.969995|138040000|27.333178000000004|\n",
      "|2010-01-07 00:00:...|            211.75|212.000006|        209.050005|            210.58|119282800|          27.28265|\n",
      "|2010-01-08 00:00:...|        210.299994|212.000006|209.06000500000002|211.98000499999998|111902700|         27.464034|\n",
      "|2010-01-11 00:00:...|212.79999700000002|213.000002|        208.450005|210.11000299999998|115557400|         27.221758|\n",
      "+--------------------+------------------+----------+------------------+------------------+---------+------------------+\n",
      "only showing top 6 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(\"Close<500\").show(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|                Date|\n",
      "+--------------------+\n",
      "|2014-06-09 00:00:...|\n",
      "|2014-06-10 00:00:...|\n",
      "|2014-06-11 00:00:...|\n",
      "|2014-06-12 00:00:...|\n",
      "|2014-06-13 00:00:...|\n",
      "|2014-06-16 00:00:...|\n",
      "+--------------------+\n",
      "only showing top 6 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(\"Close<100\").select('Date').show(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+------------------+----------+---------+------------------+\n",
      "|                Date|              Open|      High|               Low|     Close|   Volume|         Adj Close|\n",
      "+--------------------+------------------+----------+------------------+----------+---------+------------------+\n",
      "|2010-02-01 00:00:...|192.36999699999998|     196.0|191.29999899999999|194.729998|187469100|         25.229131|\n",
      "|2010-02-02 00:00:...|        195.909998|196.319994|193.37999299999998|195.859997|174585600|25.375532999999997|\n",
      "|2010-02-03 00:00:...|        195.169994|200.200003|        194.420004|199.229994|153832000|25.812148999999998|\n",
      "|2010-02-04 00:00:...|        196.730003|198.370001|        191.570005|192.050003|189413000|         24.881912|\n",
      "|2010-02-05 00:00:...|192.63000300000002|     196.0|        190.850002|195.460001|212576700|25.323710000000002|\n",
      "+--------------------+------------------+----------+------------------+----------+---------+------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter((df['Close']<200) & ~(df['Open']>200)).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+------+------+---------+---------+\n",
      "|                Date|              Open|      High|   Low| Close|   Volume|Adj Close|\n",
      "+--------------------+------------------+----------+------+------+---------+---------+\n",
      "|2010-01-22 00:00:...|206.78000600000001|207.499996|197.16|197.75|220441900|25.620401|\n",
      "+--------------------+------------------+----------+------+------+---------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(df['Low']== 197.16).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Date=datetime.datetime(2010, 1, 22, 0, 0), Open=206.78000600000001, High=207.499996, Low=197.16, Close=197.75, Volume=220441900, Adj Close=25.620401)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.filter(df['Low']== 197.16).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "result=df.filter(df['Low']== 197.16).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Date=datetime.datetime(2010, 1, 22, 0, 0), Open=206.78000600000001, High=207.499996, Low=197.16, Close=197.75, Volume=220441900, Adj Close=25.620401)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "row=result[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Adj Close': 25.620401,\n",
       " 'Close': 197.75,\n",
       " 'Date': datetime.datetime(2010, 1, 22, 0, 0),\n",
       " 'High': 207.499996,\n",
       " 'Low': 197.16,\n",
       " 'Open': 206.78000600000001,\n",
       " 'Volume': 220441900}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row.asDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "206.78000600000001"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row.Open"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220441900"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row.Volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['Close', 'Date', 'Open', 'Low', 'Adj Close', 'High', 'Volume'])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row.asDict().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}