big_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "307ee95a-409d-4f2b-beb7-6bb654b56bbc",
   "metadata": {},
   "source": [
    "# Big Data\n",
    "## starting the spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "03e50a94-43ed-43ac-b9cb-f80b3c115aad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: findspark in c:\\users\\enes\\anaconda3\\lib\\site-packages (2.0.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install findspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "aee598af-6da0-430c-ba93-8de10e0e9cac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2345ae53-abdb-4d91-aac9-4435e32e9ce1",
   "metadata": {},
   "outputs": [],
   "source": [
    "findspark.init(\"C:\\spark\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23c1e4c3-2a7b-4ac4-9606-2f341c0d6984",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.conf import SparkConf\n",
    "from pyspark import SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "eb3dcea0-c0ac-4c0b-8ecf-0aff3d745361",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark=SparkSession.builder \\\n",
    "    .master(\"local\") \\\n",
    "    .appName(\"spark\") \\\n",
    "    .getOrCreate()\n",
    "    \n",
    "sc=spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fc932e44-1433-4062-929f-dcc29237a628",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://172.25.173.108:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>spark</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local appName=spark>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aa718d93-4a3d-4afa-b43b-c7db3cf6accf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.3.0'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89c2f6ad-1fef-4a3b-b867-0bb1efcd635f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'spark'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.appName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "abbc4068-e6c5-4b73-8262-2963e5a1328d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sc.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9c465e6-06b7-4b00-bfc5-e7ed75c4d4cd",
   "metadata": {},
   "source": [
    "### basic data frame operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "33a43a1c-404b-49c1-9252-9aa72f5f1117",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df=spark.read.csv(\"./churn.csv\", header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6b658f9d-b742-4bbf-98bb-2a547f495119",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: integer (nullable = true)\n",
      " |-- Names: string (nullable = true)\n",
      " |-- Age: double (nullable = true)\n",
      " |-- Total_Purchase: double (nullable = true)\n",
      " |-- Account_Manager: integer (nullable = true)\n",
      " |-- Years: double (nullable = true)\n",
      " |-- Num_Sites: double (nullable = true)\n",
      " |-- Churn: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1a6a57ef-0737-4a43-bd6b-dda92dab0cef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.sql.dataframe.DataFrame"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(spark_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f1ab9e63-8073-4436-b97c-71444793d4b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[_c0: int, Names: string, Age: double, Total_Purchase: double, Account_Manager: int, Years: double, Num_Sites: double, Churn: int]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7dd090c9-50de-4b6f-85b4-c1bcb7cf5460",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "df=sns.load_dataset(\"diamonds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "15db88a0-ed37-43be-a604-2e932925822e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4c34413a-0b95-4c5a-98fb-e97b537a5671",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carat</th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>depth</th>\n",
       "      <th>table</th>\n",
       "      <th>price</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.23</td>\n",
       "      <td>Ideal</td>\n",
       "      <td>E</td>\n",
       "      <td>SI2</td>\n",
       "      <td>61.5</td>\n",
       "      <td>55.0</td>\n",
       "      <td>326</td>\n",
       "      <td>3.95</td>\n",
       "      <td>3.98</td>\n",
       "      <td>2.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.21</td>\n",
       "      <td>Premium</td>\n",
       "      <td>E</td>\n",
       "      <td>SI1</td>\n",
       "      <td>59.8</td>\n",
       "      <td>61.0</td>\n",
       "      <td>326</td>\n",
       "      <td>3.89</td>\n",
       "      <td>3.84</td>\n",
       "      <td>2.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.23</td>\n",
       "      <td>Good</td>\n",
       "      <td>E</td>\n",
       "      <td>VS1</td>\n",
       "      <td>56.9</td>\n",
       "      <td>65.0</td>\n",
       "      <td>327</td>\n",
       "      <td>4.05</td>\n",
       "      <td>4.07</td>\n",
       "      <td>2.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.29</td>\n",
       "      <td>Premium</td>\n",
       "      <td>I</td>\n",
       "      <td>VS2</td>\n",
       "      <td>62.4</td>\n",
       "      <td>58.0</td>\n",
       "      <td>334</td>\n",
       "      <td>4.20</td>\n",
       "      <td>4.23</td>\n",
       "      <td>2.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.31</td>\n",
       "      <td>Good</td>\n",
       "      <td>J</td>\n",
       "      <td>SI2</td>\n",
       "      <td>63.3</td>\n",
       "      <td>58.0</td>\n",
       "      <td>335</td>\n",
       "      <td>4.34</td>\n",
       "      <td>4.35</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   carat      cut color clarity  depth  table  price     x     y     z\n",
       "0   0.23    Ideal     E     SI2   61.5   55.0    326  3.95  3.98  2.43\n",
       "1   0.21  Premium     E     SI1   59.8   61.0    326  3.89  3.84  2.31\n",
       "2   0.23     Good     E     VS1   56.9   65.0    327  4.05  4.07  2.31\n",
       "3   0.29  Premium     I     VS2   62.4   58.0    334  4.20  4.23  2.63\n",
       "4   0.31     Good     J     SI2   63.3   58.0    335  4.34  4.35  2.75"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f2b77f27-6659-4b88-9872-83a3f7c829f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(_c0=0, Names='Cameron Williams', Age=42.0, Total_Purchase=11066.8, Account_Manager=0, Years=7.22, Num_Sites=8.0, Churn=1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "05bbff2f-b3cf-484f-b770-9835a3c160f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "carat       float64\n",
       "cut        category\n",
       "color      category\n",
       "clarity    category\n",
       "depth       float64\n",
       "table       float64\n",
       "price         int64\n",
       "x           float64\n",
       "y           float64\n",
       "z           float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e357c3da-f7c5-4ea5-b7d0-d403b6da03d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('_c0', 'int'),\n",
       " ('Names', 'string'),\n",
       " ('Age', 'double'),\n",
       " ('Total_Purchase', 'double'),\n",
       " ('Account_Manager', 'int'),\n",
       " ('Years', 'double'),\n",
       " ('Num_Sites', 'double'),\n",
       " ('Churn', 'int')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1fcb8d94-2c5c-4acc-a0a1-b30d01f5f52d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7302358b-1159-4aa5-8f94-0c36574b0eec",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'ndim'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_7188\\3740037472.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mspark_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\spark\\python\\pyspark\\sql\\dataframe.py\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, name)\u001b[0m\n\u001b[0;32m   1986\u001b[0m         \"\"\"\n\u001b[0;32m   1987\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1988\u001b[1;33m             raise AttributeError(\n\u001b[0m\u001b[0;32m   1989\u001b[0m                 \u001b[1;34m\"'%s' object has no attribute '%s'\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1990\u001b[0m             )\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'ndim'"
     ]
    }
   ],
   "source": [
    "spark_df.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8c23b676-44ba-47fa-b2ef-afed5d57cdcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(_c0=0, Names='Cameron Williams', Age=42.0, Total_Purchase=11066.8, Account_Manager=0, Years=7.22, Num_Sites=8.0, Churn=1)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4ba039c9-469d-4556-8f74-1236202132df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|_c0|           Names| Age|Total_Purchase|Account_Manager|Years|Num_Sites|Churn|\n",
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|  0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|\n",
      "|  1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|\n",
      "|  2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|\n",
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(3, truncate=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cb8ca7d0-c5fa-461f-924e-cdc15c73d94b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "900"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3222e6e0-0952-4504-83a2-eced09a53559",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['_c0',\n",
       " 'Names',\n",
       " 'Age',\n",
       " 'Total_Purchase',\n",
       " 'Account_Manager',\n",
       " 'Years',\n",
       " 'Num_Sites',\n",
       " 'Churn']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ba337475-f3bc-4639-a54e-a7d8c9b7b240",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+-------------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "|summary|               _c0|        Names|              Age|   Total_Purchase|   Account_Manager|            Years|         Num_Sites|              Churn|\n",
      "+-------+------------------+-------------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "|  count|               900|          900|              900|              900|               900|              900|               900|                900|\n",
      "|   mean|             449.5|         null|41.81666666666667|10062.82403333334|0.4811111111111111| 5.27315555555555| 8.587777777777777|0.16666666666666666|\n",
      "| stddev|259.95191863111916|         null|6.127560416916251|2408.644531858096|0.4999208935073339|1.274449013194616|1.7648355920350969| 0.3728852122772358|\n",
      "|    min|                 0|   Aaron King|             22.0|            100.0|                 0|              1.0|               3.0|                  0|\n",
      "|    max|               899|Zachary Walsh|             65.0|         18026.01|                 1|             9.15|              14.0|                  1|\n",
      "+-------+------------------+-------------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0c498f59-6df5-40bf-8151-2ad662056be0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+\n",
      "|summary|              Age|\n",
      "+-------+-----------------+\n",
      "|  count|              900|\n",
      "|   mean|41.81666666666667|\n",
      "| stddev|6.127560416916251|\n",
      "|    min|             22.0|\n",
      "|    max|             65.0|\n",
      "+-------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.describe(\"Age\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d691b11f-23b2-4153-ac68-edcedf132e43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-------------------+\n",
      "| Age|              Names|\n",
      "+----+-------------------+\n",
      "|42.0|   Cameron Williams|\n",
      "|41.0|      Kevin Mueller|\n",
      "|38.0|        Eric Lozano|\n",
      "|42.0|      Phillip White|\n",
      "|37.0|     Cynthia Norton|\n",
      "|48.0|   Jessica Williams|\n",
      "|44.0|        Eric Butler|\n",
      "|32.0|      Zachary Walsh|\n",
      "|43.0|        Ashlee Carr|\n",
      "|40.0|     Jennifer Lynch|\n",
      "|30.0|       Paula Harris|\n",
      "|45.0|     Bruce Phillips|\n",
      "|45.0|       Craig Garner|\n",
      "|40.0|       Nicole Olson|\n",
      "|41.0|     Harold Griffin|\n",
      "|38.0|       James Wright|\n",
      "|45.0|      Doris Wilkins|\n",
      "|43.0|Katherine Carpenter|\n",
      "|53.0|     Lindsay Martin|\n",
      "|46.0|        Kathy Curry|\n",
      "+----+-------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.select(\"Age\",\"Names\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ab593352-37b6-4e69-b681-e0b912d245d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "524"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.filter(spark_df.Age>40).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "924579bf-7be1-493a-808c-7bfad42c021a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|Churn|count|\n",
      "+-----+-----+\n",
      "|    1|  150|\n",
      "|    0|  750|\n",
      "+-----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.groupby(\"Churn\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "054fea2e-fb59-42e6-ba93-a65142ecff5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----------------+\n",
      "|Churn|         avg(Age)|\n",
      "+-----+-----------------+\n",
      "|    1|42.99333333333333|\n",
      "|    0|41.58133333333333|\n",
      "+-----+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.groupby(\"Churn\").agg({\"Age\":\"mean\"}).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de42c928-a82b-4b3b-98e9-416fd0c4ca46",
   "metadata": {},
   "source": [
    "### sql operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6a796e1f-e353-4f93-a3bb-b6ca95e57d3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df.createOrReplaceTempView(\"tbl_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "315218ae-aa83-411c-8b5c-53480a35c00f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|namespace|\n",
      "+---------+\n",
      "|  default|\n",
      "+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"show databases\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5c3900b1-96dc-40b8-85c2-42af3c414700",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---------+-----------+\n",
      "|namespace|tableName|isTemporary|\n",
      "+---------+---------+-----------+\n",
      "|         |   tbl_df|       true|\n",
      "+---------+---------+-----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"show tables\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cafca2c2-1391-4894-bb01-f888d9a8343f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+\n",
      "| Age|\n",
      "+----+\n",
      "|42.0|\n",
      "|41.0|\n",
      "|38.0|\n",
      "|42.0|\n",
      "|37.0|\n",
      "|48.0|\n",
      "|44.0|\n",
      "|32.0|\n",
      "|43.0|\n",
      "|40.0|\n",
      "|30.0|\n",
      "|45.0|\n",
      "|45.0|\n",
      "|40.0|\n",
      "|41.0|\n",
      "|38.0|\n",
      "|45.0|\n",
      "|43.0|\n",
      "|53.0|\n",
      "|46.0|\n",
      "+----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"select Age from tbl_df\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cc6a0fa2-5238-44ba-9284-641c6822fe3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+\n",
      "| Age|\n",
      "+----+\n",
      "|42.0|\n",
      "|41.0|\n",
      "|38.0|\n",
      "|42.0|\n",
      "|37.0|\n",
      "+----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"select Age from tbl_df\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4496800b-c1b9-4774-aa88-918a09df0da5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----------------+\n",
      "|Churn|        mean(Age)|\n",
      "+-----+-----------------+\n",
      "|    1|42.99333333333333|\n",
      "|    0|41.58133333333333|\n",
      "+-----+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"select Churn, mean(Age) from tbl_df group by Churn\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c195082-55a3-433d-8338-b6d95c988f75",
   "metadata": {},
   "source": [
    "### big data visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e37063ce-b4f8-456d-9d2e-eb048fd385ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "14f60e84-1c57-4337-a5cd-207292abb05e",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'get'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_7188\\2323637776.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbarplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"Churn\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mspark_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mChurn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mspark_df\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\seaborn\\_decorators.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     44\u001b[0m             )\n\u001b[0;32m     45\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\seaborn\\categorical.py\u001b[0m in \u001b[0;36mbarplot\u001b[1;34m(x, y, hue, data, order, hue_order, estimator, ci, n_boot, units, seed, orient, color, palette, saturation, errcolor, errwidth, capsize, dodge, ax, **kwargs)\u001b[0m\n\u001b[0;32m   3180\u001b[0m ):\n\u001b[0;32m   3181\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3182\u001b[1;33m     plotter = _BarPlotter(x, y, hue, data, order, hue_order,\n\u001b[0m\u001b[0;32m   3183\u001b[0m                           \u001b[0mestimator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mci\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_boot\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munits\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3184\u001b[0m                           \u001b[0morient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpalette\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaturation\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\seaborn\\categorical.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, x, y, hue, data, order, hue_order, estimator, ci, n_boot, units, seed, orient, color, palette, saturation, errcolor, errwidth, capsize, dodge)\u001b[0m\n\u001b[0;32m   1582\u001b[0m                  errwidth, capsize, dodge):\n\u001b[0;32m   1583\u001b[0m         \u001b[1;34m\"\"\"Initialize the plotter.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1584\u001b[1;33m         self.establish_variables(x, y, hue, data, orient,\n\u001b[0m\u001b[0;32m   1585\u001b[0m                                  order, hue_order, units)\n\u001b[0;32m   1586\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestablish_colors\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpalette\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaturation\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\seaborn\\categorical.py\u001b[0m in \u001b[0;36mestablish_variables\u001b[1;34m(self, x, y, hue, data, orient, order, hue_order, units)\u001b[0m\n\u001b[0;32m    142\u001b[0m             \u001b[1;31m# See if we need to get variables from `data`\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    143\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mdata\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 144\u001b[1;33m                 \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    145\u001b[0m                 \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    146\u001b[0m                 \u001b[0mhue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\spark\\python\\pyspark\\sql\\dataframe.py\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, name)\u001b[0m\n\u001b[0;32m   1986\u001b[0m         \"\"\"\n\u001b[0;32m   1987\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1988\u001b[1;33m             raise AttributeError(\n\u001b[0m\u001b[0;32m   1989\u001b[0m                 \u001b[1;34m\"'%s' object has no attribute '%s'\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1990\u001b[0m             )\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'get'"
     ]
    }
   ],
   "source": [
    "sns.barplot(x = \"Churn\", y=spark_df.Churn.index, data=spark_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "cbc0e3ed-ffc9-416a-9d54-cc153c77a96b",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf=spark_df.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "51b20974-72e7-4553-9e1f-32c5b6fe0aeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_c0</th>\n",
       "      <th>Names</th>\n",
       "      <th>Age</th>\n",
       "      <th>Total_Purchase</th>\n",
       "      <th>Account_Manager</th>\n",
       "      <th>Years</th>\n",
       "      <th>Num_Sites</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Cameron Williams</td>\n",
       "      <td>42.0</td>\n",
       "      <td>11066.80</td>\n",
       "      <td>0</td>\n",
       "      <td>7.22</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Kevin Mueller</td>\n",
       "      <td>41.0</td>\n",
       "      <td>11916.22</td>\n",
       "      <td>0</td>\n",
       "      <td>6.50</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Eric Lozano</td>\n",
       "      <td>38.0</td>\n",
       "      <td>12884.75</td>\n",
       "      <td>0</td>\n",
       "      <td>6.67</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Phillip White</td>\n",
       "      <td>42.0</td>\n",
       "      <td>8010.76</td>\n",
       "      <td>0</td>\n",
       "      <td>6.71</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Cynthia Norton</td>\n",
       "      <td>37.0</td>\n",
       "      <td>9191.58</td>\n",
       "      <td>0</td>\n",
       "      <td>5.56</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   _c0             Names   Age  Total_Purchase  Account_Manager  Years  \\\n",
       "0    0  Cameron Williams  42.0        11066.80                0   7.22   \n",
       "1    1     Kevin Mueller  41.0        11916.22                0   6.50   \n",
       "2    2       Eric Lozano  38.0        12884.75                0   6.67   \n",
       "3    3     Phillip White  42.0         8010.76                0   6.71   \n",
       "4    4    Cynthia Norton  37.0         9191.58                0   5.56   \n",
       "\n",
       "   Num_Sites  Churn  \n",
       "0        8.0      1  \n",
       "1       11.0      1  \n",
       "2       12.0      1  \n",
       "3       10.0      1  \n",
       "4        9.0      1  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "36d93815-31da-412f-a042-8e47370bcf11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGwCAYAAACD0J42AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAdh0lEQVR4nO3df2yd9WHv8c9pnJgksw1OUnsWBqWq13V1wjqzZWRrCcsPlA0YY1VYQZRqmQQKjbCSlC5FugvT5KxBS6A3A4mOLfwYSyexsEpjlNCtaaMINXiNCohNoGUjETGB4tpOlmun4dw/ejm6Tvjl/MBfh9dLeqSc5/me53wfIeO3vuc5x5VqtVoNAEBBPjLeEwAAOJ5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDi1I33BE7Gm2++mVdeeSUNDQ2pVCrjPR0A4H2oVqsZGhpKW1tbPvKRd18jmZCB8sorr6S9vX28pwEAnIR9+/bl/PPPf9cxEzJQGhoakvzsAhsbG8d5NgDA+zE4OJj29vba7/F3MyED5a23dRobGwUKAEww7+f2DDfJAgDFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFmZB/zZgPj1tvvTWvvfZakmTWrFm5++67x3lGAHwQBApFe+211/Lqq6+O9zQA+IB5iwcAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIrjq+7fRdeXHxzvKXzoNfYfqlX0gf5D/psUovfOL4z3FICznBUUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojq+6p2hvTp7+tv8G4Ow2phWUdevWpVKpjNpaW1trx6vVatatW5e2trZMnTo1CxYsyPPPPz/qHMPDw1m5cmVmzpyZ6dOn56qrrsr+/ftPz9Vw1jn0iaUZ7PxcBjs/l0OfWDre0wHgAzLmt3g+9alP5cCBA7Xt2WefrR3bsGFDNm7cmM2bN2f37t1pbW3N4sWLMzQ0VBvT3d2dbdu2ZevWrdm5c2cOHTqUK664IseOHTs9VwQATHhjfounrq5u1KrJW6rVau66667cfvvtueaaa5IkDzzwQFpaWvLII4/kpptuysDAQO6///489NBDWbRoUZLk4YcfTnt7e5566qlcfvnlp3g5AMDZYMwrKC+++GLa2toye/bs/MEf/EH+8z//M0myd+/e9PX1ZcmSJbWx9fX1ufTSS7Nr164kSW9vb44ePTpqTFtbWzo7O2tj3s7w8HAGBwdHbQDA2WtMgTJv3rw8+OCD+fa3v51vfOMb6evry/z58/PjH/84fX19SZKWlpZRz2lpaakd6+vry5QpU3Leeee945i3s379+jQ1NdW29vb2sUwbAJhgxhQoS5cuze///u9nzpw5WbRoUf7pn/4pyc/eynlLpVIZ9ZxqtXrCvuO915i1a9dmYGCgtu3bt28s0wYAJphT+h6U6dOnZ86cOXnxxRdr96UcvxJy8ODB2qpKa2trRkZG0t/f/45j3k59fX0aGxtHbQDA2euUAmV4eDgvvPBCfv7nfz6zZ89Oa2trtm/fXjs+MjKSHTt2ZP78+UmSrq6uTJ48edSYAwcO5LnnnquNAQAY06d41qxZkyuvvDIXXHBBDh48mD/7sz/L4OBgbrzxxlQqlXR3d6enpycdHR3p6OhIT09Ppk2bluuuuy5J0tTUlOXLl2f16tWZMWNGmpubs2bNmtpbRgAAyRgDZf/+/fn85z+f119/PbNmzcqv//qv5+mnn86FF16YJLntttty5MiRrFixIv39/Zk3b16efPLJNDQ01M6xadOm1NXVZdmyZTly5EgWLlyYLVu2ZNKkSaf3ygCACatSrVar4z2JsRocHExTU1MGBgbO6P0oXV9+8IydGyay3ju/MN5TACagsfz+9scCAYDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4pxQo69evT6VSSXd3d21ftVrNunXr0tbWlqlTp2bBggV5/vnnRz1veHg4K1euzMyZMzN9+vRcddVV2b9//6lMBQA4i5x0oOzevTv33Xdf5s6dO2r/hg0bsnHjxmzevDm7d+9Oa2trFi9enKGhodqY7u7ubNu2LVu3bs3OnTtz6NChXHHFFTl27NjJXwkAcNY4qUA5dOhQrr/++nzjG9/IeeedV9tfrVZz11135fbbb88111yTzs7OPPDAA/mf//mfPPLII0mSgYGB3H///fmLv/iLLFq0KJ/+9Kfz8MMP59lnn81TTz11eq4KAJjQTipQbrnllvzO7/xOFi1aNGr/3r1709fXlyVLltT21dfX59JLL82uXbuSJL29vTl69OioMW1tbens7KyNOd7w8HAGBwdHbQDA2aturE/YunVrent788wzz5xwrK+vL0nS0tIyan9LS0v++7//uzZmypQpo1Ze3hrz1vOPt379+txxxx1jnSoAMEGNaQVl3759ufXWW/O3f/u3Oeecc95xXKVSGfW4Wq2esO947zZm7dq1GRgYqG379u0by7QBgAlmTIHS29ubgwcPpqurK3V1damrq8uOHTvy9a9/PXV1dbWVk+NXQg4ePFg71trampGRkfT397/jmOPV19ensbFx1AYAnL3GFCgLFy7Ms88+mz179tS2iy++ONdff3327NmTj33sY2ltbc327dtrzxkZGcmOHTsyf/78JElXV1cmT548asyBAwfy3HPP1cYAAB9uY7oHpaGhIZ2dnaP2TZ8+PTNmzKjt7+7uTk9PTzo6OtLR0ZGenp5MmzYt1113XZKkqakpy5cvz+rVqzNjxow0NzdnzZo1mTNnzgk33QIAH05jvkn2vdx22205cuRIVqxYkf7+/sybNy9PPvlkGhoaamM2bdqUurq6LFu2LEeOHMnChQuzZcuWTJo06XRPBwCYgCrVarU63pMYq8HBwTQ1NWVgYOCM3o/S9eUHz9i5YSLrvfML4z0FYAIay+9vf4sHACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4owpUO69997MnTs3jY2NaWxszCWXXJJ//ud/rh2vVqtZt25d2traMnXq1CxYsCDPP//8qHMMDw9n5cqVmTlzZqZPn56rrroq+/fvPz1XAwCcFcYUKOeff37+/M//PM8880yeeeaZ/NZv/VZ+93d/txYhGzZsyMaNG7N58+bs3r07ra2tWbx4cYaGhmrn6O7uzrZt27J169bs3Lkzhw4dyhVXXJFjx46d3isDACasSrVarZ7KCZqbm3PnnXfmD//wD9PW1pbu7u585StfSfKz1ZKWlpZ87Wtfy0033ZSBgYHMmjUrDz30UK699tokySuvvJL29vY8/vjjufzyy9/2NYaHhzM8PFx7PDg4mPb29gwMDKSxsfFUpv+uur784Bk7N0xkvXd+YbynAExAg4ODaWpqel+/v0/6HpRjx45l69atOXz4cC655JLs3bs3fX19WbJkSW1MfX19Lr300uzatStJ0tvbm6NHj44a09bWls7OztqYt7N+/fo0NTXVtvb29pOdNgAwAYw5UJ599tn83M/9XOrr63PzzTdn27Zt+aVf+qX09fUlSVpaWkaNb2lpqR3r6+vLlClTct55573jmLezdu3aDAwM1LZ9+/aNddoAwARSN9YnfOITn8iePXvyk5/8JI8++mhuvPHG7Nixo3a8UqmMGl+tVk/Yd7z3GlNfX5/6+vqxThUAmKDGvIIyZcqUfPzjH8/FF1+c9evX56KLLsrdd9+d1tbWJDlhJeTgwYO1VZXW1taMjIykv7//HccAAJzy96BUq9UMDw9n9uzZaW1tzfbt22vHRkZGsmPHjsyfPz9J0tXVlcmTJ48ac+DAgTz33HO1MQAAY3qL56tf/WqWLl2a9vb2DA0NZevWrfnud7+bJ554IpVKJd3d3enp6UlHR0c6OjrS09OTadOm5brrrkuSNDU1Zfny5Vm9enVmzJiR5ubmrFmzJnPmzMmiRYvOyAUCABPPmALl1VdfzQ033JADBw6kqakpc+fOzRNPPJHFixcnSW677bYcOXIkK1asSH9/f+bNm5cnn3wyDQ0NtXNs2rQpdXV1WbZsWY4cOZKFCxdmy5YtmTRp0um9MgBgwjrl70EZD2P5HPWp8D0o8PZ8DwpwMj6Q70EBADhTBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFCcMQXK+vXr86u/+qtpaGjIRz/60Vx99dX5j//4j1FjqtVq1q1bl7a2tkydOjULFizI888/P2rM8PBwVq5cmZkzZ2b69Om56qqrsn///lO/GgDgrDCmQNmxY0duueWWPP3009m+fXt++tOfZsmSJTl8+HBtzIYNG7Jx48Zs3rw5u3fvTmtraxYvXpyhoaHamO7u7mzbti1bt27Nzp07c+jQoVxxxRU5duzY6bsyAGDCqlSr1erJPvm1117LRz/60ezYsSOf/exnU61W09bWlu7u7nzlK19J8rPVkpaWlnzta1/LTTfdlIGBgcyaNSsPPfRQrr322iTJK6+8kvb29jz++OO5/PLLT3id4eHhDA8P1x4PDg6mvb09AwMDaWxsPNnpv6euLz94xs4NE1nvnV8Y7ykAE9Dg4GCampre1+/vU7oHZWBgIEnS3NycJNm7d2/6+vqyZMmS2pj6+vpceuml2bVrV5Kkt7c3R48eHTWmra0tnZ2dtTHHW79+fZqammpbe3v7qUwbACjcSQdKtVrNqlWr8pu/+Zvp7OxMkvT19SVJWlpaRo1taWmpHevr68uUKVNy3nnnveOY461duzYDAwO1bd++fSc7bQBgAqg72Sd+6Utfyo9+9KPs3LnzhGOVSmXU42q1esK+473bmPr6+tTX15/sVAGACeakVlBWrlyZb33rW/nXf/3XnH/++bX9ra2tSXLCSsjBgwdrqyqtra0ZGRlJf3//O44BAD7cxhQo1Wo1X/rSl/IP//AP+Zd/+ZfMnj171PHZs2entbU127dvr+0bGRnJjh07Mn/+/CRJV1dXJk+ePGrMgQMH8txzz9XGAAAfbmN6i+eWW27JI488kn/8x39MQ0NDbaWkqakpU6dOTaVSSXd3d3p6etLR0ZGOjo709PRk2rRpue6662pjly9fntWrV2fGjBlpbm7OmjVrMmfOnCxatOj0XyEAMOGMKVDuvffeJMmCBQtG7f+bv/mbfPGLX0yS3HbbbTly5EhWrFiR/v7+zJs3L08++WQaGhpq4zdt2pS6urosW7YsR44cycKFC7Nly5ZMmjTp1K4GADgrnNL3oIyXsXyO+lT4HhR4e74HBTgZH9j3oAAAnAkCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKI5AAQCKI1AAgOIIFACgOAIFACiOQAEAiiNQAIDiCBQAoDgCBQAojkABAIojUACA4ggUAKA4AgUAKM6YA+V73/terrzyyrS1taVSqeSxxx4bdbxarWbdunVpa2vL1KlTs2DBgjz//POjxgwPD2flypWZOXNmpk+fnquuuir79+8/pQsBAM4eYw6Uw4cP56KLLsrmzZvf9viGDRuycePGbN68Obt3705ra2sWL16coaGh2pju7u5s27YtW7duzc6dO3Po0KFcccUVOXbs2MlfCQBw1qgb6xOWLl2apUuXvu2xarWau+66K7fffnuuueaaJMkDDzyQlpaWPPLII7npppsyMDCQ+++/Pw899FAWLVqUJHn44YfT3t6ep556KpdffvkpXA4AcDY4rfeg7N27N319fVmyZEltX319fS699NLs2rUrSdLb25ujR4+OGtPW1pbOzs7amOMNDw9ncHBw1AYAnL1Oa6D09fUlSVpaWkbtb2lpqR3r6+vLlClTct55573jmOOtX78+TU1Nta29vf10ThsAKMwZ+RRPpVIZ9bharZ6w73jvNmbt2rUZGBiobfv27TttcwUAynNaA6W1tTVJTlgJOXjwYG1VpbW1NSMjI+nv73/HMcerr69PY2PjqA0AOHud1kCZPXt2Wltbs3379tq+kZGR7NixI/Pnz0+SdHV1ZfLkyaPGHDhwIM8991xtDADw4TbmT/EcOnQoL730Uu3x3r17s2fPnjQ3N+eCCy5Id3d3enp60tHRkY6OjvT09GTatGm57rrrkiRNTU1Zvnx5Vq9enRkzZqS5uTlr1qzJnDlzap/qAQA+3MYcKM8880wuu+yy2uNVq1YlSW688cZs2bIlt912W44cOZIVK1akv78/8+bNy5NPPpmGhobaczZt2pS6urosW7YsR44cycKFC7Nly5ZMmjTpNFwSADDRVarVanW8JzFWg4ODaWpqysDAwBm9H6Xryw+esXPDRNZ75xfGewrABDSW39/+Fg8AUByBAgAUR6AAAMUZ802yAHA63HrrrXnttdeSJLNmzcrdd989zjOiJAIFgHHx2muv5dVXXx3vaVAob/EAAMURKABAcQQKAFAcgQIAFMdNssCH0st/Ome8p/Ch99OfzEgy6f/9+xX/TQpxwf96drynkMQKCgBQIIECABRHoAAAxREoAEBxBAoAUByf4gFgXDTXH3vbf0MiUAAYJ1/99E/GewoUzFs8AEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQHIECABRHoAAAxREoAEBxxjVQ7rnnnsyePTvnnHNOurq68v3vf388pwMAFGLcAuWb3/xmuru7c/vtt+eHP/xhPvOZz2Tp0qV5+eWXx2tKAEAhxi1QNm7cmOXLl+eP/uiP8slPfjJ33XVX2tvbc++9947XlACAQtSNx4uOjIykt7c3f/zHfzxq/5IlS7Jr164Txg8PD2d4eLj2eGBgIEkyODh4Rud5bPjIGT0/TFRn+mfvgzD0f46N9xSgSGfy5/utc1er1fccOy6B8vrrr+fYsWNpaWkZtb+lpSV9fX0njF+/fn3uuOOOE/a3t7efsTkC76zpf9883lMAzpT1TWf8JYaGhtLU9O6vMy6B8pZKpTLqcbVaPWFfkqxduzarVq2qPX7zzTfzxhtvZMaMGW87nrPL4OBg2tvbs2/fvjQ2No73dIDTyM/3h0u1Ws3Q0FDa2trec+y4BMrMmTMzadKkE1ZLDh48eMKqSpLU19envr5+1L5zzz33TE6RAjU2NvofGJyl/Hx/eLzXyslbxuUm2SlTpqSrqyvbt28ftX/79u2ZP3/+eEwJACjIuL3Fs2rVqtxwww25+OKLc8kll+S+++7Lyy+/nJtv9t42AHzYjVugXHvttfnxj3+cP/3TP82BAwfS2dmZxx9/PBdeeOF4TYlC1dfX50/+5E9OeJsPmPj8fPNOKtX381kfAIAPkL/FAwAUR6AAAMURKABAcQQKAFAcgULx7rnnnsyePTvnnHNOurq68v3vf3+8pwScou9973u58sor09bWlkqlkscee2y8p0RhBApF++Y3v5nu7u7cfvvt+eEPf5jPfOYzWbp0aV5++eXxnhpwCg4fPpyLLroomzdvHu+pUCgfM6Zo8+bNy6/8yq/k3nvvre375Cc/mauvvjrr168fx5kBp0ulUsm2bdty9dVXj/dUKIgVFIo1MjKS3t7eLFmyZNT+JUuWZNeuXeM0KwA+CAKFYr3++us5duzYCX9AsqWl5YQ/NAnA2UWgULxKpTLqcbVaPWEfAGcXgUKxZs6cmUmTJp2wWnLw4METVlUAOLsIFIo1ZcqUdHV1Zfv27aP2b9++PfPnzx+nWQHwQRi3v2YM78eqVatyww035OKLL84ll1yS++67Ly+//HJuvvnm8Z4acAoOHTqUl156qfZ479692bNnT5qbm3PBBReM48wohY8ZU7x77rknGzZsyIEDB9LZ2ZlNmzbls5/97HhPCzgF3/3ud3PZZZedsP/GG2/Mli1bPvgJURyBAgAUxz0oAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESjAB6ZSqeSxxx4b72kAE4BAAU6bvr6+rFy5Mh/72MdSX1+f9vb2XHnllfnOd74z3lMDJhh/LBA4Lf7rv/4rv/Ebv5Fzzz03GzZsyNy5c3P06NF8+9vfzi233JJ///d/PyOve/To0UyePPmMnBsYP1ZQgNNixYoVqVQq+cEPfpDPfe5z+YVf+IV86lOfyqpVq/L000/Xxr3++uv5vd/7vUybNi0dHR351re+VTu2ZcuWnHvuuaPO+9hjj6VSqdQer1u3Lr/8y7+cv/7rv66t1FSr1VQqlfzVX/3VO54bmFgECnDK3njjjTzxxBO55ZZbMn369BOO///Rcccdd2TZsmX50Y9+lN/+7d/O9ddfnzfeeGNMr/fSSy/l7//+7/Poo49mz549p/XcQBkECnDKXnrppVSr1fziL/7ie4794he/mM9//vP5+Mc/np6enhw+fDg/+MEPxvR6IyMjeeihh/LpT386c+fOra2wnI5zA2UQKMApq1arSTLqrZh3Mnfu3Nq/p0+fnoaGhhw8eHBMr3fhhRdm1qxZZ+TcQBkECnDKOjo6UqlU8sILL7zn2ONvaK1UKnnzzTeTJB/5yEdqsfOWo0ePnnCOt3sb6b3ODUwsAgU4Zc3Nzbn88svzl3/5lzl8+PAJx3/yk5+8r/PMmjUrQ0NDo87x/99jAnx4CBTgtLjnnnty7Nix/Nqv/VoeffTRvPjii3nhhRfy9a9/PZdccsn7Ose8efMybdq0fPWrX81LL72URx55JFu2bDmzEweKJFCA02L27Nn5t3/7t1x22WVZvXp1Ojs7s3jx4nznO9/Jvffe+77O0dzcnIcffjiPP/545syZk7/7u7/LunXrzuzEgSJVqse/4QsAMM6soAAAxREoAEBxBAoAUByBAgAUR6AAAMURKABAcQQKAFAcgQIAFEegAADFESgAQHEECgBQnP8LOky4jZY0NJEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.barplot(x = \"Churn\", y=sdf.Churn.index, data=sdf);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f5749e60-f75d-430a-b8ae-ca2edc9b0c23",
   "metadata": {},
   "outputs": [],
   "source": [
    "#we will be more comfortable if the data is reduced. we translated and visualized pandas df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "3f789376-e642-42fd-b6b5-a996300b270b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a=spark_df.groupby(\"Churn\").count().toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "68296499-6bd2-4cb0-baed-18e58f082005",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Churn</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Churn  count\n",
       "0      1    150\n",
       "1      0    750"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5e4d6606-d359-4c91-ba42-3e6a557268f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "838685b5-e8d6-49fd-a6d8-42ff268ba665",
   "metadata": {},
   "source": [
    "### machine learning and data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0e7fdd4e-c266-464d-ae2e-011bcce69c40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|_c0|           Names| Age|Total_Purchase|Account_Manager|Years|Num_Sites|Churn|\n",
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|  0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|\n",
      "|  1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|\n",
      "|  2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|\n",
      "|  3|   Phillip White|42.0|       8010.76|              0| 6.71|     10.0|    1|\n",
      "|  4|  Cynthia Norton|37.0|       9191.58|              0| 5.56|      9.0|    1|\n",
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df=spark.read.csv(\"./churn.csv\", header=True, inferSchema=True)\n",
    "spark_df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "5fcc0e7c-551a-4959-81fb-8ba1ea2bccba",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df= spark_df.toDF(*[c.lower() for c in spark_df.columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "61e86f9c-d587-4d4f-806d-21ec6d1476c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|_c0|           names| age|total_purchase|account_manager|years|num_sites|churn|\n",
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|  0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|\n",
      "|  1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|\n",
      "|  2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|\n",
      "|  3|   Phillip White|42.0|       8010.76|              0| 6.71|     10.0|    1|\n",
      "|  4|  Cynthia Norton|37.0|       9191.58|              0| 5.56|      9.0|    1|\n",
      "+---+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "8dad8dbc-6bd6-4ccf-a05b-1e9715c54f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df=spark_df.withColumnRenamed(\"_c0\",\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "1ed6710d-603a-44cd-b8ad-c7ef51b99c41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|index|           names| age|total_purchase|account_manager|years|num_sites|churn|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "|    0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|\n",
      "|    1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|\n",
      "|    2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|\n",
      "|    3|   Phillip White|42.0|       8010.76|              0| 6.71|     10.0|    1|\n",
      "|    4|  Cynthia Norton|37.0|       9191.58|              0| 5.56|      9.0|    1|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "4db174cd-7f14-48b6-9642-fe32d4473b5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "900"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7608b2c8-1244-4c6c-a513-30cc7f537384",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(spark_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "6207ec1d-7274-4906-a985-14a327c1aaaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['index',\n",
       " 'names',\n",
       " 'age',\n",
       " 'total_purchase',\n",
       " 'account_manager',\n",
       " 'years',\n",
       " 'num_sites',\n",
       " 'churn']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "3d4633c5-ef05-4974-8de6-bb65a32db69f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+-------------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "|summary|             index|        names|              age|   total_purchase|   account_manager|            years|         num_sites|              churn|\n",
      "+-------+------------------+-------------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "|  count|               900|          900|              900|              900|               900|              900|               900|                900|\n",
      "|   mean|             449.5|         null|41.81666666666667|10062.82403333334|0.4811111111111111| 5.27315555555555| 8.587777777777777|0.16666666666666666|\n",
      "| stddev|259.95191863111916|         null|6.127560416916251|2408.644531858096|0.4999208935073339|1.274449013194616|1.7648355920350969| 0.3728852122772358|\n",
      "|    min|                 0|   Aaron King|             22.0|            100.0|                 0|              1.0|               3.0|                  0|\n",
      "|    max|               899|Zachary Walsh|             65.0|         18026.01|                 1|             9.15|              14.0|                  1|\n",
      "+-------+------------------+-------------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "83530496-ab8e-4614-bc00-5ac0ddbf6aa4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------------+---------------+-----+---------+-----+\n",
      "| age|total_purchase|account_manager|years|num_sites|churn|\n",
      "+----+--------------+---------------+-----+---------+-----+\n",
      "|42.0|       11066.8|              0| 7.22|      8.0|    1|\n",
      "|41.0|      11916.22|              0|  6.5|     11.0|    1|\n",
      "|38.0|      12884.75|              0| 6.67|     12.0|    1|\n",
      "|42.0|       8010.76|              0| 6.71|     10.0|    1|\n",
      "|37.0|       9191.58|              0| 5.56|      9.0|    1|\n",
      "|48.0|      10356.02|              0| 5.12|      8.0|    1|\n",
      "|44.0|      11331.58|              1| 5.23|     11.0|    1|\n",
      "|32.0|       9885.12|              1| 6.92|      9.0|    1|\n",
      "|43.0|       14062.6|              1| 5.46|     11.0|    1|\n",
      "|40.0|       8066.94|              1| 7.11|     11.0|    1|\n",
      "|30.0|      11575.37|              1| 5.22|      8.0|    1|\n",
      "|45.0|       8771.02|              1| 6.64|     11.0|    1|\n",
      "|45.0|       8988.67|              1| 4.84|     11.0|    1|\n",
      "|40.0|       8283.32|              1|  5.1|     13.0|    1|\n",
      "|41.0|       6569.87|              1|  4.3|     11.0|    1|\n",
      "|38.0|      10494.82|              1| 6.81|     12.0|    1|\n",
      "|45.0|       8213.41|              1| 7.35|     11.0|    1|\n",
      "|43.0|      11226.88|              0| 8.08|     12.0|    1|\n",
      "|53.0|       5515.09|              0| 6.85|      8.0|    1|\n",
      "|46.0|        8046.4|              1| 5.69|      8.0|    1|\n",
      "+----+--------------+---------------+-----+---------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.select(\"age\",\"total_purchase\",\"account_manager\",\"years\",\"num_sites\",\"churn\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c842d876-5573-4026-ab8b-4b77b2503cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will model whether the customer will leave or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "7f7f6a3c-9616-4683-9cab-871b939ae6d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>summary</th>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>900</td>\n",
       "      <td>41.81666666666667</td>\n",
       "      <td>6.127560416916251</td>\n",
       "      <td>22.0</td>\n",
       "      <td>65.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_purchase</th>\n",
       "      <td>900</td>\n",
       "      <td>10062.82403333334</td>\n",
       "      <td>2408.644531858096</td>\n",
       "      <td>100.0</td>\n",
       "      <td>18026.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>account_manager</th>\n",
       "      <td>900</td>\n",
       "      <td>0.4811111111111111</td>\n",
       "      <td>0.4999208935073339</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>years</th>\n",
       "      <td>900</td>\n",
       "      <td>5.27315555555555</td>\n",
       "      <td>1.274449013194616</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_sites</th>\n",
       "      <td>900</td>\n",
       "      <td>8.587777777777777</td>\n",
       "      <td>1.7648355920350969</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>churn</th>\n",
       "      <td>900</td>\n",
       "      <td>0.16666666666666666</td>\n",
       "      <td>0.3728852122772358</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0                    1                   2      3  \\\n",
       "summary          count                 mean              stddev    min   \n",
       "age                900    41.81666666666667   6.127560416916251   22.0   \n",
       "total_purchase     900    10062.82403333334   2408.644531858096  100.0   \n",
       "account_manager    900   0.4811111111111111  0.4999208935073339      0   \n",
       "years              900     5.27315555555555   1.274449013194616    1.0   \n",
       "num_sites          900    8.587777777777777  1.7648355920350969    3.0   \n",
       "churn              900  0.16666666666666666  0.3728852122772358      0   \n",
       "\n",
       "                        4  \n",
       "summary               max  \n",
       "age                  65.0  \n",
       "total_purchase   18026.01  \n",
       "account_manager         1  \n",
       "years                9.15  \n",
       "num_sites            14.0  \n",
       "churn                   1  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.select(\"age\",\"total_purchase\",\"account_manager\",\"years\",\"num_sites\",\"churn\").describe().toPandas().transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "a1f04be5-5caa-4e07-9936-936349442de3",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df =spark_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "26ce2368-24e5-452d-aea7-0828f9acc96c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we get rid of incomplete observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "89f53a86-ff56-4158-a50e-86fb42ed78d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df=spark_df.withColumn(\"age_squared\",spark_df.age**2)\n",
    "#variably produced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "ded201f0-814c-4d2e-8fd2-4c9cfdc13f48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+\n",
      "|index|           names| age|total_purchase|account_manager|years|num_sites|churn|age_squared|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+\n",
      "|    0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|     1764.0|\n",
      "|    1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|     1681.0|\n",
      "|    2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|     1444.0|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "123ab3f9-ecef-4885-a84b-e6e90d4a7137",
   "metadata": {},
   "outputs": [],
   "source": [
    "# specifying the dependent variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "0a93208c-1363-49a5-b972-37f64895b785",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import StringIndexer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "0a09d011-6038-4321-96f3-15b07559e20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "stringIndexer=StringIndexer(inputCol=\"churn\",outputCol=\"label\")\n",
    "mod=stringIndexer.fit(spark_df)\n",
    "indexed=mod.transform(spark_df)\n",
    "spark_df=indexed.withColumn(\"label\",indexed[\"label\"].cast(\"integer\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "6eb39a33-c3fe-4756-9283-883ceb54544a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+-----+\n",
      "|index|           names| age|total_purchase|account_manager|years|num_sites|churn|age_squared|label|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+-----+\n",
      "|    0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|     1764.0|    1|\n",
      "|    1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|     1681.0|    1|\n",
      "|    2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|     1444.0|    1|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+-----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "f2e513a2-81f1-4129-aaa4-38dc31565435",
   "metadata": {},
   "outputs": [],
   "source": [
    "# specifying independent variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "529b1960-7d8b-4def-818d-9124b0fb8574",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6fa68f46-bdef-4e3e-8038-48bdb61f7574",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['index',\n",
       " 'names',\n",
       " 'age',\n",
       " 'total_purchase',\n",
       " 'account_manager',\n",
       " 'years',\n",
       " 'num_sites',\n",
       " 'churn',\n",
       " 'age_squared',\n",
       " 'label']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "61962179-1a33-4634-9881-946b496271fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "independent_variables=[\"age\",\"total_purchase\",\"account_manager\",\"years\",\"num_sites\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "3bfdc700-2852-402b-a998-883c959f28bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorAssembler=VectorAssembler(inputCols=independent_variables, outputCol=\"features\")\n",
    "va_df= vectorAssembler.transform(spark_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "833b3288-c46f-499f-b599-0dcfedce6502",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+-----+--------------------+\n",
      "|index|           names| age|total_purchase|account_manager|years|num_sites|churn|age_squared|label|            features|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+-----+--------------------+\n",
      "|    0|Cameron Williams|42.0|       11066.8|              0| 7.22|      8.0|    1|     1764.0|    1|[42.0,11066.8,0.0...|\n",
      "|    1|   Kevin Mueller|41.0|      11916.22|              0|  6.5|     11.0|    1|     1681.0|    1|[41.0,11916.22,0....|\n",
      "|    2|     Eric Lozano|38.0|      12884.75|              0| 6.67|     12.0|    1|     1444.0|    1|[38.0,12884.75,0....|\n",
      "+-----+----------------+----+--------------+---------------+-----+---------+-----+-----------+-----+--------------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "va_df.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "b40b87b3-a978-4995-89f7-e6ba0a0b6b7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "last_df=va_df.select([\"features\",\"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "56f43907-ad11-431d-87ea-70be03272321",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|            features|label|\n",
      "+--------------------+-----+\n",
      "|[42.0,11066.8,0.0...|    1|\n",
      "|[41.0,11916.22,0....|    1|\n",
      "|[38.0,12884.75,0....|    1|\n",
      "+--------------------+-----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "last_df.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "51c19ea9-60db-4361-a2ae-0c159ff5b0b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "0fea2730-4ac6-4e81-b583-49ab9a8dba6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits=last_df.randomSplit([0.70,0.30])\n",
    "train_df=splits[0]\n",
    "test_df=splits[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5214c704-3df5-4e5b-99c5-a9133a6082f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[features: vector, label: int]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "75079eb7-6a86-402a-ae97-392cb1ff898e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[features: vector, label: int]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4aafee5e-b6bf-4885-932d-bf453d8644b9",
   "metadata": {},
   "source": [
    "### customer abandonment modeling using gbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "69001b03-0f7b-4b42-94e4-0530136215b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import GBTClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "9b144767-76aa-41c1-8f7b-213e432a99cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "gbm=GBTClassifier(maxIter=10,featuresCol=\"features\",labelCol=\"label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "3977882d-26d5-459f-b63f-bd91ab548baf",
   "metadata": {},
   "outputs": [],
   "source": [
    "gbm_model=gbm.fit(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "7c1e448b-d807-4d96-9358-5914bfac1062",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://172.25.173.108:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>spark</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local appName=spark>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "009f98f4-112f-4249-9ed2-975b9cd1db19",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=gbm_model.transform(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "1c3c6203-1f49-4fe8-99f0-1c2a3470ae9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[features: vector, label: int, rawPrediction: vector, probability: vector, prediction: double]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "5b0b6dad-d943-4e17-8ca7-90cd1f21243b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ac=y_pred.select(\"label\",\"prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "83b6191e-8e70-4df2-a129-30eb3cfb568d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8450704225352113"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ac.filter(ac.label==ac.prediction).count()/ac.count()\n",
    "#we will compare the real and estimated values and count true false and calculate the accuracy to increase the ac"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d942356-8d89-42a2-bf1b-7305ced487b0",
   "metadata": {},
   "source": [
    "### model tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "d6f44809-004b-4680-b090-2dc63049ec8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
    "\n",
    "evaluator=BinaryClassificationEvaluator()\n",
    "\n",
    "paramGrid=(ParamGridBuilder()\n",
    "           .addGrid(gbm.maxDepth,[2,4,6])\n",
    "           .addGrid(gbm.maxBins,[20,30])\n",
    "           .addGrid(gbm.maxIter,[10,20])\n",
    "           .build())\n",
    "\n",
    "cv=CrossValidator(estimator=gbm, estimatorParamMaps=paramGrid, evaluator=evaluator, numFolds=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "6cd57415-2ee1-4051-81f9-7526ab08c0d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cvModel=cv.fit(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "9ae95d91-db10-4e0d-86c8-5c94849f6d1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=cvModel.transform(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "90a4a6e9-e160-4710-babd-f14ecd311d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ac=y_pred.select(\"label\",\"prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "ab942846-2229-4924-841b-725c0f02b102",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8802816901408451"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ac.filter(ac.label==ac.prediction).count()/ac.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "425ab8c1-b221-405d-98ed-def53a5988e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# will the new customer leave or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "7a7abde1-0ff1-4cea-b3b8-097516692a9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['names', 'age', 'total_purchase', 'account_manager', 'years',\n",
       "       'num_sites'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "names=pd.Series([\"Murat Karahan\",\"Elif Yamac\",\"Enes Gokdemir\",\"Eren Torun\",\"Ertugrul Yerlikaya\"])\n",
    "age=pd.Series([45,20,21,20,20])\n",
    "total_purchase=pd.Series([30000,10000,60000,30000,100000])\n",
    "account_manager=pd.Series([1,0,0,1,1])\n",
    "years=pd.Series([20,10,3,8,3])\n",
    "num_sites=pd.Series([30,8,8,6,50])\n",
    "\n",
    "new_customer=pd.DataFrame({\n",
    "\n",
    "    'names':names,\n",
    "    'age':age,\n",
    "    'total_purchase':total_purchase,\n",
    "    'account_manager':account_manager,\n",
    "    'years':years,\n",
    "    'num_sites':num_sites})\n",
    "\n",
    "new_customer.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "d7c22140-54f3-403b-b257-75dff7a1cbe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_sdf=spark.createDataFrame(new_customer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "cda34a78-ee28-4157-a311-b76e7860e0c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+---+--------------+---------------+-----+---------+\n",
      "|             names|age|total_purchase|account_manager|years|num_sites|\n",
      "+------------------+---+--------------+---------------+-----+---------+\n",
      "|     Murat Karahan| 38|         30000|              1|   20|       30|\n",
      "|        Elif Yamac| 53|         10000|              0|   10|        8|\n",
      "|     Enes Gokdemir| 34|          6000|              0|    3|        8|\n",
      "|        Eren Torun| 50|         30000|              1|    8|        6|\n",
      "|Ertugrul Yerlikaya| 40|        100000|              1|   30|       50|\n",
      "+------------------+---+--------------+---------------+-----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "new_sdf.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "a12575cf-227f-4a47-9459-a317ee4b315f",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_customer=vectorAssembler.transform(new_sdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "049b0996-de15-4237-a976-b96b65f321e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "results=cvModel.transform(new_customer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "faa2cf57-307b-4fa7-8fcc-07cc54f1d29d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+----------+\n",
      "|             names|prediction|\n",
      "+------------------+----------+\n",
      "|     Murat Karahan|       1.0|\n",
      "|        Elif Yamac|       0.0|\n",
      "|     Enes Gokdemir|       0.0|\n",
      "|        Eren Torun|       0.0|\n",
      "|Ertugrul Yerlikaya|       1.0|\n",
      "+------------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results.select(\"names\",\"prediction\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "c574b888-77a6-4a17-931c-acc6063cc800",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the model was translated into a vector and independent and dependent variable estimation was performed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4541d7f1-448f-44cc-9c12-e1267911cffc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}