spark1_preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BD-JDeoGrG9F"
      },
      "source": [
        "# Preprocessing using pyspark"
      ],
      "id": "BD-JDeoGrG9F"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o3XWw4EArDo_",
        "outputId": "2792b9d1-6b37-4771-ea2e-7e75bba5c5f2"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null # install Java8\n",
        "!wget -q https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop2.7.tgz # download spark 3.1.1\n",
        "!tar xf spark-3.1.1-bin-hadoop2.7.tgz # unzip it\n",
        "!pip install -q findspark # install findspark\n",
        "!pip install pyspark==3.1.1 # install pyspark\n",
        "\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.1-bin-hadoop2.7\"\n",
        "\n",
        "import pyspark\n",
        "from pyspark import SparkConf\n",
        "from pyspark.sql import SparkSession"
      ],
      "id": "o3XWw4EArDo_",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyspark==3.1.1 in /usr/local/lib/python3.10/dist-packages (3.1.1)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.10/dist-packages (from pyspark==3.1.1) (0.10.9)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6h695oKCaMin",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "outputId": "a87c2963-4b0f-4656-f8d0-af5a15bf7545"
      },
      "source": [
        "spark = SparkSession.builder.master(\"local\").config(conf = SparkConf()).getOrCreate()\n",
        "spark"
      ],
      "id": "6h695oKCaMin",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7d631812f070>"
            ],
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://d37aa85e7552:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fopgTrWuaMil"
      },
      "source": [
        "### Read data and perform basic operations"
      ],
      "id": "fopgTrWuaMil"
    },
    {
      "cell_type": "code",
      "source": [
        "df = spark.read.csv(path = \"/content/employee.txt\", header = True, inferSchema = True)\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iypwq7PzVHlS",
        "outputId": "07e8a442-1689-4d86-dd7b-c245eca5e7f3"
      },
      "id": "iypwq7PzVHlS",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[id: int, last_name: string, email: string, gender: string, department: string, start_date: string, salary: int, job_title: string, region_id: int]"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ALq4RURbaMio",
        "outputId": "5633de31-d9a2-4bbe-c590-2adf93b20b51"
      },
      "source": [
        "df.schema"
      ],
      "id": "ALq4RURbaMio",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StructType(List(StructField(id,IntegerType,true),StructField(last_name,StringType,true),StructField(email,StringType,true),StructField(gender,StringType,true),StructField(department,StringType,true),StructField(start_date,StringType,true),StructField(salary,IntegerType,true),StructField(job_title,StringType,true),StructField(region_id,IntegerType,true)))"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jo1R5qTbaMio",
        "outputId": "0833534d-1159-41f4-da1d-b493e9102e51"
      },
      "source": [
        "df.printSchema()"
      ],
      "id": "jo1R5qTbaMio",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- id: integer (nullable = true)\n",
            " |-- last_name: string (nullable = true)\n",
            " |-- email: string (nullable = true)\n",
            " |-- gender: string (nullable = true)\n",
            " |-- department: string (nullable = true)\n",
            " |-- start_date: string (nullable = true)\n",
            " |-- salary: integer (nullable = true)\n",
            " |-- job_title: string (nullable = true)\n",
            " |-- region_id: integer (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jyx3r7JDaMip",
        "outputId": "e0b36ea4-eb0e-4d97-d9b8-733c638623d4"
      },
      "source": [
        "df.columns"
      ],
      "id": "jyx3r7JDaMip",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['id',\n",
              " 'last_name',\n",
              " 'email',\n",
              " 'gender',\n",
              " 'department',\n",
              " 'start_date',\n",
              " 'salary',\n",
              " 'job_title',\n",
              " 'region_id']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y4whAyAOaMip",
        "outputId": "dae1e764-10b2-4e6b-f802-3baf110f060e"
      },
      "source": [
        "df.take(5)"
      ],
      "id": "Y4whAyAOaMip",
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=1, last_name=\"'Kelley'\", email=\"'rkelley0@soundcloud.com'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'10/2/2009'\", salary=67470, job_title=\"'Structural Engineer'\", region_id=2),\n",
              " Row(id=2, last_name=\"'Armstrong'\", email=\"'sarmstrong1@infoseek.co.jp'\", gender=\"'Male'\", department=\"'Sports'\", start_date=\"'3/31/2008'\", salary=71869, job_title=\"'Financial Advisor'\", region_id=2),\n",
              " Row(id=3, last_name=\"'Carr'\", email=\"'fcarr2@woothemes.com'\", gender=\"'Male'\", department=\"'Automotive'\", start_date=\"'7/12/2009'\", salary=101768, job_title=\"'Recruiting Manager'\", region_id=3),\n",
              " Row(id=4, last_name=\"'Murray'\", email=\"'jmurray3@gov.uk'\", gender=\"'Female'\", department=\"'Jewelery'\", start_date=\"'12/25/2014'\", salary=96897, job_title=\"'Desktop Support Technician'\", region_id=3),\n",
              " Row(id=5, last_name=\"'Ellis'\", email=\"'jellis4@sciencedirect.com'\", gender=\"'Female'\", department=\"'Grocery'\", start_date=\"'9/19/2002'\", salary=63702, job_title=\"'Software Engineer III'\", region_id=7)]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 186
        },
        "id": "oZLrtu_FxVpp",
        "outputId": "467170c9-387a-447c-9fa6-f5841f855d5e"
      },
      "source": [
        "type(df.take(5)[0])"
      ],
      "id": "oZLrtu_FxVpp",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pyspark.sql.types.Row"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>pyspark.sql.types.Row</b><br/>def __call__(*args)</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/pyspark/sql/types.py</a>A row in :class:`DataFrame`.\n",
              "The fields in it can be accessed:\n",
              "\n",
              "* like attributes (``row.key``)\n",
              "* like dictionary values (``row[key]``)\n",
              "\n",
              "``key in row`` will search through row keys.\n",
              "\n",
              "Row can be used to create a row object by using named arguments.\n",
              "It is not allowed to omit a named argument to represent that the value is\n",
              "None or missing. This should be explicitly set to None in this case.\n",
              "\n",
              ".. versionchanged:: 3.0.0\n",
              "    Rows created from named arguments no longer have\n",
              "    field names sorted alphabetically and will be ordered in the position as\n",
              "    entered.\n",
              "\n",
              "Examples\n",
              "--------\n",
              "&gt;&gt;&gt; row = Row(name=&quot;Alice&quot;, age=11)\n",
              "&gt;&gt;&gt; row\n",
              "Row(name=&#x27;Alice&#x27;, age=11)\n",
              "&gt;&gt;&gt; row[&#x27;name&#x27;], row[&#x27;age&#x27;]\n",
              "(&#x27;Alice&#x27;, 11)\n",
              "&gt;&gt;&gt; row.name, row.age\n",
              "(&#x27;Alice&#x27;, 11)\n",
              "&gt;&gt;&gt; &#x27;name&#x27; in row\n",
              "True\n",
              "&gt;&gt;&gt; &#x27;wrong_key&#x27; in row\n",
              "False\n",
              "\n",
              "Row also can be used to create another Row like class, then it\n",
              "could be used to create Row objects, such as\n",
              "\n",
              "&gt;&gt;&gt; Person = Row(&quot;name&quot;, &quot;age&quot;)\n",
              "&gt;&gt;&gt; Person\n",
              "&lt;Row(&#x27;name&#x27;, &#x27;age&#x27;)&gt;\n",
              "&gt;&gt;&gt; &#x27;name&#x27; in Person\n",
              "True\n",
              "&gt;&gt;&gt; &#x27;wrong_key&#x27; in Person\n",
              "False\n",
              "&gt;&gt;&gt; Person(&quot;Alice&quot;, 11)\n",
              "Row(name=&#x27;Alice&#x27;, age=11)\n",
              "\n",
              "This form can also be used to create rows as tuple values, i.e. with unnamed\n",
              "fields.\n",
              "\n",
              "&gt;&gt;&gt; row1 = Row(&quot;Alice&quot;, 11)\n",
              "&gt;&gt;&gt; row2 = Row(name=&quot;Alice&quot;, age=11)\n",
              "&gt;&gt;&gt; row1 == row2\n",
              "True</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 1425);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aA4omfLaaMiq",
        "outputId": "722778e9-b14c-420d-afb5-1f109908e7e8"
      },
      "source": [
        "df.head(5)"
      ],
      "id": "aA4omfLaaMiq",
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=1, last_name=\"'Kelley'\", email=\"'rkelley0@soundcloud.com'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'10/2/2009'\", salary=67470, job_title=\"'Structural Engineer'\", region_id=2),\n",
              " Row(id=2, last_name=\"'Armstrong'\", email=\"'sarmstrong1@infoseek.co.jp'\", gender=\"'Male'\", department=\"'Sports'\", start_date=\"'3/31/2008'\", salary=71869, job_title=\"'Financial Advisor'\", region_id=2),\n",
              " Row(id=3, last_name=\"'Carr'\", email=\"'fcarr2@woothemes.com'\", gender=\"'Male'\", department=\"'Automotive'\", start_date=\"'7/12/2009'\", salary=101768, job_title=\"'Recruiting Manager'\", region_id=3),\n",
              " Row(id=4, last_name=\"'Murray'\", email=\"'jmurray3@gov.uk'\", gender=\"'Female'\", department=\"'Jewelery'\", start_date=\"'12/25/2014'\", salary=96897, job_title=\"'Desktop Support Technician'\", region_id=3),\n",
              " Row(id=5, last_name=\"'Ellis'\", email=\"'jellis4@sciencedirect.com'\", gender=\"'Female'\", department=\"'Grocery'\", start_date=\"'9/19/2002'\", salary=63702, job_title=\"'Software Engineer III'\", region_id=7)]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GcoybvTPaMiq",
        "outputId": "ead43267-8167-41fe-ceb9-4fc44a9cf24e"
      },
      "source": [
        "df.tail(5)"
      ],
      "id": "GcoybvTPaMiq",
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=996, last_name=\"'James'\", email=\"'tjamesrn@soundcloud.com'\", gender=\"'Female'\", department=\"'Games'\", start_date=\"'11/17/2013'\", salary=78433, job_title=\"'Accountant II'\", region_id=7),\n",
              " Row(id=997, last_name=\"'Reynolds'\", email=\"'dreynoldsro@blogtalkradio.com'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'4/16/2007'\", salary=120138, job_title=\"'Statistician IV'\", region_id=1),\n",
              " Row(id=998, last_name=\"'Walker'\", email=\"'kwalkerrp@unicef.org'\", gender=\"'Female'\", department=\"'Games'\", start_date=\"'2/13/2010'\", salary=60363, job_title=\"'Account Coordinator'\", region_id=1),\n",
              " Row(id=999, last_name=\"'Kennedy'\", email=\"'lkennedyrq@edublogs.org'\", gender=\"'Male'\", department=\"'Industrial'\", start_date=\"'9/22/2004'\", salary=48050, job_title=\"'Graphic Designer'\", region_id=2),\n",
              " Row(id=1000, last_name=\"'Howard'\", email=\"'showardrr@addtoany.com'\", gender=\"'Male'\", department=\"'Baby'\", start_date=\"'11/7/2003'\", salary=148687, job_title=\"'General Manager'\", region_id=3)]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C5KNz1gmaMir",
        "outputId": "5d510d68-1107-4e61-e949-b3c692777c23"
      },
      "source": [
        "df.count()"
      ],
      "id": "C5KNz1gmaMir",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1000"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AyBWev66aMir",
        "outputId": "1d46b0b3-1860-48ed-9e34-5de86d474b89"
      },
      "source": [
        "df.describe().show()"
      ],
      "id": "AyBWev66aMir",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----------------+---------+--------------------+--------+------------+----------+------------------+--------------------+-----------------+\n",
            "|summary|               id|last_name|               email|  gender|  department|start_date|            salary|           job_title|        region_id|\n",
            "+-------+-----------------+---------+--------------------+--------+------------+----------+------------------+--------------------+-----------------+\n",
            "|  count|             1000|     1000|                1000|    1000|        1000|      1000|              1000|                1000|             1000|\n",
            "|   mean|            500.5|     null|                null|    null|        null|      null|         97331.223|                null|            3.971|\n",
            "| stddev|288.8194360957494|     null|                null|    null|        null|      null|31753.109191299463|                null|2.017977014544822|\n",
            "|    min|                1|  'Adams'|'aalexandero8@his...|'Female'|'Automotive'|'1/1/2004'|             40138|'Account Coordina...|                1|\n",
            "|    max|             1000|  'Young'|'wwilliams2o@berk...|  'Male'|      'Toys'|'9/9/2014'|            149929|  'Web Developer IV'|                7|\n",
            "+-------+-----------------+---------+--------------------+--------+------------+----------+------------------+--------------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s-NvcF0RaMir",
        "outputId": "a21e90f9-de70-4f54-84aa-526c90a10e5a"
      },
      "source": [
        "df.select('id', 'last_name', 'salary').describe().show()"
      ],
      "id": "s-NvcF0RaMir",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----------------+---------+------------------+\n",
            "|summary|               id|last_name|            salary|\n",
            "+-------+-----------------+---------+------------------+\n",
            "|  count|             1000|     1000|              1000|\n",
            "|   mean|            500.5|     null|         97331.223|\n",
            "| stddev|288.8194360957494|     null|31753.109191299463|\n",
            "|    min|                1|  'Adams'|             40138|\n",
            "|    max|             1000|  'Young'|            149929|\n",
            "+-------+-----------------+---------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIMF_rmfaMis",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dd1451d3-5163-454a-cf18-bf3face8c9d1"
      },
      "source": [
        "df_sample = df.sample(withReplacement = False, fraction = 0.1, seed = 0)\n",
        "df_sample"
      ],
      "id": "nIMF_rmfaMis",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[id: int, last_name: string, email: string, gender: string, department: string, start_date: string, salary: int, job_title: string, region_id: int]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uXuzkvbhaMis",
        "outputId": "21bde604-4345-4560-b5f4-6e08e9b289f3"
      },
      "source": [
        "df_sample.count()"
      ],
      "id": "uXuzkvbhaMis",
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "103"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U29rbJMMaMit",
        "outputId": "c489b4a3-4402-449b-9b7a-30f191cdf57a"
      },
      "source": [
        "df_sample.head(5)"
      ],
      "id": "U29rbJMMaMit",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=3, last_name=\"'Carr'\", email=\"'fcarr2@woothemes.com'\", gender=\"'Male'\", department=\"'Automotive'\", start_date=\"'7/12/2009'\", salary=101768, job_title=\"'Recruiting Manager'\", region_id=3),\n",
              " Row(id=10, last_name=\"'Sanchez'\", email=\"'rsanchez9@cloudflare.com'\", gender=\"'Male'\", department=\"'Movies'\", start_date=\"'3/13/2013'\", salary=108093, job_title=\"'Sales Representative'\", region_id=1),\n",
              " Row(id=17, last_name=\"'Morgan'\", email=\"'dmorgang@123-reg.co.uk'\", gender=\"'Female'\", department=\"'Kids'\", start_date=\"'5/4/2011'\", salary=148952, job_title=\"'Programmer IV'\", region_id=6),\n",
              " Row(id=27, last_name=\"'Owens'\", email=\"'cowensq@shareasale.com'\", gender=\"'Female'\", department=\"'Home'\", start_date=\"'5/26/2005'\", salary=78698, job_title=\"'Executive Secretary'\", region_id=2),\n",
              " Row(id=47, last_name=\"'Oliver'\", email=\"'joliver1a@cnbc.com'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'8/30/2013'\", salary=42797, job_title=\"'Software Engineer III'\", region_id=5)]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "myMaVi_daMit",
        "outputId": "ad7fd965-e6c7-4395-ff07-44c48fa12a06"
      },
      "source": [
        "df_sample.describe().show()"
      ],
      "id": "myMaVi_daMit",
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+-----------+--------------------+--------+------------+-----------+------------------+-------------------+------------------+\n",
            "|summary|                id|  last_name|               email|  gender|  department| start_date|            salary|          job_title|         region_id|\n",
            "+-------+------------------+-----------+--------------------+--------+------------+-----------+------------------+-------------------+------------------+\n",
            "|  count|               103|        103|                 103|     103|         103|        103|               103|                103|               103|\n",
            "|   mean| 473.3883495145631|       null|                null|    null|        null|       null|103965.12621359223|               null| 3.766990291262136|\n",
            "| stddev|288.20340151922073|       null|                null|    null|        null|       null| 30953.85006484419|               null|1.9813042025499445|\n",
            "|    min|                 3|'Alexander'|'aalexandero8@his...|'Female'|'Automotive'|'1/12/2007'|             40218|'Account Executive'|                 1|\n",
            "|    max|               962|    'Young'|'vgarcia1m@devian...|  'Male'|      'Toys'|'9/29/2012'|            149835| 'Web Developer IV'|                 7|\n",
            "+-------+------------------+-----------+--------------------+--------+------------+-----------+------------------+-------------------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qNIVHhyhzwr-",
        "outputId": "14b7f5f1-9465-4f23-d018-b0681a69ff30"
      },
      "source": [
        "df_sample.collect()"
      ],
      "id": "qNIVHhyhzwr-",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=3, last_name=\"'Carr'\", email=\"'fcarr2@woothemes.com'\", gender=\"'Male'\", department=\"'Automotive'\", start_date=\"'7/12/2009'\", salary=101768, job_title=\"'Recruiting Manager'\", region_id=3),\n",
              " Row(id=10, last_name=\"'Sanchez'\", email=\"'rsanchez9@cloudflare.com'\", gender=\"'Male'\", department=\"'Movies'\", start_date=\"'3/13/2013'\", salary=108093, job_title=\"'Sales Representative'\", region_id=1),\n",
              " Row(id=17, last_name=\"'Morgan'\", email=\"'dmorgang@123-reg.co.uk'\", gender=\"'Female'\", department=\"'Kids'\", start_date=\"'5/4/2011'\", salary=148952, job_title=\"'Programmer IV'\", region_id=6),\n",
              " Row(id=27, last_name=\"'Owens'\", email=\"'cowensq@shareasale.com'\", gender=\"'Female'\", department=\"'Home'\", start_date=\"'5/26/2005'\", salary=78698, job_title=\"'Executive Secretary'\", region_id=2),\n",
              " Row(id=47, last_name=\"'Oliver'\", email=\"'joliver1a@cnbc.com'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'8/30/2013'\", salary=42797, job_title=\"'Software Engineer III'\", region_id=5),\n",
              " Row(id=58, last_name=\"'Baker'\", email=\"'jbaker1l@usnews.com'\", gender=\"'Male'\", department=\"'Games'\", start_date=\"'3/2/2007'\", salary=68857, job_title=\"'Information Systems Manager'\", region_id=2),\n",
              " Row(id=59, last_name=\"'Garcia'\", email=\"'vgarcia1m@deviantart.com'\", gender=\"'Female'\", department=\"'Health'\", start_date=\"'3/19/2002'\", salary=48360, job_title=\"'Registered Nurse'\", region_id=6),\n",
              " Row(id=67, last_name=\"'Mills'\", email=\"'jmills1u@printfriendly.com'\", gender=\"'Female'\", department=\"'Grocery'\", start_date=\"'8/8/2004'\", salary=114200, job_title=\"'Senior Sales Associate'\", region_id=7),\n",
              " Row(id=80, last_name=\"'Little'\", email=\"'klittle27@un.org'\", gender=\"'Male'\", department=\"'Music'\", start_date=\"'9/12/2001'\", salary=114647, job_title=\"'Editor'\", region_id=5),\n",
              " Row(id=93, last_name=\"'Knight'\", email=\"'nknight2k@cafepress.com'\", gender=\"'Female'\", department=\"'Electronics'\", start_date=\"'4/3/2014'\", salary=125847, job_title=\"'Chemical Engineer'\", region_id=5),\n",
              " Row(id=115, last_name=\"'Nguyen'\", email=\"'rnguyen36@bandcamp.com'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'6/30/2007'\", salary=58420, job_title=\"'Graphic Designer'\", region_id=1),\n",
              " Row(id=127, last_name=\"'Harper'\", email=\"'sharper3i@slideshare.net'\", gender=\"'Female'\", department=\"'Outdoors'\", start_date=\"'5/18/2000'\", salary=144639, job_title=\"'VP Quality Control'\", region_id=7),\n",
              " Row(id=139, last_name=\"'Hart'\", email=\"'rhart3u@wsj.com'\", gender=\"'Male'\", department=\"'Jewelery'\", start_date=\"'1/8/2012'\", salary=119773, job_title=\"'Pharmacist'\", region_id=3),\n",
              " Row(id=141, last_name=\"'Williamson'\", email=\"'rwilliamson3w@amazon.co.jp'\", gender=\"'Male'\", department=\"'Games'\", start_date=\"'12/12/2005'\", salary=145575, job_title=\"'VP Accounting'\", region_id=1),\n",
              " Row(id=146, last_name=\"'Thompson'\", email=\"'athompson41@booking.com'\", gender=\"'Male'\", department=\"'Movies'\", start_date=\"'2/4/2009'\", salary=133281, job_title=\"'Account Executive'\", region_id=6),\n",
              " Row(id=149, last_name=\"'Meyer'\", email=\"'jmeyer44@facebook.com'\", gender=\"'Female'\", department=\"'Toys'\", start_date=\"'5/3/2008'\", salary=76352, job_title=\"'Human Resources Assistant IV'\", region_id=4),\n",
              " Row(id=171, last_name=\"'Harris'\", email=\"'mharris4q@squarespace.com'\", gender=\"'Female'\", department=\"'Movies'\", start_date=\"'9/27/2004'\", salary=110912, job_title=\"'Geologist I'\", region_id=5),\n",
              " Row(id=178, last_name=\"'Butler'\", email=\"'cbutler4x@amazon.de'\", gender=\"'Female'\", department=\"'Sports'\", start_date=\"'2/17/2003'\", salary=78157, job_title=\"'Librarian'\", region_id=1),\n",
              " Row(id=182, last_name=\"'Gomez'\", email=\"'rgomez51@baidu.com'\", gender=\"'Female'\", department=\"'Industrial'\", start_date=\"'4/4/2009'\", salary=76932, job_title=\"'Legal Assistant'\", region_id=5),\n",
              " Row(id=185, last_name=\"'Watson'\", email=\"'rwatson54@webeden.co.uk'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'9/22/2001'\", salary=74648, job_title=\"'Health Coach III'\", region_id=2),\n",
              " Row(id=190, last_name=\"'Powell'\", email=\"'hpowell59@geocities.com'\", gender=\"'Female'\", department=\"'Health'\", start_date=\"'11/13/2002'\", salary=120577, job_title=\"'Web Designer III'\", region_id=1),\n",
              " Row(id=204, last_name=\"'Powell'\", email=\"'cpowell5n@cocolog-nifty.com'\", gender=\"'Female'\", department=\"'Beauty'\", start_date=\"'9/25/2000'\", salary=116013, job_title=\"'Tax Accountant'\", region_id=4),\n",
              " Row(id=205, last_name=\"'Graham'\", email=\"'jgraham5o@printfriendly.com'\", gender=\"'Female'\", department=\"'Games'\", start_date=\"'9/10/2008'\", salary=109058, job_title=\"'Chemical Engineer'\", region_id=7),\n",
              " Row(id=206, last_name=\"'Riley'\", email=\"'hriley5p@dailymotion.com'\", gender=\"'Male'\", department=\"'Toys'\", start_date=\"'5/6/2012'\", salary=61992, job_title=\"'Sales Representative'\", region_id=4),\n",
              " Row(id=207, last_name=\"'Harris'\", email=\"'mharris5q@utexas.edu'\", gender=\"'Female'\", department=\"'Movies'\", start_date=\"'10/2/2008'\", salary=129527, job_title=\"'Statistician II'\", region_id=2),\n",
              " Row(id=214, last_name=\"'West'\", email=\"'gwest5x@hao123.com'\", gender=\"'Male'\", department=\"'Computers'\", start_date=\"'10/31/2003'\", salary=66084, job_title=\"'Electrical Engineer'\", region_id=6),\n",
              " Row(id=226, last_name=\"'Gonzalez'\", email=\"'tgonzalez69@umich.edu'\", gender=\"'Male'\", department=\"'Health'\", start_date=\"'11/12/2005'\", salary=106833, job_title=\"'Registered Nurse'\", region_id=7),\n",
              " Row(id=238, last_name=\"'Gonzales'\", email=\"'cgonzales6l@berkeley.edu'\", gender=\"'Male'\", department=\"'Movies'\", start_date=\"'4/6/2014'\", salary=131274, job_title=\"'Clinical Specialist'\", region_id=2),\n",
              " Row(id=239, last_name=\"'Williamson'\", email=\"'jwilliamson6m@arizona.edu'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'2/16/2005'\", salary=149085, job_title=\"'Software Engineer III'\", region_id=7),\n",
              " Row(id=252, last_name=\"'Johnston'\", email=\"'jjohnston6z@irs.gov'\", gender=\"'Male'\", department=\"'Tools'\", start_date=\"'12/16/2000'\", salary=87072, job_title=\"'Structural Engineer'\", region_id=7),\n",
              " Row(id=253, last_name=\"'Marshall'\", email=\"'mmarshall70@hibu.com'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'5/12/2003'\", salary=123401, job_title=\"'Computer Systems Analyst I'\", region_id=4),\n",
              " Row(id=256, last_name=\"'Stone'\", email=\"'gstone73@google.es'\", gender=\"'Female'\", department=\"'Books'\", start_date=\"'12/1/2006'\", salary=87816, job_title=\"'Administrative Assistant I'\", region_id=2),\n",
              " Row(id=257, last_name=\"'Freeman'\", email=\"'gfreeman74@bloomberg.com'\", gender=\"'Female'\", department=\"'Home'\", start_date=\"'6/5/2012'\", salary=83804, job_title=\"'Operator'\", region_id=1),\n",
              " Row(id=271, last_name=\"'Jordan'\", email=\"'pjordan7i@squidoo.com'\", gender=\"'Female'\", department=\"'Sports'\", start_date=\"'8/7/2001'\", salary=123843, job_title=\"'Senior Sales Associate'\", region_id=7),\n",
              " Row(id=303, last_name=\"'West'\", email=\"'jwest8e@berkeley.edu'\", gender=\"'Male'\", department=\"'Electronics'\", start_date=\"'3/15/2012'\", salary=76943, job_title=\"'General Manager'\", region_id=7),\n",
              " Row(id=304, last_name=\"'Mccoy'\", email=\"'mmccoy8f@wisc.edu'\", gender=\"'Female'\", department=\"'Books'\", start_date=\"'5/26/2007'\", salary=77486, job_title=\"'Community Outreach Specialist'\", region_id=4),\n",
              " Row(id=315, last_name=\"'Smith'\", email=\"'bsmith8q@mysql.com'\", gender=\"'Male'\", department=\"'Games'\", start_date=\"'7/25/2012'\", salary=59580, job_title=\"'General Manager'\", region_id=2),\n",
              " Row(id=316, last_name=\"'Peters'\", email=\"'cpeters8r@epa.gov'\", gender=\"'Male'\", department=\"'Sports'\", start_date=\"'9/16/2010'\", salary=40418, job_title=\"'VP Marketing'\", region_id=2),\n",
              " Row(id=318, last_name=\"'Moore'\", email=\"'jmoore8t@umich.edu'\", gender=\"'Male'\", department=\"'Computers'\", start_date=\"'4/27/2003'\", salary=139304, job_title=\"'Help Desk Technician'\", region_id=3),\n",
              " Row(id=326, last_name=\"'Castillo'\", email=\"'bcastillo91@hc360.com'\", gender=\"'Female'\", department=\"'Jewelery'\", start_date=\"'12/12/2007'\", salary=130188, job_title=\"'Social Worker'\", region_id=3),\n",
              " Row(id=329, last_name=\"'Mccoy'\", email=\"'hmccoy94@miitbeian.gov.cn'\", gender=\"'Female'\", department=\"'Shoes'\", start_date=\"'7/27/2004'\", salary=86803, job_title=\"'Administrative Assistant III'\", region_id=4),\n",
              " Row(id=339, last_name=\"'Hudson'\", email=\"'rhudson9e@geocities.com'\", gender=\"'Female'\", department=\"'Jewelery'\", start_date=\"'7/23/2004'\", salary=78294, job_title=\"'Software Consultant'\", region_id=4),\n",
              " Row(id=358, last_name=\"'Gutierrez'\", email=\"'jgutierrez9x@telegraph.co.uk'\", gender=\"'Male'\", department=\"'Automotive'\", start_date=\"'6/17/2002'\", salary=67800, job_title=\"'Desktop Support Technician'\", region_id=2),\n",
              " Row(id=361, last_name=\"'Rice'\", email=\"'dricea0@hubpages.com'\", gender=\"'Female'\", department=\"'Garden'\", start_date=\"'12/6/2006'\", salary=144724, job_title=\"'Senior Quality Engineer'\", region_id=7),\n",
              " Row(id=372, last_name=\"'Harper'\", email=\"'rharperab@china.com.cn'\", gender=\"'Female'\", department=\"'Kids'\", start_date=\"'8/31/2001'\", salary=96163, job_title=\"'Safety Technician IV'\", region_id=3),\n",
              " Row(id=374, last_name=\"'Mcdonald'\", email=\"'jmcdonaldad@sbwire.com'\", gender=\"'Female'\", department=\"'Baby'\", start_date=\"'9/19/2005'\", salary=141464, job_title=\"'Programmer IV'\", region_id=4),\n",
              " Row(id=391, last_name=\"'White'\", email=\"'cwhiteau@amazon.de'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'11/4/2003'\", salary=147702, job_title=\"'VP Quality Control'\", region_id=4),\n",
              " Row(id=407, last_name=\"'Vasquez'\", email=\"'jvasquezba@dion.ne.jp'\", gender=\"'Male'\", department=\"'Baby'\", start_date=\"'4/9/2013'\", salary=54704, job_title=\"'Analog Circuit Design manager'\", region_id=3),\n",
              " Row(id=410, last_name=\"'Harris'\", email=\"'lharrisbd@ow.ly'\", gender=\"'Female'\", department=\"'Tools'\", start_date=\"'1/7/2012'\", salary=148940, job_title=\"'Senior Financial Analyst'\", region_id=6),\n",
              " Row(id=419, last_name=\"'Dixon'\", email=\"'jdixonbm@indiatimes.com'\", gender=\"'Male'\", department=\"'Health'\", start_date=\"'2/22/2008'\", salary=104224, job_title=\"'Clinical Specialist'\", region_id=5),\n",
              " Row(id=422, last_name=\"'Torres'\", email=\"'ctorresbp@yahoo.com'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'9/17/2009'\", salary=118307, job_title=\"'Staff Accountant II'\", region_id=1),\n",
              " Row(id=445, last_name=\"'Greene'\", email=\"'dgreenecc@reuters.com'\", gender=\"'Male'\", department=\"'Toys'\", start_date=\"'3/23/2002'\", salary=149835, job_title=\"'Geologist III'\", region_id=1),\n",
              " Row(id=449, last_name=\"'Gray'\", email=\"'tgraycg@fotki.com'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'4/20/2014'\", salary=134205, job_title=\"'Biostatistician I'\", region_id=4),\n",
              " Row(id=452, last_name=\"'Kennedy'\", email=\"'ckennedycj@boston.com'\", gender=\"'Female'\", department=\"'Toys'\", start_date=\"'5/10/2000'\", salary=111072, job_title=\"'Senior Quality Engineer'\", region_id=4),\n",
              " Row(id=455, last_name=\"'Hawkins'\", email=\"'jhawkinscm@furl.net'\", gender=\"'Female'\", department=\"'Automotive'\", start_date=\"'6/13/2002'\", salary=115506, job_title=\"'Product Engineer'\", region_id=2),\n",
              " Row(id=456, last_name=\"'Watson'\", email=\"'swatsoncn@wufoo.com'\", gender=\"'Female'\", department=\"'Industrial'\", start_date=\"'3/10/2012'\", salary=115679, job_title=\"'Editor'\", region_id=2),\n",
              " Row(id=463, last_name=\"'Mason'\", email=\"'dmasoncu@about.com'\", gender=\"'Male'\", department=\"'Tools'\", start_date=\"'12/21/2010'\", salary=120160, job_title=\"'Database Administrator II'\", region_id=3),\n",
              " Row(id=478, last_name=\"'Mason'\", email=\"'jmasond9@networkadvertising.org'\", gender=\"'Male'\", department=\"'Sports'\", start_date=\"'4/16/2003'\", salary=147166, job_title=\"'Systems Administrator I'\", region_id=6),\n",
              " Row(id=489, last_name=\"'Parker'\", email=\"'lparkerdk@salon.com'\", gender=\"'Female'\", department=\"'Garden'\", start_date=\"'1/16/2014'\", salary=78109, job_title=\"'Research Nurse'\", region_id=5),\n",
              " Row(id=502, last_name=\"'Banks'\", email=\"'gbanksdx@berkeley.edu'\", gender=\"'Female'\", department=\"'Computers'\", start_date=\"'10/17/2010'\", salary=136034, job_title=\"'Product Engineer'\", region_id=4),\n",
              " Row(id=572, last_name=\"'Duncan'\", email=\"'jduncanfv@etsy.com'\", gender=\"'Male'\", department=\"'Automotive'\", start_date=\"'3/17/2014'\", salary=45774, job_title=\"'Electrical Engineer'\", region_id=2),\n",
              " Row(id=581, last_name=\"'Cunningham'\", email=\"'pcunninghamg4@example.com'\", gender=\"'Female'\", department=\"'Sports'\", start_date=\"'3/25/2010'\", salary=92334, job_title=\"'GIS Technical Architect'\", region_id=3),\n",
              " Row(id=585, last_name=\"'Lopez'\", email=\"'nlopezg8@fema.gov'\", gender=\"'Male'\", department=\"'Garden'\", start_date=\"'7/19/2007'\", salary=138776, job_title=\"'Senior Editor'\", region_id=2),\n",
              " Row(id=603, last_name=\"'Webb'\", email=\"'lwebbgq@last.fm'\", gender=\"'Male'\", department=\"'Jewelery'\", start_date=\"'12/6/2000'\", salary=127569, job_title=\"'Analyst Programmer'\", region_id=7),\n",
              " Row(id=612, last_name=\"'Fowler'\", email=\"'cfowlergz@joomla.org'\", gender=\"'Female'\", department=\"'Health'\", start_date=\"'9/10/2003'\", salary=142211, job_title=\"'Software Test Engineer IV'\", region_id=4),\n",
              " Row(id=619, last_name=\"'Cole'\", email=\"'jcoleh6@php.net'\", gender=\"'Male'\", department=\"'Tools'\", start_date=\"'6/2/2014'\", salary=133190, job_title=\"'Research Nurse'\", region_id=7),\n",
              " Row(id=641, last_name=\"'Hayes'\", email=\"'rhayeshs@loc.gov'\", gender=\"'Female'\", department=\"'Health'\", start_date=\"'4/7/2010'\", salary=69454, job_title=\"'Web Designer II'\", region_id=1),\n",
              " Row(id=643, last_name=\"'Johnston'\", email=\"'cjohnstonhu@psu.edu'\", gender=\"'Male'\", department=\"'Jewelery'\", start_date=\"'3/21/2014'\", salary=123820, job_title=\"'Computer Systems Analyst IV'\", region_id=4),\n",
              " Row(id=647, last_name=\"'Washington'\", email=\"'mwashingtonhy@icio.us'\", gender=\"'Male'\", department=\"'Electronics'\", start_date=\"'10/15/2009'\", salary=57092, job_title=\"'Internal Auditor'\", region_id=2),\n",
              " Row(id=649, last_name=\"'Russell'\", email=\"'krusselli0@posterous.com'\", gender=\"'Male'\", department=\"'Shoes'\", start_date=\"'11/18/2007'\", salary=99352, job_title=\"'Recruiting Manager'\", region_id=1),\n",
              " Row(id=665, last_name=\"'Garcia'\", email=\"'pgarciaig@macromedia.com'\", gender=\"'Female'\", department=\"'Health'\", start_date=\"'11/4/2008'\", salary=135132, job_title=\"'Research Assistant III'\", region_id=7),\n",
              " Row(id=677, last_name=\"'Robertson'\", email=\"'lrobertsonis@forbes.com'\", gender=\"'Male'\", department=\"'Computers'\", start_date=\"'8/4/2014'\", salary=136492, job_title=\"'Compensation Analyst'\", region_id=4),\n",
              " Row(id=679, last_name=\"'Stone'\", email=\"'pstoneiu@narod.ru'\", gender=\"'Male'\", department=\"'Electronics'\", start_date=\"'5/15/2014'\", salary=40218, job_title=\"'Paralegal'\", region_id=2),\n",
              " Row(id=693, last_name=\"'Carr'\", email=\"'hcarrj8@latimes.com'\", gender=\"'Male'\", department=\"'Garden'\", start_date=\"'1/17/2005'\", salary=143570, job_title=\"'Actuary'\", region_id=4),\n",
              " Row(id=700, last_name=\"'Stone'\", email=\"'tstonejf@com.com'\", gender=\"'Male'\", department=\"'Garden'\", start_date=\"'7/15/2004'\", salary=100746, job_title=\"'VP Quality Control'\", region_id=6),\n",
              " Row(id=708, last_name=\"'Rose'\", email=\"'crosejn@cnbc.com'\", gender=\"'Female'\", department=\"'Automotive'\", start_date=\"'12/24/2011'\", salary=66063, job_title=\"'Software Test Engineer I'\", region_id=7),\n",
              " Row(id=709, last_name=\"'Payne'\", email=\"'cpaynejo@tinyurl.com'\", gender=\"'Female'\", department=\"'Books'\", start_date=\"'1/12/2007'\", salary=138137, job_title=\"'Developer IV'\", region_id=2),\n",
              " Row(id=722, last_name=\"'Hill'\", email=\"'khillk1@about.me'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'8/31/2002'\", salary=73262, job_title=\"'Information Systems Manager'\", region_id=5),\n",
              " Row(id=731, last_name=\"'Ferguson'\", email=\"'gfergusonka@geocities.jp'\", gender=\"'Female'\", department=\"'Home'\", start_date=\"'6/28/2009'\", salary=70524, job_title=\"'VP Accounting'\", region_id=2),\n",
              " Row(id=750, last_name=\"'Ward'\", email=\"'kwardkt@wikipedia.org'\", gender=\"'Male'\", department=\"'Music'\", start_date=\"'7/16/2014'\", salary=108228, job_title=\"'VP Sales'\", region_id=3),\n",
              " Row(id=763, last_name=\"'Bell'\", email=\"'dbelll6@wisc.edu'\", gender=\"'Male'\", department=\"'Computers'\", start_date=\"'4/22/2002'\", salary=122843, job_title=\"'Accountant IV'\", region_id=2),\n",
              " Row(id=772, last_name=\"'Elliott'\", email=\"'delliottlf@fda.gov'\", gender=\"'Male'\", department=\"'Movies'\", start_date=\"'10/13/2004'\", salary=100570, job_title=\"'Senior Financial Analyst'\", region_id=5),\n",
              " Row(id=792, last_name=\"'Mccoy'\", email=\"'jmccoylz@samsung.com'\", gender=\"'Male'\", department=\"'Industrial'\", start_date=\"'9/20/2007'\", salary=95787, job_title=\"'Engineer IV'\", region_id=2),\n",
              " Row(id=816, last_name=\"'Jackson'\", email=\"'pjacksonmn@dailymotion.com'\", gender=\"'Female'\", department=\"'Beauty'\", start_date=\"'12/29/2013'\", salary=107093, job_title=\"'Electrical Engineer'\", region_id=3),\n",
              " Row(id=829, last_name=\"'Parker'\", email=\"'bparkern0@weibo.com'\", gender=\"'Male'\", department=\"'Health'\", start_date=\"'11/27/2002'\", salary=89503, job_title=\"'Research Associate'\", region_id=2),\n",
              " Row(id=838, last_name=\"'Black'\", email=\"'rblackn9@sun.com'\", gender=\"'Male'\", department=\"'Toys'\", start_date=\"'9/28/2008'\", salary=117171, job_title=\"'Electrical Engineer'\", region_id=4),\n",
              " Row(id=847, last_name=\"'Knight'\", email=\"'dknightni@unc.edu'\", gender=\"'Female'\", department=\"'Clothing'\", start_date=\"'3/22/2008'\", salary=91532, job_title=\"'Operator'\", region_id=4),\n",
              " Row(id=848, last_name=\"'Austin'\", email=\"'jaustinnj@gov.uk'\", gender=\"'Female'\", department=\"'Shoes'\", start_date=\"'1/26/2005'\", salary=101646, job_title=\"'Environmental Tech'\", region_id=1),\n",
              " Row(id=859, last_name=\"'Clark'\", email=\"'mclarknu@bigcartel.com'\", gender=\"'Male'\", department=\"'Health'\", start_date=\"'2/19/2007'\", salary=112510, job_title=\"'Information Systems Manager'\", region_id=4),\n",
              " Row(id=873, last_name=\"'Alexander'\", email=\"'aalexandero8@histats.com'\", gender=\"'Female'\", department=\"'Toys'\", start_date=\"'7/5/2006'\", salary=140078, job_title=\"'Environmental Specialist'\", region_id=3),\n",
              " Row(id=874, last_name=\"'Spencer'\", email=\"'kspencero9@eventbrite.com'\", gender=\"'Male'\", department=\"'Books'\", start_date=\"'9/29/2012'\", salary=119769, job_title=\"'Director of Sales'\", region_id=2),\n",
              " Row(id=885, last_name=\"'Wallace'\", email=\"'mwallaceok@abc.net.au'\", gender=\"'Female'\", department=\"'Toys'\", start_date=\"'9/17/2011'\", salary=118793, job_title=\"'Research Assistant III'\", region_id=3),\n",
              " Row(id=890, last_name=\"'Howell'\", email=\"'ehowellop@dailymotion.com'\", gender=\"'Female'\", department=\"'Baby'\", start_date=\"'11/26/2012'\", salary=72534, job_title=\"'Account Executive'\", region_id=4),\n",
              " Row(id=891, last_name=\"'Richards'\", email=\"'arichardsoq@so-net.ne.jp'\", gender=\"'Male'\", department=\"'Baby'\", start_date=\"'6/9/2011'\", salary=140004, job_title=\"'Design Engineer'\", region_id=7),\n",
              " Row(id=898, last_name=\"'Lee'\", email=\"'kleeox@nature.com'\", gender=\"'Female'\", department=\"'Industrial'\", start_date=\"'12/9/2010'\", salary=115085, job_title=\"'Senior Quality Engineer'\", region_id=7),\n",
              " Row(id=900, last_name=\"'Jordan'\", email=\"'tjordanoz@instagram.com'\", gender=\"'Male'\", department=\"'Toys'\", start_date=\"'2/23/2012'\", salary=115776, job_title=\"'Database Administrator I'\", region_id=2),\n",
              " Row(id=914, last_name=\"'Armstrong'\", email=\"'carmstrongpd@4shared.com'\", gender=\"'Female'\", department=\"'Beauty'\", start_date=\"'6/7/2006'\", salary=143853, job_title=\"'Design Engineer'\", region_id=2),\n",
              " Row(id=917, last_name=\"'Gutierrez'\", email=\"'sgutierrezpg@bloglovin.com'\", gender=\"'Male'\", department=\"'Beauty'\", start_date=\"'12/11/2004'\", salary=95424, job_title=\"'VP Product Management'\", region_id=2),\n",
              " Row(id=927, last_name=\"'Howard'\", email=\"'dhowardpq@ocn.ne.jp'\", gender=\"'Male'\", department=\"'Music'\", start_date=\"'11/10/2003'\", salary=82080, job_title=\"'Analog Circuit Design manager'\", region_id=2),\n",
              " Row(id=928, last_name=\"'Young'\", email=\"'lyoungpr@psu.edu'\", gender=\"'Female'\", department=\"'Kids'\", start_date=\"'12/21/2009'\", salary=62799, job_title=\"'Operator'\", region_id=1),\n",
              " Row(id=933, last_name=\"'Williams'\", email=\"'gwilliamspw@columbia.edu'\", gender=\"'Male'\", department=\"'Baby'\", start_date=\"'3/23/2014'\", salary=131273, job_title=\"'Web Developer IV'\", region_id=6),\n",
              " Row(id=952, last_name=\"'Cole'\", email=\"'acoleqf@unblog.fr'\", gender=\"'Female'\", department=\"'Shoes'\", start_date=\"'10/21/2008'\", salary=53905, job_title=\"'Editor'\", region_id=7),\n",
              " Row(id=962, last_name=\"'Hayes'\", email=\"'ahayesqp@npr.org'\", gender=\"'Female'\", department=\"'Health'\", start_date=\"'10/13/2000'\", salary=48339, job_title=\"'Engineer III'\", region_id=5)]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WFTeBKI10eEV",
        "outputId": "658c8bef-7306-4369-ebbe-2562af0b795b"
      },
      "source": [
        "df_sample.show()"
      ],
      "id": "WFTeBKI10eEV",
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------------+--------------------+--------+-------------+------------+------+--------------------+---------+\n",
            "| id|   last_name|               email|  gender|   department|  start_date|salary|           job_title|region_id|\n",
            "+---+------------+--------------------+--------+-------------+------------+------+--------------------+---------+\n",
            "|  3|      'Carr'|'fcarr2@woothemes...|  'Male'| 'Automotive'| '7/12/2009'|101768|'Recruiting Manager'|        3|\n",
            "| 10|   'Sanchez'|'rsanchez9@cloudf...|  'Male'|     'Movies'| '3/13/2013'|108093|'Sales Representa...|        1|\n",
            "| 17|    'Morgan'|'dmorgang@123-reg...|'Female'|       'Kids'|  '5/4/2011'|148952|     'Programmer IV'|        6|\n",
            "| 27|     'Owens'|'cowensq@shareasa...|'Female'|       'Home'| '5/26/2005'| 78698|'Executive Secret...|        2|\n",
            "| 47|    'Oliver'|'joliver1a@cnbc.com'|'Female'|   'Clothing'| '8/30/2013'| 42797|'Software Enginee...|        5|\n",
            "| 58|     'Baker'|'jbaker1l@usnews....|  'Male'|      'Games'|  '3/2/2007'| 68857|'Information Syst...|        2|\n",
            "| 59|    'Garcia'|'vgarcia1m@devian...|'Female'|     'Health'| '3/19/2002'| 48360|  'Registered Nurse'|        6|\n",
            "| 67|     'Mills'|'jmills1u@printfr...|'Female'|    'Grocery'|  '8/8/2004'|114200|'Senior Sales Ass...|        7|\n",
            "| 80|    'Little'|  'klittle27@un.org'|  'Male'|      'Music'| '9/12/2001'|114647|            'Editor'|        5|\n",
            "| 93|    'Knight'|'nknight2k@cafepr...|'Female'|'Electronics'|  '4/3/2014'|125847| 'Chemical Engineer'|        5|\n",
            "|115|    'Nguyen'|'rnguyen36@bandca...|'Female'|   'Clothing'| '6/30/2007'| 58420|  'Graphic Designer'|        1|\n",
            "|127|    'Harper'|'sharper3i@slides...|'Female'|   'Outdoors'| '5/18/2000'|144639|'VP Quality Control'|        7|\n",
            "|139|      'Hart'|   'rhart3u@wsj.com'|  'Male'|   'Jewelery'|  '1/8/2012'|119773|        'Pharmacist'|        3|\n",
            "|141|'Williamson'|'rwilliamson3w@am...|  'Male'|      'Games'|'12/12/2005'|145575|     'VP Accounting'|        1|\n",
            "|146|  'Thompson'|'athompson41@book...|  'Male'|     'Movies'|  '2/4/2009'|133281| 'Account Executive'|        6|\n",
            "|149|     'Meyer'|'jmeyer44@faceboo...|'Female'|       'Toys'|  '5/3/2008'| 76352|'Human Resources ...|        4|\n",
            "|171|    'Harris'|'mharris4q@square...|'Female'|     'Movies'| '9/27/2004'|110912|       'Geologist I'|        5|\n",
            "|178|    'Butler'|'cbutler4x@amazon...|'Female'|     'Sports'| '2/17/2003'| 78157|         'Librarian'|        1|\n",
            "|182|     'Gomez'|'rgomez51@baidu.com'|'Female'| 'Industrial'|  '4/4/2009'| 76932|   'Legal Assistant'|        5|\n",
            "|185|    'Watson'|'rwatson54@webede...|'Female'|   'Clothing'| '9/22/2001'| 74648|  'Health Coach III'|        2|\n",
            "+---+------------+--------------------+--------+-------------+------------+------+--------------------+---------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(df_sample)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 186
        },
        "id": "CzsZban9XWiw",
        "outputId": "e62767aa-2b82-4262-bcd4-defda6129155"
      },
      "id": "CzsZban9XWiw",
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pyspark.sql.dataframe.DataFrame"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>pyspark.sql.dataframe.DataFrame</b><br/>def __init__(jdf, sql_ctx)</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/pyspark/sql/dataframe.py</a>A distributed collection of data grouped into named columns.\n",
              "\n",
              "A :class:`DataFrame` is equivalent to a relational table in Spark SQL,\n",
              "and can be created using various functions in :class:`SparkSession`::\n",
              "\n",
              "    people = spark.read.parquet(&quot;...&quot;)\n",
              "\n",
              "Once created, it can be manipulated using the various domain-specific-language\n",
              "(DSL) functions defined in: :class:`DataFrame`, :class:`Column`.\n",
              "\n",
              "To select a column from the :class:`DataFrame`, use the apply method::\n",
              "\n",
              "    ageCol = people.age\n",
              "\n",
              "A more concrete example::\n",
              "\n",
              "    # To create DataFrame using SparkSession\n",
              "    people = spark.read.parquet(&quot;...&quot;)\n",
              "    department = spark.read.parquet(&quot;...&quot;)\n",
              "\n",
              "    people.filter(people.age &gt; 30).join(department, people.deptId == department.id) \\\n",
              "      .groupBy(department.name, &quot;gender&quot;).agg({&quot;salary&quot;: &quot;avg&quot;, &quot;age&quot;: &quot;max&quot;})\n",
              "\n",
              ".. versionadded:: 1.3.0</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 41);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HIKiNjyTaMit"
      },
      "source": [
        "### Preprocessing"
      ],
      "id": "HIKiNjyTaMit"
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Min-Max Scaling"
      ],
      "metadata": {
        "id": "DJYoHVpQWqwE"
      },
      "id": "DJYoHVpQWqwE"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BL9rViaNaMiv"
      },
      "source": [
        "from pyspark.ml import feature\n",
        "from pyspark.ml.linalg import Vectors"
      ],
      "id": "BL9rViaNaMiv",
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D1q6CfqxaMiv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e698ed21-a918-49ad-98b6-c9852c437d31"
      },
      "source": [
        "df2 = spark.createDataFrame(data = [(1, Vectors.dense([10.0, 10000.0, 1.0])),\n",
        "                                    (2, Vectors.dense([20.0, 20000.0, 2.0])),\n",
        "                                    (3, Vectors.dense([30.0, 30000.0, 3.0]))],\n",
        "                            schema = [\"id\", \"features\"])\n",
        "df2"
      ],
      "id": "D1q6CfqxaMiv",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[id: bigint, features: vector]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-fpFjXYwaMiw",
        "outputId": "5d370e9a-3821-420b-c4ca-2fc1eb41b2c9"
      },
      "source": [
        "df2.show()"
      ],
      "id": "-fpFjXYwaMiw",
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------------------+\n",
            "| id|          features|\n",
            "+---+------------------+\n",
            "|  1|[10.0,10000.0,1.0]|\n",
            "|  2|[20.0,20000.0,2.0]|\n",
            "|  3|[30.0,30000.0,3.0]|\n",
            "+---+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tG21mW1WaMiw",
        "outputId": "3a07589e-b891-4a2b-dfea-7eca64915b47"
      },
      "source": [
        "df2.schema"
      ],
      "id": "tG21mW1WaMiw",
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StructType(List(StructField(id,LongType,true),StructField(features,VectorUDT,true)))"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cno3rGTvaMiw",
        "outputId": "942f5a2e-85fa-40a3-b132-6ab5aa11d7fa"
      },
      "source": [
        "df2.columns"
      ],
      "id": "Cno3rGTvaMiw",
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['id', 'features']"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h9-bXvxYaMix",
        "outputId": "e960f22c-300b-4121-f706-805cda3f3e5b"
      },
      "source": [
        "df2.collect()"
      ],
      "id": "h9-bXvxYaMix",
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=1, features=DenseVector([10.0, 10000.0, 1.0])),\n",
              " Row(id=2, features=DenseVector([20.0, 20000.0, 2.0])),\n",
              " Row(id=3, features=DenseVector([30.0, 30000.0, 3.0]))]"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oNSrzcbaMix",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3393b40f-b464-44d2-d695-8cb5506d83d5"
      },
      "source": [
        "minmax_scaler = feature.MinMaxScaler(inputCol = \"features\",\n",
        "                                     outputCol = \"minmax_features\")\n",
        "minmax_scaler = minmax_scaler.fit(dataset = df2)\n",
        "df2_scaled = minmax_scaler.transform(dataset = df2)\n",
        "df2_scaled.show()"
      ],
      "id": "4oNSrzcbaMix",
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------------------+---------------+\n",
            "| id|          features|minmax_features|\n",
            "+---+------------------+---------------+\n",
            "|  1|[10.0,10000.0,1.0]|      (3,[],[])|\n",
            "|  2|[20.0,20000.0,2.0]|  [0.5,0.5,0.5]|\n",
            "|  3|[30.0,30000.0,3.0]|  [1.0,1.0,1.0]|\n",
            "+---+------------------+---------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QlqUUmBj02wY",
        "outputId": "8a0a9ef2-4bd0-4906-c51d-c9db56ebdcc1"
      },
      "source": [
        "df2_scaled.limit(2).collect()"
      ],
      "id": "QlqUUmBj02wY",
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=1, features=DenseVector([10.0, 10000.0, 1.0]), minmax_features=SparseVector(3, {})),\n",
              " Row(id=2, features=DenseVector([20.0, 20000.0, 2.0]), minmax_features=DenseVector([0.5, 0.5, 0.5]))]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iEz-esgNaMiy",
        "outputId": "aee0f553-d5b8-4e39-9450-8eb05d11827d"
      },
      "source": [
        "df2_scaled.collect()"
      ],
      "id": "iEz-esgNaMiy",
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(id=1, features=DenseVector([10.0, 10000.0, 1.0]), minmax_features=SparseVector(3, {})),\n",
              " Row(id=2, features=DenseVector([20.0, 20000.0, 2.0]), minmax_features=DenseVector([0.5, 0.5, 0.5])),\n",
              " Row(id=3, features=DenseVector([30.0, 30000.0, 3.0]), minmax_features=DenseVector([1.0, 1.0, 1.0]))]"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "30LjJQgtaMiy",
        "outputId": "63f883c8-b5eb-4dd7-851c-0d8ab87a9856"
      },
      "source": [
        "df2_scaled.select(\"features\", \"minmax_features\").show()"
      ],
      "id": "30LjJQgtaMiy",
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------------+---------------+\n",
            "|          features|minmax_features|\n",
            "+------------------+---------------+\n",
            "|[10.0,10000.0,1.0]|      (3,[],[])|\n",
            "|[20.0,20000.0,2.0]|  [0.5,0.5,0.5]|\n",
            "|[30.0,30000.0,3.0]|  [1.0,1.0,1.0]|\n",
            "+------------------+---------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v18JPFjNaMiy"
      },
      "source": [
        "#### Standardization"
      ],
      "id": "v18JPFjNaMiy"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mDZGlsw5aMiz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b8d05f9-00f8-43fc-d8ce-89cfce9586cd"
      },
      "source": [
        "df2 = spark.createDataFrame(data = [(1, Vectors.dense([10.0, 10000.0, 1.0])),\n",
        "                                    (2, Vectors.dense([20.0, 20000.0, 2.0])),\n",
        "                                    (3, Vectors.dense([30.0, 30000.0, 3.0])),\n",
        "                                    (3, Vectors.dense([30.0, 30000.0, 3.0])),\n",
        "                                    (3, Vectors.dense([40.0, 40000.0, 4.0]))],\n",
        "                            schema = [\"id\", \"features\"])\n",
        "df2.show()"
      ],
      "id": "mDZGlsw5aMiz",
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------------------+\n",
            "| id|          features|\n",
            "+---+------------------+\n",
            "|  1|[10.0,10000.0,1.0]|\n",
            "|  2|[20.0,20000.0,2.0]|\n",
            "|  3|[30.0,30000.0,3.0]|\n",
            "|  3|[30.0,30000.0,3.0]|\n",
            "|  3|[40.0,40000.0,4.0]|\n",
            "+---+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fOrrTwH1aMiz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ccaf3e9f-f5dd-458b-fe7c-cabb018d65ae"
      },
      "source": [
        "scaler = feature.StandardScaler(inputCol = \"features\",\n",
        "                                outputCol = \"s_features\",\n",
        "                                withMean = True,\n",
        "                                withStd = True)\n",
        "scaler = scaler.fit(dataset = df2)\n",
        "df2_scaled = scaler.transform(dataset = df2)\n",
        "df2_scaled.show()"
      ],
      "id": "fOrrTwH1aMiz",
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------------------+--------------------+\n",
            "| id|          features|          s_features|\n",
            "+---+------------------+--------------------+\n",
            "|  1|[10.0,10000.0,1.0]|[-1.4032928308912...|\n",
            "|  2|[20.0,20000.0,2.0]|[-0.5262348115842...|\n",
            "|  3|[30.0,30000.0,3.0]|[0.35082320772281...|\n",
            "|  3|[30.0,30000.0,3.0]|[0.35082320772281...|\n",
            "|  3|[40.0,40000.0,4.0]|[1.22788122702984...|\n",
            "+---+------------------+--------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t5Fwx6L_aMi0",
        "outputId": "ec6f8f6b-87cc-40d6-98e6-fa8fa60e6093"
      },
      "source": [
        "df2_scaled.show(truncate = False)"
      ],
      "id": "t5Fwx6L_aMi0",
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------------------+-------------------------------------------------------------+\n",
            "|id |features          |s_features                                                   |\n",
            "+---+------------------+-------------------------------------------------------------+\n",
            "|1  |[10.0,10000.0,1.0]|[-1.403292830891247,-1.4032928308912467,-1.403292830891247]  |\n",
            "|2  |[20.0,20000.0,2.0]|[-0.5262348115842176,-0.5262348115842175,-0.5262348115842177]|\n",
            "|3  |[30.0,30000.0,3.0]|[0.3508232077228117,0.35082320772281167,0.35082320772281167] |\n",
            "|3  |[30.0,30000.0,3.0]|[0.3508232077228117,0.35082320772281167,0.35082320772281167] |\n",
            "|3  |[40.0,40000.0,4.0]|[1.2278812270298411,1.2278812270298407,1.227881227029841]    |\n",
            "+---+------------------+-------------------------------------------------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FTW3bgxxaMi1"
      },
      "source": [
        "#### Bucketizer"
      ],
      "id": "FTW3bgxxaMi1"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oF1PwYoGaMi1"
      },
      "source": [
        "df2 = spark.createDataFrame(data = [(1, 5000.0),\n",
        "                                    (2, 15000.0),\n",
        "                                    (3, 30000.0),\n",
        "                                    (3, 35000.0),\n",
        "                                    (3, 40000.0)],\n",
        "                            schema = [\"id\", \"points\"])"
      ],
      "id": "oF1PwYoGaMi1",
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sji0Eh80aMi1"
      },
      "source": [
        "splits = [-float(\"inf\"), 0, 10000, 30000, float(\"inf\")]"
      ],
      "id": "sji0Eh80aMi1",
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sr9C-RdvaMi2"
      },
      "source": [
        "bucketizer = feature.Bucketizer(splits = splits,\n",
        "                                inputCol = \"points\",\n",
        "                                outputCol = \"points_buckets\")"
      ],
      "id": "Sr9C-RdvaMi2",
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5jtODElXaMi2"
      },
      "source": [
        "df2_bucket = bucketizer.transform(dataset = df2)"
      ],
      "id": "5jtODElXaMi2",
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jyWuaqCvaMi2",
        "outputId": "d4f9a1f5-d8e2-4ec7-dd82-b92aa488f9ec"
      },
      "source": [
        "df2_bucket.show()"
      ],
      "id": "jyWuaqCvaMi2",
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+-------+--------------+\n",
            "| id| points|points_buckets|\n",
            "+---+-------+--------------+\n",
            "|  1| 5000.0|           1.0|\n",
            "|  2|15000.0|           2.0|\n",
            "|  3|30000.0|           3.0|\n",
            "|  3|35000.0|           3.0|\n",
            "|  3|40000.0|           3.0|\n",
            "+---+-------+--------------+\n",
            "\n"
          ]
        }
      ]
    }
  ]
}