pyspark_winequality.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_tKN9JRNYxik",
        "outputId": "a8ce184a-fafb-4593-9e55-a19d41a87c12"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.5.0.tar.gz (316.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m316.9/316.9 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n",
            "Building wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.5.0-py2.py3-none-any.whl size=317425344 sha256=23b7192dff2c85f0eb7c658f520cce11f8f3d4a808b8e8cb2978eafdc3584bbd\n",
            "  Stored in directory: /root/.cache/pip/wheels/41/4e/10/c2cf2467f71c678cfc8a6b9ac9241e5e44a01940da8fbb17fc\n",
            "Successfully built pyspark\n",
            "Installing collected packages: pyspark\n",
            "Successfully installed pyspark-3.5.0\n"
          ]
        }
      ],
      "source": [
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pyspark\n",
        "import pandas as pd\n",
        "\n",
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "v2ImZ6vMY0Zk"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession.builder.appName(\"practice\").getOrCreate()"
      ],
      "metadata": {
        "id": "Vk3FjP5OY8Yj"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "id": "Lh26vyW2ZK28",
        "outputId": "1bf7dfef-5785-442b-b525-c36c66bac363"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7a6b887341f0>"
            ],
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://3a8543301772:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.5.0</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>practice</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark = spark.read.option('header','true').csv(\"winequality-white.csv\")"
      ],
      "metadata": {
        "id": "j4m15ij4ZNzc"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O3o9rnlrZe8N",
        "outputId": "4f7bb7af-a764-44af-bec9-15df250ab311"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+\n",
            "|fixed acidity|volatile acidity|citric acid|residual sugar|chlorides|free sulfur dioxide|total sulfur dioxide|density|  pH|sulphates|alcohol|quality|\n",
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+\n",
            "|            7|            0.27|       0.36|          20.7|    0.045|                 45|                 170|  1.001|   3|     0.45|    8.8|      6|\n",
            "|          6.3|             0.3|       0.34|           1.6|    0.049|                 14|                 132|  0.994| 3.3|     0.49|    9.5|      6|\n",
            "|          8.1|            0.28|        0.4|           6.9|     0.05|                 30|                  97| 0.9951|3.26|     0.44|   10.1|      6|\n",
            "|          7.2|            0.23|       0.32|           8.5|    0.058|                 47|                 186| 0.9956|3.19|      0.4|    9.9|      6|\n",
            "|          7.2|            0.23|       0.32|           8.5|    0.058|                 47|                 186| 0.9956|3.19|      0.4|    9.9|      6|\n",
            "|          8.1|            0.28|        0.4|           6.9|     0.05|                 30|                  97| 0.9951|3.26|     0.44|   10.1|      6|\n",
            "|          6.2|            0.32|       0.16|             7|    0.045|                 30|                 136| 0.9949|3.18|     0.47|    9.6|      6|\n",
            "|            7|            0.27|       0.36|          20.7|    0.045|                 45|                 170|  1.001|   3|     0.45|    8.8|      6|\n",
            "|          6.3|             0.3|       0.34|           1.6|    0.049|                 14|                 132|  0.994| 3.3|     0.49|    9.5|      6|\n",
            "|          8.1|            0.22|       0.43|           1.5|    0.044|                 28|                 129| 0.9938|3.22|     0.45|     11|      6|\n",
            "|          8.1|            0.27|       0.41|          1.45|    0.033|                 11|                  63| 0.9908|2.99|     0.56|     12|      5|\n",
            "|          8.6|            0.23|        0.4|           4.2|    0.035|                 17|                 109| 0.9947|3.14|     0.53|    9.7|      5|\n",
            "|          7.9|            0.18|       0.37|           1.2|     0.04|                 16|                  75|  0.992|3.18|     0.63|   10.8|      5|\n",
            "|          6.6|            0.16|        0.4|           1.5|    0.044|                 48|                 143| 0.9912|3.54|     0.52|   12.4|      7|\n",
            "|          8.3|            0.42|       0.62|         19.25|     0.04|                 41|                 172| 1.0002|2.98|     0.67|    9.7|      5|\n",
            "|          6.6|            0.17|       0.38|           1.5|    0.032|                 28|                 112| 0.9914|3.25|     0.55|   11.4|      7|\n",
            "|          6.3|            0.48|       0.04|           1.1|    0.046|                 30|                  99| 0.9928|3.24|     0.36|    9.6|      6|\n",
            "|          6.2|            0.66|       0.48|           1.2|    0.029|                 29|                  75| 0.9892|3.33|     0.39|   12.8|      8|\n",
            "|          7.4|            0.34|       0.42|           1.1|    0.033|                 17|                 171| 0.9917|3.12|     0.53|   11.3|      6|\n",
            "|          6.5|            0.31|       0.14|           7.5|    0.044|                 34|                 133| 0.9955|3.22|      0.5|    9.5|      5|\n",
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "##check the schema\n",
        "df_spark.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "naQynq8bZ96M",
        "outputId": "52ad589b-5473-481a-d20d-7929ab0d2b1d"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- fixed acidity: string (nullable = true)\n",
            " |-- volatile acidity: string (nullable = true)\n",
            " |-- citric acid: string (nullable = true)\n",
            " |-- residual sugar: string (nullable = true)\n",
            " |-- chlorides: string (nullable = true)\n",
            " |-- free sulfur dioxide: string (nullable = true)\n",
            " |-- total sulfur dioxide: string (nullable = true)\n",
            " |-- density: string (nullable = true)\n",
            " |-- pH: string (nullable = true)\n",
            " |-- sulphates: string (nullable = true)\n",
            " |-- alcohol: string (nullable = true)\n",
            " |-- quality: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xlbei27maBgM",
        "outputId": "0edd5122-6ba0-4214-b71b-2ef421d7c06d"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['fixed acidity',\n",
              " 'volatile acidity',\n",
              " 'citric acid',\n",
              " 'residual sugar',\n",
              " 'chlorides',\n",
              " 'free sulfur dioxide',\n",
              " 'total sulfur dioxide',\n",
              " 'density',\n",
              " 'pH',\n",
              " 'sulphates',\n",
              " 'alcohol',\n",
              " 'quality']"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark.select('quality','density').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WxfkcA5Ebbhr",
        "outputId": "c77d542a-a3ef-445a-aef5-810d5010a913"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-------+\n",
            "|quality|density|\n",
            "+-------+-------+\n",
            "|      6|  1.001|\n",
            "|      6|  0.994|\n",
            "|      6| 0.9951|\n",
            "|      6| 0.9956|\n",
            "|      6| 0.9956|\n",
            "|      6| 0.9951|\n",
            "|      6| 0.9949|\n",
            "|      6|  1.001|\n",
            "|      6|  0.994|\n",
            "|      6| 0.9938|\n",
            "|      5| 0.9908|\n",
            "|      5| 0.9947|\n",
            "|      5|  0.992|\n",
            "|      7| 0.9912|\n",
            "|      5| 1.0002|\n",
            "|      7| 0.9914|\n",
            "|      6| 0.9928|\n",
            "|      8| 0.9892|\n",
            "|      6| 0.9917|\n",
            "|      5| 0.9955|\n",
            "+-------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark.describe().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RYU4PqPSbmk8",
        "outputId": "f491887e-13c1-4b47-c6df-3559b10a65f9"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+-------------------+-------------------+-----------------+--------------------+-------------------+--------------------+--------------------+-------------------+-------------------+------------------+------------------+\n",
            "|summary|     fixed acidity|   volatile acidity|        citric acid|   residual sugar|           chlorides|free sulfur dioxide|total sulfur dioxide|             density|                 pH|          sulphates|           alcohol|           quality|\n",
            "+-------+------------------+-------------------+-------------------+-----------------+--------------------+-------------------+--------------------+--------------------+-------------------+-------------------+------------------+------------------+\n",
            "|  count|              4898|               4898|               4898|             4898|                4898|               4898|                4898|                4898|               4898|               4898|              4898|              4898|\n",
            "|   mean| 6.854787668436075|0.27824111882401087|0.33419150673743736|6.391414863209486|  0.0457723560636995|  35.30808493262556|  138.36065741118824|  0.9940273764801896| 3.1882666394446693| 0.4898468762760325|10.514267047770149|  5.87790935075541|\n",
            "| stddev|0.8438682276875127|0.10079454842486532|0.12101980420298254|5.072057784014878|0.021847968093728805|  17.00713732523259|  42.498064554142985|0.002990906916936997|0.15100059961506673|0.11412583394883222| 1.230620567752269|0.8856385749678322|\n",
            "|    min|                10|               0.08|                  0|              0.6|               0.009|                 10|                  10|             0.98711|               2.72|               0.22|                10|                 3|\n",
            "|    max|               9.9|                1.1|               1.66|              9.9|               0.346|                 98|                  99|             1.03898|               3.82|               1.08|               9.9|                 9|\n",
            "+-------+------------------+-------------------+-------------------+-----------------+--------------------+-------------------+--------------------+--------------------+-------------------+-------------------+------------------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "columns = df_spark.columns\n",
        "\n",
        "columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GTte8u--da08",
        "outputId": "13455c76-b532-45c6-a394-a7c04cde4106"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['fixed acidity',\n",
              " 'volatile acidity',\n",
              " 'citric acid',\n",
              " 'residual sugar',\n",
              " 'chlorides',\n",
              " 'free sulfur dioxide',\n",
              " 'total sulfur dioxide',\n",
              " 'density',\n",
              " 'pH',\n",
              " 'sulphates',\n",
              " 'alcohol',\n",
              " 'quality']"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for column  in columns:\n",
        "  df_spark = df_spark.withColumn(column, df_spark[column].cast(\"float\"))"
      ],
      "metadata": {
        "id": "O0yk3gmgcF2L"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4AzX-BH0us0d",
        "outputId": "fa62fda2-171c-4e68-a377-3e5ad7d1d0ef"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- fixed acidity: float (nullable = true)\n",
            " |-- volatile acidity: float (nullable = true)\n",
            " |-- citric acid: float (nullable = true)\n",
            " |-- residual sugar: float (nullable = true)\n",
            " |-- chlorides: float (nullable = true)\n",
            " |-- free sulfur dioxide: float (nullable = true)\n",
            " |-- total sulfur dioxide: float (nullable = true)\n",
            " |-- density: float (nullable = true)\n",
            " |-- pH: float (nullable = true)\n",
            " |-- sulphates: float (nullable = true)\n",
            " |-- alcohol: float (nullable = true)\n",
            " |-- quality: float (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_spark.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bzq852nNuvTN",
        "outputId": "8f3c716e-601a-49be-9a05-b6316dc0ebdf"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+\n",
            "|fixed acidity|volatile acidity|citric acid|residual sugar|chlorides|free sulfur dioxide|total sulfur dioxide|density|  pH|sulphates|alcohol|quality|\n",
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+\n",
            "|          7.0|            0.27|       0.36|          20.7|    0.045|               45.0|               170.0|  1.001| 3.0|     0.45|    8.8|    6.0|\n",
            "|          6.3|             0.3|       0.34|           1.6|    0.049|               14.0|               132.0|  0.994| 3.3|     0.49|    9.5|    6.0|\n",
            "|          8.1|            0.28|        0.4|           6.9|     0.05|               30.0|                97.0| 0.9951|3.26|     0.44|   10.1|    6.0|\n",
            "|          7.2|            0.23|       0.32|           8.5|    0.058|               47.0|               186.0| 0.9956|3.19|      0.4|    9.9|    6.0|\n",
            "|          7.2|            0.23|       0.32|           8.5|    0.058|               47.0|               186.0| 0.9956|3.19|      0.4|    9.9|    6.0|\n",
            "|          8.1|            0.28|        0.4|           6.9|     0.05|               30.0|                97.0| 0.9951|3.26|     0.44|   10.1|    6.0|\n",
            "|          6.2|            0.32|       0.16|           7.0|    0.045|               30.0|               136.0| 0.9949|3.18|     0.47|    9.6|    6.0|\n",
            "|          7.0|            0.27|       0.36|          20.7|    0.045|               45.0|               170.0|  1.001| 3.0|     0.45|    8.8|    6.0|\n",
            "|          6.3|             0.3|       0.34|           1.6|    0.049|               14.0|               132.0|  0.994| 3.3|     0.49|    9.5|    6.0|\n",
            "|          8.1|            0.22|       0.43|           1.5|    0.044|               28.0|               129.0| 0.9938|3.22|     0.45|   11.0|    6.0|\n",
            "|          8.1|            0.27|       0.41|          1.45|    0.033|               11.0|                63.0| 0.9908|2.99|     0.56|   12.0|    5.0|\n",
            "|          8.6|            0.23|        0.4|           4.2|    0.035|               17.0|               109.0| 0.9947|3.14|     0.53|    9.7|    5.0|\n",
            "|          7.9|            0.18|       0.37|           1.2|     0.04|               16.0|                75.0|  0.992|3.18|     0.63|   10.8|    5.0|\n",
            "|          6.6|            0.16|        0.4|           1.5|    0.044|               48.0|               143.0| 0.9912|3.54|     0.52|   12.4|    7.0|\n",
            "|          8.3|            0.42|       0.62|         19.25|     0.04|               41.0|               172.0| 1.0002|2.98|     0.67|    9.7|    5.0|\n",
            "|          6.6|            0.17|       0.38|           1.5|    0.032|               28.0|               112.0| 0.9914|3.25|     0.55|   11.4|    7.0|\n",
            "|          6.3|            0.48|       0.04|           1.1|    0.046|               30.0|                99.0| 0.9928|3.24|     0.36|    9.6|    6.0|\n",
            "|          6.2|            0.66|       0.48|           1.2|    0.029|               29.0|                75.0| 0.9892|3.33|     0.39|   12.8|    8.0|\n",
            "|          7.4|            0.34|       0.42|           1.1|    0.033|               17.0|               171.0| 0.9917|3.12|     0.53|   11.3|    6.0|\n",
            "|          6.5|            0.31|       0.14|           7.5|    0.044|               34.0|               133.0| 0.9955|3.22|      0.5|    9.5|    5.0|\n",
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training = df_spark"
      ],
      "metadata": {
        "id": "3nSHWsJ9u_09"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D6AHTP5cvefN",
        "outputId": "605dd156-2966-4d81-f5f3-ef693798ce95"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['fixed acidity',\n",
              " 'volatile acidity',\n",
              " 'citric acid',\n",
              " 'residual sugar',\n",
              " 'chlorides',\n",
              " 'free sulfur dioxide',\n",
              " 'total sulfur dioxide',\n",
              " 'density',\n",
              " 'pH',\n",
              " 'sulphates',\n",
              " 'alcohol',\n",
              " 'quality']"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training = df_spark.drop(\"quality\")\n",
        "target = df_spark.select(\"quality\")"
      ],
      "metadata": {
        "id": "-ozKHH8xv0tl"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U2dPkUYIv7le",
        "outputId": "b9ed7235-92b0-4be7-d7f6-e6d9655988e0"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['fixed acidity',\n",
              " 'volatile acidity',\n",
              " 'citric acid',\n",
              " 'residual sugar',\n",
              " 'chlorides',\n",
              " 'free sulfur dioxide',\n",
              " 'total sulfur dioxide',\n",
              " 'density',\n",
              " 'pH',\n",
              " 'sulphates',\n",
              " 'alcohol']"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "target.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0e9GK_GXwRst",
        "outputId": "79390ef0-fc8e-45bb-9e31-8ae6231bee3c"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['quality']"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "featureAssembler = VectorAssembler(inputCols=training.columns,outputCol=\"Inpedent Featrues\")"
      ],
      "metadata": {
        "id": "h3uwTTgYvgp8"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output = featureAssembler.transform(df_spark)"
      ],
      "metadata": {
        "id": "fFpbAN81wgVt"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CeK-Letrw6qO",
        "outputId": "1a5a0760-c9b7-4814-bb1e-afe301816920"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+--------------------+\n",
            "|fixed acidity|volatile acidity|citric acid|residual sugar|chlorides|free sulfur dioxide|total sulfur dioxide|density|  pH|sulphates|alcohol|quality|   Inpedent Featrues|\n",
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+--------------------+\n",
            "|          7.0|            0.27|       0.36|          20.7|    0.045|               45.0|               170.0|  1.001| 3.0|     0.45|    8.8|    6.0|[7.0,0.2700000107...|\n",
            "|          6.3|             0.3|       0.34|           1.6|    0.049|               14.0|               132.0|  0.994| 3.3|     0.49|    9.5|    6.0|[6.30000019073486...|\n",
            "|          8.1|            0.28|        0.4|           6.9|     0.05|               30.0|                97.0| 0.9951|3.26|     0.44|   10.1|    6.0|[8.10000038146972...|\n",
            "|          7.2|            0.23|       0.32|           8.5|    0.058|               47.0|               186.0| 0.9956|3.19|      0.4|    9.9|    6.0|[7.19999980926513...|\n",
            "|          7.2|            0.23|       0.32|           8.5|    0.058|               47.0|               186.0| 0.9956|3.19|      0.4|    9.9|    6.0|[7.19999980926513...|\n",
            "|          8.1|            0.28|        0.4|           6.9|     0.05|               30.0|                97.0| 0.9951|3.26|     0.44|   10.1|    6.0|[8.10000038146972...|\n",
            "|          6.2|            0.32|       0.16|           7.0|    0.045|               30.0|               136.0| 0.9949|3.18|     0.47|    9.6|    6.0|[6.19999980926513...|\n",
            "|          7.0|            0.27|       0.36|          20.7|    0.045|               45.0|               170.0|  1.001| 3.0|     0.45|    8.8|    6.0|[7.0,0.2700000107...|\n",
            "|          6.3|             0.3|       0.34|           1.6|    0.049|               14.0|               132.0|  0.994| 3.3|     0.49|    9.5|    6.0|[6.30000019073486...|\n",
            "|          8.1|            0.22|       0.43|           1.5|    0.044|               28.0|               129.0| 0.9938|3.22|     0.45|   11.0|    6.0|[8.10000038146972...|\n",
            "|          8.1|            0.27|       0.41|          1.45|    0.033|               11.0|                63.0| 0.9908|2.99|     0.56|   12.0|    5.0|[8.10000038146972...|\n",
            "|          8.6|            0.23|        0.4|           4.2|    0.035|               17.0|               109.0| 0.9947|3.14|     0.53|    9.7|    5.0|[8.60000038146972...|\n",
            "|          7.9|            0.18|       0.37|           1.2|     0.04|               16.0|                75.0|  0.992|3.18|     0.63|   10.8|    5.0|[7.90000009536743...|\n",
            "|          6.6|            0.16|        0.4|           1.5|    0.044|               48.0|               143.0| 0.9912|3.54|     0.52|   12.4|    7.0|[6.59999990463256...|\n",
            "|          8.3|            0.42|       0.62|         19.25|     0.04|               41.0|               172.0| 1.0002|2.98|     0.67|    9.7|    5.0|[8.30000019073486...|\n",
            "|          6.6|            0.17|       0.38|           1.5|    0.032|               28.0|               112.0| 0.9914|3.25|     0.55|   11.4|    7.0|[6.59999990463256...|\n",
            "|          6.3|            0.48|       0.04|           1.1|    0.046|               30.0|                99.0| 0.9928|3.24|     0.36|    9.6|    6.0|[6.30000019073486...|\n",
            "|          6.2|            0.66|       0.48|           1.2|    0.029|               29.0|                75.0| 0.9892|3.33|     0.39|   12.8|    8.0|[6.19999980926513...|\n",
            "|          7.4|            0.34|       0.42|           1.1|    0.033|               17.0|               171.0| 0.9917|3.12|     0.53|   11.3|    6.0|[7.40000009536743...|\n",
            "|          6.5|            0.31|       0.14|           7.5|    0.044|               34.0|               133.0| 0.9955|3.22|      0.5|    9.5|    5.0|[6.5,0.3100000023...|\n",
            "+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "finalised_data = output.select(\"Inpedent Featrues\",\"quality\")"
      ],
      "metadata": {
        "id": "xTlJGqSIw8Mc"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "finalised_data.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ONPhteBuxPPN",
        "outputId": "de498a40-3e60-405d-ad41-6470165afc1a"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-------+\n",
            "|   Inpedent Featrues|quality|\n",
            "+--------------------+-------+\n",
            "|[7.0,0.2700000107...|    6.0|\n",
            "|[6.30000019073486...|    6.0|\n",
            "|[8.10000038146972...|    6.0|\n",
            "|[7.19999980926513...|    6.0|\n",
            "|[7.19999980926513...|    6.0|\n",
            "|[8.10000038146972...|    6.0|\n",
            "|[6.19999980926513...|    6.0|\n",
            "|[7.0,0.2700000107...|    6.0|\n",
            "|[6.30000019073486...|    6.0|\n",
            "|[8.10000038146972...|    6.0|\n",
            "|[8.10000038146972...|    5.0|\n",
            "|[8.60000038146972...|    5.0|\n",
            "|[7.90000009536743...|    5.0|\n",
            "|[6.59999990463256...|    7.0|\n",
            "|[8.30000019073486...|    5.0|\n",
            "|[6.59999990463256...|    7.0|\n",
            "|[6.30000019073486...|    6.0|\n",
            "|[6.19999980926513...|    8.0|\n",
            "|[7.40000009536743...|    6.0|\n",
            "|[6.5,0.3100000023...|    5.0|\n",
            "+--------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "train_data, test_data = finalised_data.randomSplit([0.75,0.25])\n",
        "regressor = LinearRegression(featuresCol=\"Inpedent Featrues\",labelCol=\"quality\")\n",
        "regressor = regressor.fit(train_data)"
      ],
      "metadata": {
        "id": "8xf5b2lixjGs"
      },
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "regressor.coefficients"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FvbVK-YyHjN",
        "outputId": "0d94543d-d35e-4270-88ac-cc25ed5413c3"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DenseVector([0.1049, -1.7877, -0.0506, 0.095, -0.4355, 0.0038, 0.0, -198.5572, 0.8209, 0.6204, 0.1432])"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "regressor.intercept"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6PPUfZH2yLB9",
        "outputId": "ad94d07d-4ee0-483e-a17e-28f0e60ef012"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "197.88970846582902"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prediction = regressor.evaluate(test_data)"
      ],
      "metadata": {
        "id": "A4s_ohclyOZN"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prediction.predictions.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N1W_57DOyU1d",
        "outputId": "7845e323-9d7f-4637-c9fe-09d367379a80"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-------+------------------+\n",
            "|   Inpedent Featrues|quality|        prediction|\n",
            "+--------------------+-------+------------------+\n",
            "|[4.59999990463256...|    5.0| 5.233917008855855|\n",
            "|[4.69999980926513...|    5.0| 5.803705282145643|\n",
            "|[4.69999980926513...|    5.0| 6.135023765517957|\n",
            "|[4.80000019073486...|    7.0|  6.76205730107219|\n",
            "|[4.80000019073486...|    7.0| 6.420770150941337|\n",
            "|[4.80000019073486...|    7.0|  6.54897265391898|\n",
            "|[4.80000019073486...|    5.0| 5.787913942142836|\n",
            "|[4.80000019073486...|    6.0|5.7346959031388565|\n",
            "|[4.80000019073486...|    4.0| 5.162563204892962|\n",
            "|[4.90000009536743...|    6.0|  5.74311388548378|\n",
            "|[4.90000009536743...|    5.0|  5.81991253857845|\n",
            "|[4.90000009536743...|    5.0| 5.474389274051845|\n",
            "|[5.0,0.2399999946...|    6.0|  6.61440189465759|\n",
            "|[5.0,0.2549999952...|    6.0| 6.493937982512733|\n",
            "|[5.0,0.2700000107...|    7.0| 6.905946557364473|\n",
            "|[5.0,0.4550000131...|    7.0| 6.976980725614851|\n",
            "|[5.0,0.6100000143...|    5.0|  6.47635059939833|\n",
            "|[5.09999990463256...|    5.0| 6.179685457841998|\n",
            "|[5.09999990463256...|    6.0| 5.401600441226208|\n",
            "|[5.09999990463256...|    5.0|6.3951457972930825|\n",
            "+--------------------+-------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prediction.meanSquaredError"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lNFUYsbxyXky",
        "outputId": "a0c39838-9ef6-4ae5-af1a-c05fb9040b4d"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5626814518548674"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prediction.meanAbsoluteError"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C_C6c-kcyofO",
        "outputId": "e84a7a63-4f4b-4741-de87-832f72d0eb24"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5856804034677476"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "r5bpCwwRytce"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}