hpo2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hpo2.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/poojitharamachandra/HPO/blob/master/hpo2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "eXSBvcIlvDYk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "import torch.nn as nn\n",
        "\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "import torchvision.datasets as dsets\n",
        "\n",
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "from torch.autograd import Variable\n",
        "\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SiNxrNrFGvlT",
        "colab_type": "code",
        "outputId": "96ac19c3-c52d-43f0-87a4-f894849e21ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "cell_type": "code",
      "source": [
        "!pip3 install torchvision"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.2.2.post3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.12.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.16.3)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (4.3.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.0.1.post2)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision) (0.46)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wvrcWGPjvDZC",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class CNNModel(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "\n",
        "        super(CNNModel, self).__init__()\n",
        "        # Convolution 1\n",
        "\n",
        "        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=3, stride=1, padding=0)\n",
        "\n",
        "        self.relu1 = nn.ReLU()\n",
        "\n",
        "        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n",
        "\n",
        "     \n",
        "\n",
        "        # Convolution 2\n",
        "\n",
        "        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=0)\n",
        "\n",
        "        self.relu2 = nn.ReLU()\n",
        "\n",
        "        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n",
        "        \n",
        "        \n",
        "        # Convolution 3\n",
        "\n",
        "        #self.cnn3 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=0)\n",
        "\n",
        "        #self.relu3 = nn.ReLU()\n",
        "\n",
        "        #self.maxpool3 = nn.MaxPool2d(kernel_size=2)\n",
        "\n",
        "        \n",
        "\n",
        "        # Fully connected 1 (readout)\n",
        "        #10 classes\n",
        "\n",
        "        self.fc1 = nn.Linear(32* 5 * 5, 10) \n",
        "\n",
        "    \n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Convolution 1\n",
        "\n",
        "        out = self.cnn1(x)\n",
        "\n",
        "        out = self.relu1(out)\n",
        "\n",
        "        out = self.maxpool1(out)     \n",
        "\n",
        "        # Convolution 2 \n",
        "\n",
        "        out = self.cnn2(out)\n",
        "\n",
        "        out = self.relu2(out) \n",
        "\n",
        "        out = self.maxpool2(out)\n",
        "        \n",
        "        # Convolution 3 \n",
        "\n",
        "        #out = self.cnn3(out)\n",
        "\n",
        "        #out = self.relu3(out) \n",
        "\n",
        "        #out = self.maxpool3(out)\n",
        "\n",
        "        \n",
        "\n",
        "        # Resize\n",
        "\n",
        "        # Original size: (100, 32, 7, 7)\n",
        "\n",
        "        # out.size(0): 100\n",
        "\n",
        "        # New out size: (100, 32*7*7)\n",
        "\n",
        "        out = out.view(out.size(0), -1)\n",
        "\n",
        "\n",
        "        # Linear function (readout)\n",
        "\n",
        "        out = self.fc1(out)   \n",
        "\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1pfU7AnNvDY6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "import requests\n",
        "import numpy as np\n",
        "from torch.utils.data import Dataset\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "\n",
        "\n",
        "class KMNIST(Dataset):\n",
        "    \"\"\"\n",
        "    Dataset class for use with pytorch for the Kuzushiji-MNIST dataset as given in\n",
        "    Deep Learning for Classical Japanese Literature. Tarin Clanuwat et al. arXiv:1812.01718\n",
        "\n",
        "    Kuzushiji-MNIST contains 70,000 28x28 grayscale images spanning 10 classes (one from each column of hiragana),\n",
        "    and is perfectly balanced like the original MNIST dataset (6k/1k train/test for each class).\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, data_dir='.', train: bool = True, transform=None):\n",
        "        \"\"\"\n",
        "        :param data_dir: Directory of the data\n",
        "        :param train: Use training or test set\n",
        "        :param transform: pytorch transforms for data augmentation\n",
        "        \"\"\"\n",
        "\n",
        "        self.__urls = [\n",
        "            'http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-imgs.npz',\n",
        "            'http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-labels.npz',\n",
        "            'http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-imgs.npz',\n",
        "            'http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-labels.npz',\n",
        "        ]\n",
        "\n",
        "        t_str = 'train' if train else 'test'\n",
        "        imgs_fn = 'kmnist-{}-imgs.npz'.format(t_str)\n",
        "        labels_fn = 'kmnist-{}-labels.npz'.format(t_str)\n",
        "\n",
        "        if not os.path.exists(data_dir):\n",
        "            os.mkdir(os.path.abspath(data_dir))\n",
        "        if not os.path.exists(os.path.abspath(os.path.join(data_dir, imgs_fn))):\n",
        "            self.__download(os.path.abspath(data_dir))\n",
        "\n",
        "        imgs_fn = os.path.abspath(os.path.join(data_dir, imgs_fn))\n",
        "        labels_fn = os.path.abspath(os.path.join(data_dir, labels_fn))\n",
        "\n",
        "        self.images = np.load(imgs_fn)['arr_0']\n",
        "        self.labels = np.load(labels_fn)['arr_0']\n",
        "        self.n_classes = len(np.unique(self.labels))\n",
        "        self.class_labels, self.class_frequency = np.unique(self.labels, return_counts=True)\n",
        "        self.class_frequency = self.class_frequency / np.sum(self.class_frequency)\n",
        "        self.data_dir = data_dir\n",
        "        self.img_rows = 28\n",
        "        self.img_cols = 28\n",
        "        self.channels = 1  # only gray scale\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        image = np.expand_dims(self.images[idx], axis=-1)\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        label = np.int(self.labels[idx])\n",
        "        return image, label\n",
        "\n",
        "    def __download(self, data_dir):\n",
        "        print('Datadir', data_dir)\n",
        "        for url in self.__urls:\n",
        "            fn = os.path.basename(url)\n",
        "            req = requests.get(url, stream=True)\n",
        "            print('Downloading {}'.format(fn))\n",
        "            with open(os.path.join(data_dir, fn), 'wb') as fh:\n",
        "                for chunck in req.iter_content(chunk_size=1024):\n",
        "                    if chunck:\n",
        "                        fh.write(chunck)\n",
        "            print('done')\n",
        "        print('All files downloaded')\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nBe70Qd4PJxJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_dataset=KMNIST(transform=transforms.ToTensor())\n",
        "test_dataset=KMNIST(train=False, transform=transforms.ToTensor())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-HHe5hmYpxFW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "na3Ga7lN0u_m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def prepare_cnn(cfg):\n",
        "  print(cfg)\n",
        "  cfg = {k : cfg[k] for k in cfg if cfg[k]}\n",
        "  print(cfg)\n",
        "  run_cnn(**cfg)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8kBo_TtL8Yzq",
        "colab_type": "code",
        "outputId": "f3b5b7c2-82d5-4b93-8a0c-89eb34f819ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "cell_type": "code",
      "source": [
        "!apt-get install build-essential swig"
      ],
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "build-essential is already the newest version (12.4ubuntu1).\n",
            "swig is already the newest version (3.0.12-1).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-410\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 6 not upgraded.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7nqZ6YZzAvzg",
        "colab_type": "code",
        "outputId": "68542a18-2843-4970-ec44-b0ed8ec0fcd0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1401
        }
      },
      "cell_type": "code",
      "source": [
        "!curl https://raw.githubusercontent.com/automl/smac3/master/requirements.txt | xargs -n 1 -L 1 pip install"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100   220  100   220    0     0   1189      0 --:--:-- --:--:-- --:--:--  1189\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (40.9.0)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (0.29.7)\n",
            "Requirement already satisfied: numpy>=1.7.1 in /usr/local/lib/python3.6/dist-packages (1.16.3)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (1.2.1)\n",
            "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from scipy>=0.18.1) (1.16.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (1.12.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (5.4.8)\n",
            "Requirement already satisfied: pynisher>=0.4.1 in /usr/local/lib/python3.6/dist-packages (0.5.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (from pynisher>=0.4.1) (5.4.8)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pynisher>=0.4.1) (40.9.0)\n",
            "Requirement already satisfied: docutils>=0.3 in /usr/local/lib/python3.6/dist-packages (from pynisher>=0.4.1) (0.14)\n",
            "Requirement already satisfied: ConfigSpace<0.5,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (0.4.10)\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from ConfigSpace<0.5,>=0.4.6) (0.29.7)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from ConfigSpace<0.5,>=0.4.6) (1.16.3)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.6/dist-packages (from ConfigSpace<0.5,>=0.4.6) (2.4.0)\n",
            "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from ConfigSpace<0.5,>=0.4.6) (3.6.6)\n",
            "Requirement already satisfied: scikit-learn>=0.18.0 in /usr/local/lib/python3.6/dist-packages (0.20.3)\n",
            "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18.0) (1.16.3)\n",
            "Requirement already satisfied: scipy>=0.13.3 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18.0) (1.2.1)\n",
            "Requirement already satisfied: pyrfr>=0.5.0 in /usr/local/lib/python3.6/dist-packages (0.8.0)\n",
            "Requirement already satisfied: sphinx in /usr/local/lib/python3.6/dist-packages (1.8.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from sphinx) (1.12.0)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.6/dist-packages (from sphinx) (2.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from sphinx) (40.9.0)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.6/dist-packages (from sphinx) (2.10.1)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.6/dist-packages (from sphinx) (1.2.1)\n",
            "Requirement already satisfied: docutils>=0.11 in /usr/local/lib/python3.6/dist-packages (from sphinx) (0.14)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.6/dist-packages (from sphinx) (0.7.12)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from sphinx) (2.21.0)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.6/dist-packages (from sphinx) (1.1.0)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.6/dist-packages (from sphinx) (2.6.0)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.6/dist-packages (from sphinx) (1.1.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from sphinx) (19.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.3->sphinx) (1.1.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx) (1.24.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx) (2019.3.9)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx) (2.8)\n",
            "Requirement already satisfied: pytz>=0a in /usr/local/lib/python3.6/dist-packages (from babel!=2.0,>=1.3->sphinx) (2018.9)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->sphinx) (2.4.0)\n",
            "Requirement already satisfied: sphinx_rtd_theme in /usr/local/lib/python3.6/dist-packages (0.4.3)\n",
            "Requirement already satisfied: sphinx in /usr/local/lib/python3.6/dist-packages (from sphinx_rtd_theme) (1.8.5)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (2.6.0)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (2.21.0)\n",
            "Requirement already satisfied: docutils>=0.11 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (0.14)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (2.10.1)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (1.1.0)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (1.1.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (40.9.0)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (2.1.3)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (1.2.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (19.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (1.12.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.6/dist-packages (from sphinx->sphinx_rtd_theme) (0.7.12)\n",
            "Requirement already satisfied: pytz>=0a in /usr/local/lib/python3.6/dist-packages (from babel!=2.0,>=1.3->sphinx->sphinx_rtd_theme) (2018.9)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->sphinx_rtd_theme) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->sphinx_rtd_theme) (2019.3.9)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->sphinx_rtd_theme) (1.24.2)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->sphinx_rtd_theme) (2.8)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.3->sphinx->sphinx_rtd_theme) (1.1.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->sphinx->sphinx_rtd_theme) (2.4.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (0.12.5)\n",
            "Requirement already satisfied: nose>=1.3.0 in /usr/local/lib/python3.6/dist-packages (1.3.7)\n",
            "Requirement already satisfied: pyDOE in /usr/local/lib/python3.6/dist-packages (0.3.8)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from pyDOE) (1.2.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pyDOE) (1.16.3)\n",
            "Requirement already satisfied: sobol_seq in /usr/local/lib/python3.6/dist-packages (0.1.2)\n",
            "Requirement already satisfied: statsmodels in /usr/local/lib/python3.6/dist-packages (0.9.0)\n",
            "Requirement already satisfied: patsy in /usr/local/lib/python3.6/dist-packages (from statsmodels) (0.5.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from statsmodels) (0.24.2)\n",
            "Requirement already satisfied: numpy>=1.4 in /usr/local/lib/python3.6/dist-packages (from patsy->statsmodels) (1.16.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from patsy->statsmodels) (1.12.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/dist-packages (from pandas->statsmodels) (2.5.3)\n",
            "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas->statsmodels) (2018.9)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "vjgs9W8evDZf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def run_cnn(lr):\n",
        "  \n",
        "  print(\"lr-------\",lr)\n",
        "  \n",
        "  learning_rate = lr\n",
        "  \n",
        "  epochs=5\n",
        "  \n",
        "  model = CNNModel()\n",
        "  \n",
        "  if torch.cuda.is_available():\n",
        "      model.cuda()\n",
        "      \n",
        "  batch_size=128\n",
        "  \n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, \n",
        "                             betas=(0.9, 0.999), eps=1e-08, weight_decay=0.0001, amsgrad=False)\n",
        "  train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, \n",
        "                                           shuffle=True)\n",
        "  test_loader = torch.utils.data.DataLoader(dataset=test_dataset,  batch_size=batch_size,\n",
        "                                          shuffle=False )\n",
        "  iter = 0\n",
        "\n",
        "  for epoch in range(epochs):\n",
        "      train_accuracies = np.zeros(epochs)\n",
        "      for i, (images, labels) in enumerate(train_loader):        \n",
        "          model.train()\n",
        "          if torch.cuda.is_available():\n",
        "              images = Variable(images.cuda())\n",
        "              labels = Variable(labels.cuda())\n",
        "\n",
        "          else:\n",
        "              images = Variable(images)\n",
        "              labels = Variable(labels)\n",
        "\n",
        "          optimizer.zero_grad()\n",
        "          outputs = model(images)\n",
        "          loss = criterion(outputs, labels)\n",
        "          loss.backward()\n",
        "          optimizer.step()\n",
        "          model.eval()\n",
        "\n",
        "          for images, labels in test_loader:\n",
        "              correct = 0\n",
        "              total = 0\n",
        "              if torch.cuda.is_available():\n",
        "                  images = Variable(images.cuda())\n",
        "              else:\n",
        "                  images = Variable(images)\n",
        "              outputs = model(images)\n",
        "              _, predicted = torch.max(outputs.data, 1)\n",
        "              total += labels.size(0)\n",
        "                  \n",
        "              if torch.cuda.is_available():\n",
        "                  correct += (predicted.cpu() == labels.cpu()).sum()\n",
        "              else:\n",
        "                  correct += (predicted == labels).sum()             \n",
        "\n",
        "          accuracy = 100 * correct / total\n",
        "             \n",
        "      train_accuracies[epoch]=accuracy\n",
        "      print('Epoch: {}. Loss: {}. Accuracy: {}'.format(epoch, loss, accuracy))          \n",
        "      \n",
        "  print(train_accuracies[epochs-1])                         \n",
        "  return train_accuracies[epochs-1]\n",
        "             \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uvUX2MIuCe7_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install smac --no-cache"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KavB86X-5t-t",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Import ConfigSpace and different types of parameters\n",
        "from smac.configspace import ConfigurationSpace\n",
        "from ConfigSpace.hyperparameters import CategoricalHyperparameter, \\\n",
        "    UniformFloatHyperparameter, UniformIntegerHyperparameter\n",
        "from ConfigSpace.conditions import InCondition\n",
        "\n",
        "# Import SMAC-utilities\n",
        "from smac.tae.execute_func import ExecuteTAFuncDict\n",
        "from smac.scenario.scenario import Scenario\n",
        "from smac.facade.smac_facade import SMAC"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "68bSmDXyvDZq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "cs = ConfigurationSpace()\n",
        "\n",
        "lr = UniformFloatHyperparameter(\"lr\", 0.0001, 0.1, default_value=0.001)\n",
        "cs.add_hyperparameter(lr)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9uKrnV2TQWiz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#batch_size = CategoricalHyperparameter(\"batch_size\", [128, 256], default_value=128)\n",
        "#cs.add_hyperparameter(batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "EXXiV4eBwL7J",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Scenario object\n",
        "scenario = Scenario({\"run_obj\": \"quality\",   # we optimize quality (alternatively runtime)\n",
        "                     \"runcount-limit\": 5,  # maximum function evaluations\n",
        "                     \"cs\": cs,               # configuration space\n",
        "                     \"deterministic\": \"true\",\n",
        "                     #\"abort_on_first_run_crash\": \"false\"\n",
        "                     })"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "c-i-E1Y3vDZu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Optimize, using a SMAC-object\n",
        "print(\"Optimizing! Depending on your machine, this might take a few minutes.\")\n",
        "smac = SMAC(scenario=scenario, rng=np.random.RandomState(42),\n",
        "        tae_runner=prepare_cnn)\n",
        "\n",
        "print(\"searching for incumbent config!\")\n",
        "\n",
        "smac.solver.intensifier.tae_runner.use_pynisher = False\n",
        "\n",
        "incumbent = smac.optimize()\n",
        "\n",
        "inc_value = prepare_cnn(incumbent)\n",
        "\n",
        "print(\"Optimized Value: %.2f\" % (inc_value))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}