blogabot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "import selenium\n",
    "from selenium import webdriver\n",
    "import time\n",
    "driver = webdriver.Firefox(executable_path=r'C:\\Users\\hp\\Downloads\\geckodriver-v0.23.0-win64\\geckodriver.exe')\n",
    "#now the tab will have opened, now we will enter the website. \n",
    "driver.get('http://www.youtube.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "StaleElementReferenceException",
     "evalue": "Message: The element reference of <paper-button id=\"button\" class=\"style-scope ytd-button-renderer style-primary size-default\"> is stale; either the element is no longer attached to the DOM, it is not in the current frame context, or the document has been refreshed\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mStaleElementReferenceException\u001b[0m            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-56a5774ae3a0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0melems\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_elements_by_tag_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"paper-button\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0melem\u001b[0m \u001b[1;32min\u001b[0m \u001b[0melems\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[1;32mif\u001b[0m \u001b[0melem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m\"SIGN IN\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m         \u001b[0melem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mtext\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m         \u001b[1;34m\"\"\"The text of the element.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGET_ELEMENT_TEXT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     77\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36m_execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    631\u001b[0m             \u001b[0mparams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    632\u001b[0m         \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 633\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    634\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    635\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mStaleElementReferenceException\u001b[0m: Message: The element reference of <paper-button id=\"button\" class=\"style-scope ytd-button-renderer style-primary size-default\"> is stale; either the element is no longer attached to the DOM, it is not in the current frame context, or the document has been refreshed\n"
     ]
    }
   ],
   "source": [
    "#the below two lines do not work, gives error\n",
    "#elem=driver.find_element_by_class_name('style-scope ytd-button-renderer style-blue-text size-default')\n",
    "#elem.click()\n",
    "elems=driver.find_elements_by_tag_name(\"paper-button\")\n",
    "for elem in elems:\n",
    "    if elem.text==\"SIGN IN\":\n",
    "        elem.click()\n",
    "#this gives an error, after activating the \"sign in\" box. Suggestions would be good to avoid this error         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_into_box=driver.find_element_by_name(\"identifier\")\n",
    "get_into_box.send_keys(\"put your email no. here\")\n",
    "get_into_box.submit()\n",
    "\n",
    "get_button_next=driver.find_elements_by_id(\"identifierNext\")\n",
    "for button in get_button_next:\n",
    "    button.click()\n",
    "time.sleep(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_pass=driver.find_element_by_name(\"password\")\n",
    "get_pass.send_keys(\"put your password here\")\n",
    "get_pass.submit()\n",
    "\n",
    "get_going=driver.find_element_by_id(\"passwordNext\")\n",
    "get_going.click()\n",
    "time.sleep(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_it=driver.find_element_by_name(\"search_query\")\n",
    "search_it.send_keys(\"web scraping\")\n",
    "search_it.submit()\n",
    "\n",
    "click_search=driver.find_element_by_id(\"search-icon-legacy\")\n",
    "click_search.click()\n",
    "#resource=driver.page_source\n",
    "#resources get a lot of things, but as you can see below, it does not let you see the hrefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "hrefs=driver.find_elements_by_tag_name('a')\n",
    "cd1=[]\n",
    "for href in hrefs:\n",
    "    buff=href.get_attribute('href')\n",
    "    if buff!=None and buff.split(\"/\")[2]=='www.youtube.com':\n",
    "        cd1.append(buff)\n",
    "#it worked once\n",
    "#lets try once more, but a bit more accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this bit does not work, as the whole source does not work, read the next block for explicit why it doesn't work\n",
    "'''from bs4 import BeautifulSoup\n",
    "soup=BeautifulSoup(resource,'html.parser')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"yt-simple-endpoint\" data=\"[[data.navigationEndpoint]]\" href$=\"[[computeHref_(data.navigationEndpoint)]]\" id=\"endpoint\" tabindex=\"-1\" title=\"[[getSimpleString(data.formattedTitle)]]\">\n",
       " <paper-item aria-label$=\"[[ariaLabel]]\" role=\"option\">\n",
       " <yt-icon class=\"guide-icon\" disable-upgrade$=\"[[data.thumbnail]]\" hidden=\"[[data.thumbnail]]\" icon=\"[[getIconType(data.icon.iconType)]]\">\n",
       " </yt-icon>\n",
       " <yt-img-shadow disable-upgrade$=\"[[!data.thumbnail]]\" height=\"24\" hidden=\"[[!data.thumbnail]]\" thumbnail=\"[[data.thumbnail]]\" width=\"24\">\n",
       " </yt-img-shadow>\n",
       " <span class=\"title\">[[getSimpleString(data.formattedTitle)]]</span>\n",
       " <span class=\"guide-entry-count\">\n",
       "           [[data.count]]\n",
       "         </span>\n",
       " <div class=\"style-scope ytd-thumbnail\" id=\"mouseover-overlay\"></div>\n",
       " <div class=\"style-scope ytd-thumbnail\" id=\"hover-overlays\"></div>\n",
       " </a>,\n",
       " <a aria-label=\"Dynamic Javascript Scraping - Web scraping with Beautiful Soup 4 p.4 by sentdex 2 years ago 15 minutes 85,549 views\" class=\"yt-simple-endpoint style-scope ytd-video-renderer\" href=\"/watch?v=FSH77vnOGqU\" id=\"video-title\" title=\"Dynamic Javascript Scraping - Web scraping with Beautiful Soup 4 p.4\">\n",
       "                 Dynamic Javascript Scraping - Web scraping with Beautiful Soup 4 p.4\n",
       "               </a>,\n",
       " <a class=\"yt-simple-endpoint style-scope yt-formatted-string\" href=\"/user/sentdex\" spellcheck=\"false\">sentdex</a>,\n",
       " <a href=\"//www.youtube.com/yt/about/\" slot=\"guide-links-primary\" style=\"display: none;\">About</a>,\n",
       " <a href=\"//www.youtube.com/yt/press/\" slot=\"guide-links-primary\" style=\"display: none;\">Press</a>,\n",
       " <a href=\"//www.youtube.com/yt/copyright/\" slot=\"guide-links-primary\" style=\"display: none;\">Copyright</a>,\n",
       " <a href=\"/t/contact_us\" slot=\"guide-links-primary\" style=\"display: none;\">Contact us</a>,\n",
       " <a href=\"//www.youtube.com/yt/creators/\" slot=\"guide-links-primary\" style=\"display: none;\">Creators</a>,\n",
       " <a href=\"//www.youtube.com/yt/advertise/\" slot=\"guide-links-primary\" style=\"display: none;\">Advertise</a>,\n",
       " <a href=\"//www.youtube.com/yt/dev/\" slot=\"guide-links-primary\" style=\"display: none;\">Developers</a>,\n",
       " <a href=\"/t/terms\" slot=\"guide-links-secondary\" style=\"display: none;\">Terms</a>,\n",
       " <a href=\"https://www.google.com/intl/en/policies/privacy/\" slot=\"guide-links-secondary\" style=\"display: none;\">Privacy</a>,\n",
       " <a href=\"//www.youtube.com/yt/policyandsafety/\" slot=\"guide-links-secondary\" style=\"display: none;\">Policy &amp; Safety</a>,\n",
       " <a href=\"/testtube\" slot=\"guide-links-secondary\" style=\"display: none;\">Test new features</a>,\n",
       " <a href=\"https://www.google.com/support/legal/answer/3110420\">Legal Help page</a>]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#offline downloaded source, hides all the links, upgrades, icons and other dynamic or interactive objects. Therefore soup is useless.\n",
    "#soup.find_all('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['watch', 'v=0_VZ7NpVw1Y']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#this helps to check whether it is a valid watchable href. Others have to be cancelled. That's why, this is the way to check.\n",
    "cd[10].split(\"/\")[3].split(\"?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.youtube.com/',\n",
       " 'https://www.youtube.com/upload',\n",
       " 'https://www.youtube.com/user/KriscoartProductions',\n",
       " 'https://www.youtube.com/watch?v=0_VZ7NpVw1Y',\n",
       " 'https://www.youtube.com/watch?v=0_VZ7NpVw1Y',\n",
       " 'https://www.youtube.com/user/hiteshitube',\n",
       " 'https://www.youtube.com/watch?v=XQgXKtPSzUI',\n",
       " 'https://www.youtube.com/watch?v=XQgXKtPSzUI',\n",
       " 'https://www.youtube.com/user/DataScienceDojo',\n",
       " 'https://www.youtube.com/watch?v=mKxFfjNyj3c',\n",
       " 'https://www.youtube.com/watch?v=mKxFfjNyj3c',\n",
       " 'https://www.youtube.com/user/edurekaIN',\n",
       " 'https://www.youtube.com/watch?v=lGF9gONeY-Q',\n",
       " 'https://www.youtube.com/watch?v=lGF9gONeY-Q',\n",
       " 'https://www.youtube.com/user/mrarpitkhurana155',\n",
       " 'https://www.youtube.com/watch?v=ZdOrH50WqEo',\n",
       " 'https://www.youtube.com/watch?v=ZdOrH50WqEo',\n",
       " 'https://www.youtube.com/channel/UC0uI3fwNpZe558n7ycVnSeg',\n",
       " 'https://www.youtube.com/watch?v=LoziivfAAjE',\n",
       " 'https://www.youtube.com/watch?v=LoziivfAAjE',\n",
       " 'https://www.youtube.com/user/TechGuyWeb',\n",
       " 'https://www.youtube.com/watch?v=ng2o98k983k',\n",
       " 'https://www.youtube.com/watch?v=ng2o98k983k',\n",
       " 'https://www.youtube.com/user/schafer5',\n",
       " 'https://www.youtube.com/watch?v=pvaAEYHf8QI',\n",
       " 'https://www.youtube.com/watch?v=pvaAEYHf8QI',\n",
       " 'https://www.youtube.com/channel/UCIKbbV7ae7LAWa8cGnvjSPA',\n",
       " 'https://www.youtube.com/watch?v=vsmxMLmroyQ',\n",
       " 'https://www.youtube.com/watch?v=vsmxMLmroyQ',\n",
       " 'https://www.youtube.com/channel/UCjFO5t0MLyQaidKGpGoRewg',\n",
       " 'https://www.youtube.com/watch?v=KaU7QaUM1RA',\n",
       " 'https://www.youtube.com/watch?v=KaU7QaUM1RA',\n",
       " 'https://www.youtube.com/user/ahmadFakharani',\n",
       " 'https://www.youtube.com/watch?v=1UYBAn69Qrk',\n",
       " 'https://www.youtube.com/watch?v=1UYBAn69Qrk',\n",
       " 'https://www.youtube.com/channel/UC8CB0ZkvogP7tnCTDR-zV7g',\n",
       " 'https://www.youtube.com/watch?v=eUYMiztBEdY',\n",
       " 'https://www.youtube.com/watch?v=eUYMiztBEdY',\n",
       " 'https://www.youtube.com/channel/UC8butISFwT-Wl7EV0hUK0BQ',\n",
       " 'https://www.youtube.com/watch?v=aIPqt-OdmS0',\n",
       " 'https://www.youtube.com/watch?v=aIPqt-OdmS0',\n",
       " 'https://www.youtube.com/user/sentdex',\n",
       " 'https://www.youtube.com/watch?v=3xQTJi2tqgk',\n",
       " 'https://www.youtube.com/watch?v=3xQTJi2tqgk',\n",
       " 'https://www.youtube.com/user/CodingEntrepreneurs',\n",
       " 'https://www.youtube.com/watch?v=X3paOmcrTjQ',\n",
       " 'https://www.youtube.com/watch?v=X3paOmcrTjQ',\n",
       " 'https://www.youtube.com/user/Simplilearn',\n",
       " 'https://www.youtube.com/watch?v=_uQrJ0TkZlc',\n",
       " 'https://www.youtube.com/watch?v=_uQrJ0TkZlc',\n",
       " 'https://www.youtube.com/user/programmingwithmosh',\n",
       " 'https://www.youtube.com/watch?v=8JfDAm9y_7s',\n",
       " 'https://www.youtube.com/watch?v=8JfDAm9y_7s',\n",
       " 'https://www.youtube.com/user/edurekaIN',\n",
       " 'https://www.youtube.com/watch?v=PI1-1TtFz50',\n",
       " 'https://www.youtube.com/watch?v=PI1-1TtFz50',\n",
       " 'https://www.youtube.com/user/hiteshitube',\n",
       " 'https://www.youtube.com/watch?v=a6fIbtFB46g',\n",
       " 'https://www.youtube.com/watch?v=a6fIbtFB46g',\n",
       " 'https://www.youtube.com/user/schafer5',\n",
       " 'https://www.youtube.com/watch?v=7QcNoVurj8U',\n",
       " 'https://www.youtube.com/watch?v=7QcNoVurj8U',\n",
       " 'https://www.youtube.com/user/jcartwri1',\n",
       " 'https://www.youtube.com/',\n",
       " 'https://www.youtube.com/feed/trending',\n",
       " 'https://www.youtube.com/feed/subscriptions',\n",
       " 'https://www.youtube.com/feed/library',\n",
       " 'https://www.youtube.com/feed/history',\n",
       " 'https://www.youtube.com/playlist?list=WL',\n",
       " 'https://www.youtube.com/playlist?list=LL4jqKcSXYRSJvsSD6mOs0EQ',\n",
       " 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',\n",
       " 'https://www.youtube.com/channel/UCh5bICCatQ70Fx4-jwAmWKQ',\n",
       " 'https://www.youtube.com/channel/UC0xRgccM7ghCMP2fpbFo3mA',\n",
       " 'https://www.youtube.com/channel/UCNJcSUSzUeFm8W9P7UUlSeQ',\n",
       " 'https://www.youtube.com/channel/UCiaIajIJcIbzr0_9BoCcdOA',\n",
       " 'https://www.youtube.com/channel/UCj22tfcQrWG7EMEKS0qLeEg',\n",
       " 'https://www.youtube.com/channel/UCOUbzLwquaQgKACIc2vJYdw',\n",
       " 'https://www.youtube.com/premium',\n",
       " 'https://www.youtube.com/channel/UClgRkhTL3_hImCAmdLfDE4g',\n",
       " 'https://www.youtube.com/gaming',\n",
       " 'https://www.youtube.com/channel/UC4R8DWoMoI7CAwX8_LjQHig',\n",
       " 'https://www.youtube.com/account',\n",
       " 'https://www.youtube.com/reporthistory',\n",
       " 'https://www.youtube.com/yt/about/',\n",
       " 'https://www.youtube.com/yt/press/',\n",
       " 'https://www.youtube.com/yt/copyright/',\n",
       " 'https://www.youtube.com/t/contact_us',\n",
       " 'https://www.youtube.com/yt/creators/',\n",
       " 'https://www.youtube.com/yt/advertise/',\n",
       " 'https://www.youtube.com/yt/dev/',\n",
       " 'https://www.youtube.com/t/terms',\n",
       " 'https://www.youtube.com/yt/policyandsafety/',\n",
       " 'https://www.youtube.com/testtube']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cd1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cd1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "ename": "NoSuchElementException",
     "evalue": "Message: Unable to locate element: [id=\"placeholder-area\"]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-60-5ce613f87084>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[1;31m#I will wait until placeholder is clickable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m         \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m30\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m         \u001b[0mcomment_box\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_id\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"placeholder-area\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m         \u001b[0mcomment_box\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element_by_id\u001b[1;34m(self, id_)\u001b[0m\n\u001b[0;32m    358\u001b[0m             \u001b[0melement\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_id\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'foo'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    359\u001b[0m         \"\"\"\n\u001b[1;32m--> 360\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mid_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    361\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    362\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements_by_id\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mid_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    976\u001b[0m         return self.execute(Command.FIND_ELEMENT, {\n\u001b[0;32m    977\u001b[0m             \u001b[1;34m'using'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 978\u001b[1;33m             'value': value})['value']\n\u001b[0m\u001b[0;32m    979\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    980\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: Unable to locate element: [id=\"placeholder-area\"]\n"
     ]
    }
   ],
   "source": [
    "for webpage in cd1:\n",
    "    if webpage.split(\"/\")[3].split(\"?\")[0]=='watch':\n",
    "        driver.get(webpage)\n",
    "        pause=driver.find_element_by_css_selector('button.ytp-play-button.ytp-button')\n",
    "        pause.click()\n",
    "        \n",
    "        #I will wait until placeholder is clickable\n",
    "        time.sleep(30)\n",
    "        comment_box=driver.find_element_by_id(\"placeholder-area\")\n",
    "        comment_box.click()\n",
    "\n",
    "        comment_now=driver.find_element_by_id(\"contenteditable-textarea\")\n",
    "        comment_now.send_keys(\"for excellent tutorial on web scraping, and readymade codes, please visit https://shyambhu20.blogspot.com/2019/01/web-scrapping-things-you-will-not-know.html\")\n",
    "        button=driver.find_element_by_id(\"submit-button\")\n",
    "        button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this is the portion of code, which finds the code and comments on the comment box.\n",
    "comment_box=driver.find_element_by_id(\"placeholder-area\")\n",
    "comment_box.click()\n",
    "\n",
    "comment_now=driver.find_element_by_id(\"contenteditable-textarea\")\n",
    "comment_now.send_keys(\"for excellent tutorial on web scraping, and readymade codes, please visit https://shyambhu20.blogspot.com/2019/01/web-scrapping-things-you-will-not-know.html\")\n",
    "button=driver.find_element_by_id(\"submit-button\")\n",
    "button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we have to use scrolling, or maybe zooming out into 33-50%\n",
    "#we have to create a set out of the hrefs.\n",
    "#we have to use scrolling and zooming out the youtube main query feed too, to get more videos\n",
    "#these are not done now,but you can pull request and we can discuss so."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}