5th_sept_web_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To install requests-html and jupyter in virtualenv\n",
    "pipenv install requests-html jupyter\n",
    "### To start jupyter notebook from virtualenv\n",
    "pipenv run jupyter notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "from requests_html import HTMLSession\n",
    "\n",
    "session = HTMLSession()\n",
    "\n",
    "website_url = \"https://news.ycombinator.com/\"\n",
    "\n",
    "response = session.get(website_url)\n",
    "\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Why Finnish babies sleep in cardboard boxes (2013) (bbc.com)', 'Mach kernel (wikipedia.org)', 'Google: Enabling developers and organizations to use differential privacy (googleblog.com)', 'Graphics Programming Black Book by Michael Abrash – Markdown source (2001) (github.com)', 'Human speech may have a universal transmission rate: 39 bits per second (sciencemag.org)', 'Secushare: A decentralized, secure social network built on GNUnet (secushare.org)', 'A database of Facebook users’ phone numbers found online (techcrunch.com)', 'The Next Hot Job: Pretending to Be a Robot (wsj.com)', 'Weighted blankets might ease insomnia and anxiety (washingtonpost.com)', 'Creating a Magical Videobooth for a Wedding (jamesporter.me)', 'Johns Hopkins Launches Center for Psychedelic Research (jhu.edu)', 'Ask HN: Good Remote Work Literature?', 'Sally Floyd, who helped things run smoothly online, has died (nytimes.com)', 'Judge Rules Terrorism Watchlist Violates Constitutional Rights (nytimes.com)', 'Drone Bubble Bursts, Wiping Out Startups and Hammering VC Firms (bloomberg.com)', 'Woman is first to receive cornea made from ‘reprogrammed’ stem cells (nature.com)', 'Browser Fingerprinting: An Introduction and the Challenges Ahead (torproject.org)', 'In an area without cellphone reception, I was unable to open the car (theatlantic.com)', 'Amazon’s Shipping Empire Is Challenging UPS and FedEx (wsj.com)', \"Holes in T. rex's skull now thought to have regulated body temperature (sciencedaily.com)\", 'CircleCI Security Incident (circleci.com)', 'From bitter loss to sweet relief: baking as therapy (spectator.co.uk)', 'A mile or two off Yarmouth (2012) (bbc.co.uk)', 'Lego to open 160 new stores as other toy retailers suffer (ft.com)', 'Google’s GDPR Workaround (brave.com)', 'Tesla’s Autopilot found partly to blame for 2018 crash on the 405 (latimes.com)', 'Whatever happened to Six Sigma? (qz.com)', 'Fuzzy and semantic search for captioned YouTube videos (github.com)', 'Dr. Donald Lindberg, who opened medical research to the world, has died (nytimes.com)', 'Show HN: fast visual-inertial odometry/SLAM for AR/VR/Robotics (github.com)']\n"
     ]
    }
   ],
   "source": [
    "links_table = response.html.find(\".itemlist\", first=True)\n",
    "\n",
    "article_name_and_link_list = links_table.find(\".athing\")\n",
    "\n",
    "article_headlines = []\n",
    "\n",
    "for article_html in article_name_and_link_list:\n",
    "    article_headlines.append(article_html.text.split(\"\\n\")[1])\n",
    "    \n",
    "print(article_headlines)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rank': 1, 'headline': 'Why Finnish babies sleep in cardboard boxes (2013) (bbc.com)', 'url': 'https://www.bbc.com/news/magazine-22751415'}\n",
      "{'rank': 2, 'headline': 'Mach kernel (wikipedia.org)', 'url': 'https://en.wikipedia.org/wiki/Mach_(kernel)'}\n",
      "{'rank': 3, 'headline': 'Google: Enabling developers and organizations to use differential privacy (googleblog.com)', 'url': 'https://developers.googleblog.com/2019/09/enabling-developers-and-organizations.html'}\n",
      "{'rank': 4, 'headline': 'Graphics Programming Black Book by Michael Abrash – Markdown source (2001) (github.com)', 'url': 'https://github.com/jagregory/abrash-black-book'}\n",
      "{'rank': 5, 'headline': 'Human speech may have a universal transmission rate: 39 bits per second (sciencemag.org)', 'url': 'https://www.sciencemag.org/news/2019/09/human-speech-may-have-universal-transmission-rate-39-bits-second'}\n",
      "{'rank': 6, 'headline': 'Secushare: A decentralized, secure social network built on GNUnet (secushare.org)', 'url': 'https://secushare.org/'}\n",
      "{'rank': 7, 'headline': 'A database of Facebook users’ phone numbers found online (techcrunch.com)', 'url': 'https://techcrunch.com/2019/09/04/facebook-phone-numbers-exposed/'}\n",
      "{'rank': 8, 'headline': 'The Next Hot Job: Pretending to Be a Robot (wsj.com)', 'url': 'https://www.wsj.com/articles/the-next-hot-job-pretending-to-be-a-robot-11567224001?mod=rsswn'}\n",
      "{'rank': 9, 'headline': 'Weighted blankets might ease insomnia and anxiety (washingtonpost.com)', 'url': 'https://www.washingtonpost.com/lifestyle/home/weighted-blankets-might-ease-insomnia-and-anxiety-heres-what-to-know-before-buying-one/2019/01/18/71b1aefa-1a6f-11e9-8813-cb9dec761e73_story.html'}\n",
      "{'rank': 10, 'headline': 'Creating a Magical Videobooth for a Wedding (jamesporter.me)', 'url': 'https://www.jamesporter.me/2019/08/30/wedding-videobooth.html'}\n",
      "{'rank': 11, 'headline': 'Johns Hopkins Launches Center for Psychedelic Research (jhu.edu)', 'url': 'https://hub.jhu.edu/2019/09/04/hopkins-launches-psychedelic-center/'}\n",
      "{'rank': 12, 'headline': 'Ask HN: Good Remote Work Literature?', 'url': 'item?id=20884974'}\n",
      "{'rank': 13, 'headline': 'Sally Floyd, who helped things run smoothly online, has died (nytimes.com)', 'url': 'https://www.nytimes.com/2019/09/04/science/sally-floyd-dead.html'}\n",
      "{'rank': 14, 'headline': 'Judge Rules Terrorism Watchlist Violates Constitutional Rights (nytimes.com)', 'url': 'https://www.nytimes.com/2019/09/04/us/politics/terrorism-watchlist-constitution.html'}\n",
      "{'rank': 15, 'headline': 'Drone Bubble Bursts, Wiping Out Startups and Hammering VC Firms (bloomberg.com)', 'url': 'https://www.bloomberg.com/news/articles/2019-08-30/drone-bubble-bursts-wiping-out-startups-and-hammering-vc-firms'}\n",
      "{'rank': 16, 'headline': 'Woman is first to receive cornea made from ‘reprogrammed’ stem cells (nature.com)', 'url': 'https://www.nature.com/articles/d41586-019-02597-2'}\n",
      "{'rank': 17, 'headline': 'Browser Fingerprinting: An Introduction and the Challenges Ahead (torproject.org)', 'url': 'https://blog.torproject.org/browser-fingerprinting-introduction-and-challenges-ahead'}\n",
      "{'rank': 18, 'headline': 'In an area without cellphone reception, I was unable to open the car (theatlantic.com)', 'url': 'https://www.theatlantic.com/ideas/archive/2019/09/zipcar-into-the-wilderness/597217/'}\n",
      "{'rank': 19, 'headline': 'Amazon’s Shipping Empire Is Challenging UPS and FedEx (wsj.com)', 'url': 'https://www.wsj.com/articles/how-amazons-shipping-empire-is-challenging-ups-and-fedex-11567071003?mod=rsswn'}\n",
      "{'rank': 20, 'headline': \"Holes in T. rex's skull now thought to have regulated body temperature (sciencedaily.com)\", 'url': 'https://www.sciencedaily.com/releases/2019/09/190904125337.htm'}\n",
      "{'rank': 21, 'headline': 'CircleCI Security Incident (circleci.com)', 'url': 'https://support.circleci.com/hc/en-us/articles/360034852194-Security-Incident-on-8-31-2019-Details-and-FAQs-'}\n",
      "{'rank': 22, 'headline': 'From bitter loss to sweet relief: baking as therapy (spectator.co.uk)', 'url': 'https://www.spectator.co.uk/2019/08/from-bitter-loss-to-sweet-relief-baking-as-therapy/'}\n",
      "{'rank': 23, 'headline': 'A mile or two off Yarmouth (2012) (bbc.co.uk)', 'url': 'https://www.bbc.co.uk/blogs/adamcurtis/entries/512cde83-3afb-3048-9ece-dba774b10f89'}\n",
      "{'rank': 24, 'headline': 'Lego to open 160 new stores as other toy retailers suffer (ft.com)', 'url': 'https://www.ft.com/content/d2b7554e-ce18-11e9-99a4-b5ded7a7fe3f'}\n",
      "{'rank': 25, 'headline': 'Google’s GDPR Workaround (brave.com)', 'url': 'https://brave.com/google-gdpr-workaround/'}\n",
      "{'rank': 26, 'headline': 'Tesla’s Autopilot found partly to blame for 2018 crash on the 405 (latimes.com)', 'url': 'https://www.latimes.com/business/story/2019-09-04/tesla-autopilot-is-found-partly-to-blame-for-2018-freeway-crash'}\n",
      "{'rank': 27, 'headline': 'Whatever happened to Six Sigma? (qz.com)', 'url': 'https://qz.com/work/1635960/whatever-happened-to-six-sigma/'}\n",
      "{'rank': 28, 'headline': 'Fuzzy and semantic search for captioned YouTube videos (github.com)', 'url': 'https://github.com/RameshAditya/scoper'}\n",
      "{'rank': 29, 'headline': 'Dr. Donald Lindberg, who opened medical research to the world, has died (nytimes.com)', 'url': 'https://www.nytimes.com/2019/09/02/obituaries/dr-donald-lindberg-dead.html'}\n",
      "{'rank': 30, 'headline': 'Show HN: fast visual-inertial odometry/SLAM for AR/VR/Robotics (github.com)', 'url': 'https://github.com/ucla-vision/xivo'}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def article_html_to_dict(single_article):\n",
    "    article_rank_raw = single_article.find(\".rank\", first=True).text\n",
    "\n",
    "    article_rank = int(article_rank_raw.replace(\".\",\"\"))\n",
    "    article_url = single_article.find(\".storylink\", first=True).attrs[\"href\"]\n",
    "    article_headline = single_article.text.split(\"\\n\")[1]\n",
    "\n",
    "    article_data = {\"rank\": article_rank, \n",
    "                    \"headline\": article_headline, \n",
    "                    \"url\": article_url}\n",
    "    \n",
    "    return article_data\n",
    "\n",
    "for single_article in article_name_and_link_list:\n",
    "    print(article_html_to_dict(single_article))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Element 'td' class=('subtext',)>\n"
     ]
    }
   ],
   "source": [
    "single_article_subtext = response.html.find(\".subtext\")[0]\n",
    "\n",
    "print(single_article_subtext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}