notebooks/3_scrape_articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "from newspaper import Article\n",
    "from tqdm.auto import tqdm\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data directory\n",
    "data_dir = Path(\"../data\")\n",
    "\n",
    "input_dir = data_dir / \"input\"\n",
    "raw_dir = data_dir / \"raw\"\n",
    "output_dir = data_dir / \"processed\"\n",
    "sample_dir = data_dir / \"samples\"\n",
    "\n",
    "rss_dir = raw_dir / \"rss\"\n",
    "twitter_dir = raw_dir / \"twitter\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load all URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "rss_urls = pd.read_csv(raw_dir / \"cleaned_rss.csv\")\n",
    "tweet_urls = pd.read_csv(raw_dir / \"cleaned_tweets.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>channel</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>url_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>medpage</td>\n",
       "      <td>rss</td>\n",
       "      <td>https://www.medpagetoday.com/infectiousdisease...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>medpage</td>\n",
       "      <td>rss</td>\n",
       "      <td>https://www.medpagetoday.com/special-reports/e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>medpage</td>\n",
       "      <td>rss</td>\n",
       "      <td>https://www.medpagetoday.com/infectiousdisease...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>medpage</td>\n",
       "      <td>rss</td>\n",
       "      <td>https://www.medpagetoday.com/pulmonology/gener...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>medpage</td>\n",
       "      <td>rss</td>\n",
       "      <td>https://www.medpagetoday.com/infectiousdisease...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7120</th>\n",
       "      <td>popsci</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.popsci.com/story/diy/how-to-use-an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7121</th>\n",
       "      <td>popsci</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.popsci.com/how-to-make-perfect-pie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7122</th>\n",
       "      <td>popsci</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.popsci.com/story/technology/airbus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7123</th>\n",
       "      <td>popsci</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.popsci.com/story/gear/deer-birds-w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7124</th>\n",
       "      <td>popsci</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.popsci.com/story/reviews/best-knif...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7125 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         source  channel                                                url\n",
       "url_id                                                                     \n",
       "0       medpage      rss  https://www.medpagetoday.com/infectiousdisease...\n",
       "1       medpage      rss  https://www.medpagetoday.com/special-reports/e...\n",
       "2       medpage      rss  https://www.medpagetoday.com/infectiousdisease...\n",
       "3       medpage      rss  https://www.medpagetoday.com/pulmonology/gener...\n",
       "4       medpage      rss  https://www.medpagetoday.com/infectiousdisease...\n",
       "...         ...      ...                                                ...\n",
       "7120     popsci  twitter  https://www.popsci.com/story/diy/how-to-use-an...\n",
       "7121     popsci  twitter  https://www.popsci.com/how-to-make-perfect-pie...\n",
       "7122     popsci  twitter  https://www.popsci.com/story/technology/airbus...\n",
       "7123     popsci  twitter  https://www.popsci.com/story/gear/deer-birds-w...\n",
       "7124     popsci  twitter  https://www.popsci.com/story/reviews/best-knif...\n",
       "\n",
       "[7125 rows x 3 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls = rss_urls[[\"source\", \"channel\", \"url\"]]\n",
    "urls = urls.append(tweet_urls[[\"source\", \"channel\", \"url\"]])\n",
    "# urls[\"url_section\"] = None\n",
    "urls = urls.reset_index(drop=True)\n",
    "urls.index.name = \"url_id\"\n",
    "urls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and parse articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_published(article, source):\n",
    "    published = \"\"\n",
    "    if source in [\"guardian\", \"nyt\", \"newsmed\", \"healthday\", \"popsci\"]:\n",
    "        published = article.meta_data[\"article\"][\"published_time\"]\n",
    "    elif source in [\"medpage\"]:\n",
    "        published = article.meta_data[\"dc.date\"]\n",
    "    elif source == \"ifls\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        published = soup.find_all(\"span\", class_=\"date\")[0].string\n",
    "    elif source == \"wired\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        published = soup.find_all(\"time\", attrs={\"data-testid\":\"ContentHeaderPublishDate\"})[0].string\n",
    "    return pd.to_datetime(published).isoformat()\n",
    "\n",
    "def get_modified(article, source):\n",
    "    modified = \"\"\n",
    "    if source in [\"guardian\", \"nyt\", \"newsmed\", \"healthday\", \"popsci\"]:\n",
    "        modified = article.meta_data[\"article\"][\"modified_time\"]\n",
    "    return pd.to_datetime(modified).isoformat()\n",
    "\n",
    "def get_section(article, source):\n",
    "    section = \"\"\n",
    "    if source in [\"guardian\", \"nyt\", \"healthday\"]:\n",
    "        section = article.meta_data[\"article\"][\"section\"]\n",
    "    elif source == \"popsci\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        section = soup.find_all(\"ul\", class_=\"article-categories\")[0].find_all(\"a\")[0].string\n",
    "    elif source == \"wired\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        section = soup.find_all(\"a\", class_=\"rubric__link\")[0].span.string\n",
    "    elif source == \"ifls\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        section = soup.find_all(\"a\", class_=\"category\")[0].string\n",
    "    elif source == \"medpage\":\n",
    "        section = article.meta_data[\"sailthru.topcat\"]\n",
    "    elif source == \"newsmed\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        section = soup.find_all(class_=\"active-site-sections-menu-btn\")[0].string\n",
    "        section = section.split(\" Home\")[0]\n",
    "        \n",
    "    if len(section) == 0:\n",
    "        section = \"\"\n",
    "    return section.strip()\n",
    "\n",
    "def get_keywords(article, source):\n",
    "    keywords = \"\"\n",
    "    if source in [\"guardian\", \"nyt\", \"wired\", \"newsmed\", \"medpage\"]:\n",
    "        if \"news_keywords\" in article.meta_data:\n",
    "            keywords = article.meta_data[\"news_keywords\"]\n",
    "        elif \"keywords\" in article.meta_data:\n",
    "            keywords = article.meta_data[\"keywords\"]\n",
    "        elif \"tag\" in article.meta_data[\"article\"]:\n",
    "            keywords = article.meta_data[\"article\"][\"tag\"]\n",
    "    elif source == \"popsci\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        keywords = soup.find_all(\"div\", class_=\"tag-item\")\n",
    "        keywords = [tag.a.string.strip() for tag in keywords]\n",
    "        keywords = \", \".join(keywords)\n",
    "    elif source in [\"ifls\", \"healthday\"]:\n",
    "        article.nlp()\n",
    "        keywords = \", \".join(article.keywords)\n",
    "    if len(keywords) == 0:\n",
    "        keywords = \"\"\n",
    "    return keywords\n",
    "    \n",
    "def get_authors(article, source):\n",
    "    authors = \"\"\n",
    "    if source in [\"guardian\", \"nyt\", \"popsci\"]:\n",
    "        authors = \", \".join(article.authors)\n",
    "    elif source == \"wired\":\n",
    "        authors = article.meta_data[\"author\"]\n",
    "    elif source == \"ifls\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        authors = soup.find_all(\"div\", class_=\"author\")[0]\n",
    "        authors = authors.find_all(class_=\"name\")\n",
    "        if len(authors) > 0:\n",
    "            authors = authors[0].text.split(\"By \")[-1]\n",
    "        else:\n",
    "            authors = \"\"\n",
    "    elif source == \"newsmed\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        authors = soup.find_all(\"span\", attrs={\"itemprop\":\"name\"})\n",
    "        if len(authors) > 0:\n",
    "            authors = authors[0].string\n",
    "        else:\n",
    "            authors = \"\"\n",
    "    elif source == \"medpage\":\n",
    "        authors = article.meta_data[\"sailthru.author\"]\n",
    "    elif source == \"healthday\":\n",
    "        soup = BeautifulSoup(article.html, 'lxml')\n",
    "        authors = soup.find_all(\"a\", class_=\"post-author__name\")[0].string\n",
    "        \n",
    "    if len(authors) == 0:\n",
    "        authors = \"\"\n",
    "    return authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "domains = {\n",
    "    \"guardian\": \"theguardian\",\n",
    "    \"healthday\": \"healthday\",\n",
    "    \"ifls\": \"iflscience\",\n",
    "    \"medpage\": \"medpagetoday\",\n",
    "    \"newsmed\": \"news-med\",\n",
    "    \"nyt\": \"nytimes\",\n",
    "    \"popsci\": \"popsci\",\n",
    "    \"wired\": \"wired\"\n",
    "}\n",
    "\n",
    "def process_url(url, source):\n",
    "    # Init metadata\n",
    "    meta = {\n",
    "        \"published\": \"\",\n",
    "        \"modified\": \"\",\n",
    "        \"section\": \"\",\n",
    "        \"keywords\": \"\",\n",
    "        \"authors\": \"\",\n",
    "        \"title\": \"\",\n",
    "        \"canonical_url\": \"\",\n",
    "        \"success\": True,\n",
    "        \"error_msg\": \"\"\n",
    "    }\n",
    "        \n",
    "    # Attempt to parse content\n",
    "    try:\n",
    "        article = Article(url)\n",
    "        article.download()\n",
    "        article.parse()\n",
    "    except Exception as e:\n",
    "        meta[\"success\"] = False\n",
    "        meta[\"error_msg\"] = str(e)\n",
    "        return meta\n",
    "    \n",
    "    # use canonical link to determine if link is not from source\n",
    "    if domains[source] not in article.canonical_link:\n",
    "        meta[\"success\"] = False\n",
    "        meta[\"error_msg\"] = \"URL with wrong domain\"\n",
    "        return meta\n",
    "\n",
    "    meta[\"title\"] = article.title\n",
    "    meta[\"canonical_url\"] = article.canonical_link\n",
    "    \n",
    "    try:\n",
    "        meta[\"published\"] = get_published(article, source)\n",
    "    except:\n",
    "        meta[\"published\"] = \"parse_error\"\n",
    "    \n",
    "    try:\n",
    "        meta[\"modified\"] = get_modified(article, source)\n",
    "    except:\n",
    "        meta[\"modified\"] = \"parse_error\"\n",
    "    \n",
    "    try:\n",
    "        meta[\"section\"] = get_section(article, source)\n",
    "    except:\n",
    "        meta[\"section\"] = \"parse_error\"\n",
    "    \n",
    "    try:\n",
    "        meta[\"keywords\"] = get_keywords(article, source)\n",
    "    except:\n",
    "        meta[\"keywords\"] = \"parse_error\"\n",
    "    \n",
    "    try:\n",
    "        meta[\"authors\"] = get_authors(article, source)\n",
    "    except:\n",
    "        meta[\"authors\"] = \"parse_error\"\n",
    "    \n",
    "    return meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "outcols = [\n",
    "    \"published\",\n",
    "    \"modified\",\n",
    "    \"section\",\n",
    "    \"keywords\",\n",
    "    \"authors\",\n",
    "    \"title\",\n",
    "    \"canonical_url\",\n",
    "    \"success\",\n",
    "    \"msg\",\n",
    "]\n",
    "outcols = list(urls.columns) + outcols\n",
    "    \n",
    "outfile = output_dir / \"articles.csv\"\n",
    "if outfile.exists():\n",
    "    outputs = pd.read_csv(outfile, index_col=\"url_id\")\n",
    "else:\n",
    "    outputs = urls.copy()\n",
    "    outputs = outputs.reindex(columns=outcols, fill_value=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9aead1edfab4765b7e6b4789bed17b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/612 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = missing_urls = outputs[outputs.success != True]\n",
    "df = df.sample(frac=1)\n",
    "total = len(df)\n",
    "\n",
    "for ix, row in tqdm(df.iterrows(), total=total):\n",
    "    try:\n",
    "        meta = process_url(row.url, row.source)\n",
    "    except Exception as e:\n",
    "        print(row.url, row.source)\n",
    "        print(traceback.format_exc())\n",
    "        continue\n",
    "\n",
    "    for k, v in meta.items():\n",
    "        outputs.loc[ix, k] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>channel</th>\n",
       "      <th>url</th>\n",
       "      <th>published</th>\n",
       "      <th>modified</th>\n",
       "      <th>section</th>\n",
       "      <th>keywords</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "      <th>canonical_url</th>\n",
       "      <th>success</th>\n",
       "      <th>error_msg</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>url_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3744</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/space/breaking-mars...</td>\n",
       "      <td>2021-04-30T17:32:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Space</td>\n",
       "      <td>trip, perseverance, mars, week, survive, scien...</td>\n",
       "      <td>Dr Alfredo Carpineti</td>\n",
       "      <td>Breaking: Mars Helicopter Is Now A Fully Opera...</td>\n",
       "      <td>https://www.iflscience.com/space/breaking-mars...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3745</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/health-and-medicine...</td>\n",
       "      <td>2021-04-22T12:42:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Health and Medicine</td>\n",
       "      <td>entire, troop, speed, ski, finnish, good, russ...</td>\n",
       "      <td>James Felton</td>\n",
       "      <td>The Finnish Soldier Who Took His Entire Troop'...</td>\n",
       "      <td>https://www.iflscience.com/health-and-medicine...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3746</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/physics/the-better-...</td>\n",
       "      <td>2021-04-30T17:19:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Physics</td>\n",
       "      <td>clock, accuracy, clocks, vibration, relationsh...</td>\n",
       "      <td>Dr Alfredo Carpineti</td>\n",
       "      <td>The Better The Clock The More Entropy It Might...</td>\n",
       "      <td>https://www.iflscience.com/physics/the-better-...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3747</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/editors-blog/this-i...</td>\n",
       "      <td>2021-04-30T17:04:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Editor's Blog</td>\n",
       "      <td>fact, perforated, way, paper, hang, toilet, ac...</td>\n",
       "      <td>Dr Alfredo Carpineti</td>\n",
       "      <td>This Is How You Should Hang Your Toilet Paper ...</td>\n",
       "      <td>https://www.iflscience.com/editors-blog/this-i...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3748</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/health-and-medicine...</td>\n",
       "      <td>2021-04-30T17:06:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Health and Medicine</td>\n",
       "      <td>harmful, fermented, decompose, months, left, r...</td>\n",
       "      <td>Jack Dunhill</td>\n",
       "      <td>Some People Are Eating Raw, Rotten \"High Meat\"...</td>\n",
       "      <td>https://www.iflscience.com/health-and-medicine...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4326</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/chemistry/seafloor-...</td>\n",
       "      <td>2021-02-26T16:40:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Chemistry</td>\n",
       "      <td>products, sediment, hydrogen, microbes, sauvag...</td>\n",
       "      <td>Stephen Luntz</td>\n",
       "      <td>Seafloor Microbes Are Living On The Products O...</td>\n",
       "      <td>https://www.iflscience.com/chemistry/seafloor-...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4327</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/editors-blog/cats-d...</td>\n",
       "      <td>2021-03-01T13:13:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Editor's Blog</td>\n",
       "      <td>pets, site, monkeys, pet, study, unearthed, ca...</td>\n",
       "      <td>Benjamin Taub</td>\n",
       "      <td>Cats, Dogs, And Monkeys Unearthed At Ancient E...</td>\n",
       "      <td>https://www.iflscience.com/editors-blog/cats-d...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4328</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/plants-and-animals/...</td>\n",
       "      <td>2021-03-01T14:43:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Plants and Animals</td>\n",
       "      <td>cleverly, forest, survive, embrace, freezing, ...</td>\n",
       "      <td>Katy Evans</td>\n",
       "      <td>Alligators In Oklahoma Cleverly Embrace Becomi...</td>\n",
       "      <td>https://www.iflscience.com/plants-and-animals/...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4329</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/physics/scientists-...</td>\n",
       "      <td>2021-02-26T17:27:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Physics</td>\n",
       "      <td>space, technologies, crystal, able, structure,...</td>\n",
       "      <td>Dr Alfredo Carpineti</td>\n",
       "      <td>Watch The First Ever Video Of A Space-Time Cry...</td>\n",
       "      <td>https://www.iflscience.com/physics/scientists-...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4330</th>\n",
       "      <td>ifls</td>\n",
       "      <td>twitter</td>\n",
       "      <td>https://www.iflscience.com/plants-and-animals/...</td>\n",
       "      <td>2019-08-20T18:13:00</td>\n",
       "      <td>NaT</td>\n",
       "      <td>Plants and Animals</td>\n",
       "      <td>far, thousand, lake, single, near, sampled, re...</td>\n",
       "      <td>Stephen Luntz</td>\n",
       "      <td>For A Thousand Years People Came From Far Away...</td>\n",
       "      <td>https://www.iflscience.com/plants-and-animals/...</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>587 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       source  channel                                                url  \\\n",
       "url_id                                                                      \n",
       "3744     ifls  twitter  https://www.iflscience.com/space/breaking-mars...   \n",
       "3745     ifls  twitter  https://www.iflscience.com/health-and-medicine...   \n",
       "3746     ifls  twitter  https://www.iflscience.com/physics/the-better-...   \n",
       "3747     ifls  twitter  https://www.iflscience.com/editors-blog/this-i...   \n",
       "3748     ifls  twitter  https://www.iflscience.com/health-and-medicine...   \n",
       "...       ...      ...                                                ...   \n",
       "4326     ifls  twitter  https://www.iflscience.com/chemistry/seafloor-...   \n",
       "4327     ifls  twitter  https://www.iflscience.com/editors-blog/cats-d...   \n",
       "4328     ifls  twitter  https://www.iflscience.com/plants-and-animals/...   \n",
       "4329     ifls  twitter  https://www.iflscience.com/physics/scientists-...   \n",
       "4330     ifls  twitter  https://www.iflscience.com/plants-and-animals/...   \n",
       "\n",
       "                  published modified              section  \\\n",
       "url_id                                                      \n",
       "3744    2021-04-30T17:32:00      NaT                Space   \n",
       "3745    2021-04-22T12:42:00      NaT  Health and Medicine   \n",
       "3746    2021-04-30T17:19:00      NaT              Physics   \n",
       "3747    2021-04-30T17:04:00      NaT        Editor's Blog   \n",
       "3748    2021-04-30T17:06:00      NaT  Health and Medicine   \n",
       "...                     ...      ...                  ...   \n",
       "4326    2021-02-26T16:40:00      NaT            Chemistry   \n",
       "4327    2021-03-01T13:13:00      NaT        Editor's Blog   \n",
       "4328    2021-03-01T14:43:00      NaT   Plants and Animals   \n",
       "4329    2021-02-26T17:27:00      NaT              Physics   \n",
       "4330    2019-08-20T18:13:00      NaT   Plants and Animals   \n",
       "\n",
       "                                                 keywords  \\\n",
       "url_id                                                      \n",
       "3744    trip, perseverance, mars, week, survive, scien...   \n",
       "3745    entire, troop, speed, ski, finnish, good, russ...   \n",
       "3746    clock, accuracy, clocks, vibration, relationsh...   \n",
       "3747    fact, perforated, way, paper, hang, toilet, ac...   \n",
       "3748    harmful, fermented, decompose, months, left, r...   \n",
       "...                                                   ...   \n",
       "4326    products, sediment, hydrogen, microbes, sauvag...   \n",
       "4327    pets, site, monkeys, pet, study, unearthed, ca...   \n",
       "4328    cleverly, forest, survive, embrace, freezing, ...   \n",
       "4329    space, technologies, crystal, able, structure,...   \n",
       "4330    far, thousand, lake, single, near, sampled, re...   \n",
       "\n",
       "                     authors  \\\n",
       "url_id                         \n",
       "3744    Dr Alfredo Carpineti   \n",
       "3745            James Felton   \n",
       "3746    Dr Alfredo Carpineti   \n",
       "3747    Dr Alfredo Carpineti   \n",
       "3748            Jack Dunhill   \n",
       "...                      ...   \n",
       "4326           Stephen Luntz   \n",
       "4327           Benjamin Taub   \n",
       "4328              Katy Evans   \n",
       "4329    Dr Alfredo Carpineti   \n",
       "4330           Stephen Luntz   \n",
       "\n",
       "                                                    title  \\\n",
       "url_id                                                      \n",
       "3744    Breaking: Mars Helicopter Is Now A Fully Opera...   \n",
       "3745    The Finnish Soldier Who Took His Entire Troop'...   \n",
       "3746    The Better The Clock The More Entropy It Might...   \n",
       "3747    This Is How You Should Hang Your Toilet Paper ...   \n",
       "3748    Some People Are Eating Raw, Rotten \"High Meat\"...   \n",
       "...                                                   ...   \n",
       "4326    Seafloor Microbes Are Living On The Products O...   \n",
       "4327    Cats, Dogs, And Monkeys Unearthed At Ancient E...   \n",
       "4328    Alligators In Oklahoma Cleverly Embrace Becomi...   \n",
       "4329    Watch The First Ever Video Of A Space-Time Cry...   \n",
       "4330    For A Thousand Years People Came From Far Away...   \n",
       "\n",
       "                                            canonical_url  success error_msg  \n",
       "url_id                                                                        \n",
       "3744    https://www.iflscience.com/space/breaking-mars...     True            \n",
       "3745    https://www.iflscience.com/health-and-medicine...     True            \n",
       "3746    https://www.iflscience.com/physics/the-better-...     True            \n",
       "3747    https://www.iflscience.com/editors-blog/this-i...     True            \n",
       "3748    https://www.iflscience.com/health-and-medicine...     True            \n",
       "...                                                   ...      ...       ...  \n",
       "4326    https://www.iflscience.com/chemistry/seafloor-...     True            \n",
       "4327    https://www.iflscience.com/editors-blog/cats-d...     True            \n",
       "4328    https://www.iflscience.com/plants-and-animals/...     True            \n",
       "4329    https://www.iflscience.com/physics/scientists-...     True            \n",
       "4330    https://www.iflscience.com/plants-and-animals/...     True            \n",
       "\n",
       "[587 rows x 12 columns]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs[(outputs.source==\"ifls\")&(outputs.published.notna())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = outputs.replace(\"\", None)\n",
    "outputs.to_csv(outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample = outputs[outputs.success].groupby(\"source\", as_index=False).apply(lambda x: x.sample(10)).reset_index(drop=True)\n",
    "# sample.index.name = \"id\"\n",
    "# sample.to_csv(sample_dir / \"all_venues_sample_05052021.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "altmetric-news",
   "language": "python",
   "name": "altmetric-news"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}