image_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Scrape the web to create test dataset\n",
    "# this notebook requires a bing key from https://www.microsoft.com/en-us/bing/apis/bing-image-search-api\n",
    "\n",
    "# !pip install -Uqq fastbook    # first run only\n",
    "import fastbook\n",
    "fastbook.setup_book()\n",
    "\n",
    "from fastbook import *\n",
    "from fastai.core import *\n",
    "from fastai.vision import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keys\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adapted from bing custom search documentation:\n",
    "# https://docs.microsoft.com/en-us/azure/cognitive-services/bing-image-search/quickstarts/python\n",
    "def bing_custom_search(key:str, query, color='ColorOnly', minWidth=170, maxWidth=230, minHeight=170, maxHeight=230, count=150, aspect='All', license = 'Public', imageType='Photo'):\n",
    "    params = {'q':query, 'count':count, 'color':color, 'minWidth':minWidth, 'maxWidth':maxWidth, 'minHeight':minHeight, 'maxHeight':maxHeight, 'aspect':aspect, 'license':license, 'imageType':imageType}\n",
    "    headers = {\"Ocp-Apim-Subscription-Key\":key}\n",
    "    search_url = \"https://api.bing.microsoft.com/v7.0/images/search\"\n",
    "    response = requests.get(search_url, headers=headers, params=params)\n",
    "    response.raise_for_status()\n",
    "    search_results = response.json()   \n",
    "    return L(search_results['value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "queries = ['old photography', 'legacy photo', 'old europe', 'vintage usa', 'old asia', 'old world photography', 'historical photography', 'old australia photography', 'historic australia photography', 'australia photography', 'australian war photography', 'australian land']\n",
    "\n",
    "color = 'Monochrome'\n",
    "count = 150\n",
    "aspect = 'Square'\n",
    "license = 'Public'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "minWidth = 170\n",
    "maxWidth = 230\n",
    "minHeight = 170\n",
    "maxHeight = 230\n",
    "\n",
    "res_SR = []\n",
    "for q in queries:\n",
    "    time.sleep(1) # azure free tier only allows 1 query per second\n",
    "    results = bing_custom_search(keys.bing_key, q, color, minWidth, maxWidth, minHeight, maxHeight, count, aspect, license)\n",
    "    urls = results .attrgot('contentUrl')\n",
    "    #for u in urls:\n",
    "    res_SR = L(res_SR, urls).concat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "minWidth = 400\n",
    "maxWidth = 3000\n",
    "minHeight = 400\n",
    "maxHeight = 3000\n",
    "\n",
    "queries_damaged = ['old portrait damaged', 'old torn photo', 'old damaged photo', 'damaged photography', 'scratched photos', 'scratched photograph', 'extremely damaged photo']\n",
    "res_damaged = []\n",
    "for q in queries_damaged:\n",
    "    time.sleep(1) # azure free tier only allows 1 query per second\n",
    "    results = bing_custom_search(keys.bing_key, q, color, minWidth, maxWidth, minHeight, maxHeight, count, aspect, license)\n",
    "    urls = results.attrgot('contentUrl')\n",
    "    res_damaged = L(res_damaged, urls).concat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#1118) ['https://upload.wikimedia.org/wikipedia/commons/thumb/6/66/Isakov_House_old_photo.jpg/200px-Isakov_House_old_photo.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Ann-Margret_Publicity.jpg/170px-Ann-Margret_Publicity.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/4/4f/Milwaukee_City_Hall_Old_Public_Domain_Photo.jpg/204px-Milwaukee_City_Hall_Old_Public_Domain_Photo.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/6/66/Isakov_House_old_photo.jpg/220px-Isakov_House_old_photo.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/6/6a/Alexis_Smith_Photograph_of_Hollywood_celebrities_and_dignitaries_at_Roosevelt_Birthday_Ball_activities_in_Washington%2C_%28left_to..._-_NARA_-_199317.jpg/220px-Alexis_Smith_Photograph_of_Hollywood_celebrities_and_dignitaries_at_Roosevelt_Birthday_Ball_activities_in_Washington%2C_%28left_to..._-_NARA_-_199317.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/7/74/An_Old_Rear-Tenement_in_Roosevelt_Street.png/220px-An_Old_Rear-Tenement_in_Roosevelt_Street.png','https://upload.wikimedia.org/wikipedia/commons/thumb/1/1e/Hofkonditor_Zieglers_G%C3%A5rd_vintage_photo.jpg/180px-Hofkonditor_Zieglers_G%C3%A5rd_vintage_photo.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/4/40/Route_18_east.jpg/220px-Route_18_east.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/b/ba/Cecil_Beaton_Photographs-_Political_and_Military_Personalities%3B_Chahnaz%2C_Princess_of_Iran%2C_Fawzieh%2C_Queen_of_Iran%2C_Mohammed_Reza_Pahlevi_CBM2404.jpg/220px-Cecil_Beaton_Photographs-_Political_and_Military_Personalities%3B_Chahnaz%2C_Princess_of_Iran%2C_Fawzieh%2C_Queen_of_Iran%2C_Mohammed_Reza_Pahlevi_CBM2404.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/2/24/Medicine_Man.png/180px-Medicine_Man.png'...]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_SR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#969) ['http://xroads.virginia.edu/~MA03/pricola/FSA/images/LOC%20pics/Lange--damaged%20child.jpg','https://upload.wikimedia.org/wikipedia/commons/5/54/Damaged_building%2C_ruins_Fortepan_3047.jpg','https://globalvoices.org/wp-content/uploads/2016/11/IMG_1-10.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Damaged_building%2C_ruins_Fortepan_3047.jpg/1200px-Damaged_building%2C_ruins_Fortepan_3047.jpg','https://upload.wikimedia.org/wikipedia/commons/d/d7/Russia%2C_Stary_Oskol_Fortepan_12207.jpg','http://2.bp.blogspot.com/-70RfAcKZLdw/U5m6l9wbZSI/AAAAAAAAChY/pNoVhVkpKk0/s1600/Ultimate+Collection+Of+Rare+Historical+Photos.+A+Big+Piece+Of+History+(200+Pictures)+-+B-36.jpg','https://farm8.staticflickr.com/7030/6834680647_efdcbb441c_z.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Damaged_building%2C_ruins_Fortepan_3047.jpg/574px-Damaged_building%2C_ruins_Fortepan_3047.jpg','https://upload.wikimedia.org/wikipedia/commons/1/1b/War_Comes_To_School-_Life_at_Peckham_Central_School%2C_London%2C_England%2C_1943_D12220.jpg','https://farm9.staticflickr.com/8187/8121233385_57321d7df6_z.jpg'...]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_damaged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "minWidth = 700\n",
    "maxWidth = 1300\n",
    "minHeight = 700\n",
    "maxHeight = 1300\n",
    "\n",
    "res_large = L() #fastai list object\n",
    "for q in queries:\n",
    "    time.sleep(1) # azure free tier only allows 1 query per second\n",
    "    results = bing_custom_search(keys.bing_key, q, color, size, count, aspect, license)\n",
    "    urls = results.attrgot('contentUrl')\n",
    "    res_large = L(res_large, urls).concat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "minWidth = 300\n",
    "maxWidth = 600\n",
    "minHeight = 300\n",
    "maxHeight = 600\n",
    "\n",
    "res_med = L() #fastai list object\n",
    "for q in queries:\n",
    "    results = bing_custom_search(keys.bing_key, q, color, size, count, aspect, license)\n",
    "    urls = results.attrgot('contentUrl')\n",
    "    res_med = L(res_med, urls).concat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#743) ['https://upload.wikimedia.org/wikipedia/commons/8/8f/PSM_V52_D027_Old_tom_australian_aboriginal.jpg','https://c2.staticflickr.com/4/3577/3527609960_d34d4a71f3.jpg','https://upload.wikimedia.org/wikipedia/commons/8/89/Elizabeth_street_cnr_Hunter.jpg','https://upload.wikimedia.org/wikipedia/commons/2/20/Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/2/20/Seizure_of_blackbirder_Daphne.jpg/1200px-Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/c/ce/Light_horse_walers.jpg','https://upload.wikimedia.org/wikipedia/commons/b/b3/Broadway_Sydney_1910.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/WRPywellByGardner.jpg/1200px-WRPywellByGardner.jpg','http://upload.wikimedia.org/wikipedia/commons/d/d4/Australian_pilots_of_No._452_Squadron_relax_outside_their_dispersal_hut_at_Kirton-in-Lindsey%2C_18_June_1941._CH2883.jpg','https://c1.staticflickr.com/9/8231/8554082058_11af66cc54.jpg'...]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#743) ['https://upload.wikimedia.org/wikipedia/commons/8/8f/PSM_V52_D027_Old_tom_australian_aboriginal.jpg','https://c2.staticflickr.com/4/3577/3527609960_d34d4a71f3.jpg','https://upload.wikimedia.org/wikipedia/commons/8/89/Elizabeth_street_cnr_Hunter.jpg','https://upload.wikimedia.org/wikipedia/commons/2/20/Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/2/20/Seizure_of_blackbirder_Daphne.jpg/1200px-Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/c/ce/Light_horse_walers.jpg','https://upload.wikimedia.org/wikipedia/commons/b/b3/Broadway_Sydney_1910.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/WRPywellByGardner.jpg/1200px-WRPywellByGardner.jpg','http://upload.wikimedia.org/wikipedia/commons/d/d4/Australian_pilots_of_No._452_Squadron_relax_outside_their_dispersal_hut_at_Kirton-in-Lindsey%2C_18_June_1941._CH2883.jpg','https://c1.staticflickr.com/9/8231/8554082058_11af66cc54.jpg'...]"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_med"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "964 797\n"
     ]
    }
   ],
   "source": [
    "# delete duplicate urls\n",
    "res_SR_unique = list(dict.fromkeys(res_SR))\n",
    "res_damaged_unique = list(dict.fromkeys(res_damaged))\n",
    "res_large_unique = res_large.unique()\n",
    "res_med_unique = res_med.unique()\n",
    "print(len(res_SR_unique), len(res_damaged_unique))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#603) ['https://upload.wikimedia.org/wikipedia/commons/8/8f/PSM_V52_D027_Old_tom_australian_aboriginal.jpg','https://c2.staticflickr.com/4/3577/3527609960_d34d4a71f3.jpg','https://upload.wikimedia.org/wikipedia/commons/8/89/Elizabeth_street_cnr_Hunter.jpg','https://upload.wikimedia.org/wikipedia/commons/2/20/Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/2/20/Seizure_of_blackbirder_Daphne.jpg/1200px-Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/c/ce/Light_horse_walers.jpg','https://upload.wikimedia.org/wikipedia/commons/b/b3/Broadway_Sydney_1910.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/WRPywellByGardner.jpg/1200px-WRPywellByGardner.jpg','http://upload.wikimedia.org/wikipedia/commons/d/d4/Australian_pilots_of_No._452_Squadron_relax_outside_their_dispersal_hut_at_Kirton-in-Lindsey%2C_18_June_1941._CH2883.jpg','https://c1.staticflickr.com/9/8231/8554082058_11af66cc54.jpg'...]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_large_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#607) ['https://upload.wikimedia.org/wikipedia/commons/8/8f/PSM_V52_D027_Old_tom_australian_aboriginal.jpg','https://c2.staticflickr.com/4/3577/3527609960_d34d4a71f3.jpg','https://upload.wikimedia.org/wikipedia/commons/8/89/Elizabeth_street_cnr_Hunter.jpg','https://upload.wikimedia.org/wikipedia/commons/2/20/Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/2/20/Seizure_of_blackbirder_Daphne.jpg/1200px-Seizure_of_blackbirder_Daphne.jpg','https://upload.wikimedia.org/wikipedia/commons/c/ce/Light_horse_walers.jpg','https://upload.wikimedia.org/wikipedia/commons/b/b3/Broadway_Sydney_1910.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/WRPywellByGardner.jpg/1200px-WRPywellByGardner.jpg','http://upload.wikimedia.org/wikipedia/commons/d/d4/Australian_pilots_of_No._452_Squadron_relax_outside_their_dispersal_hut_at_Kirton-in-Lindsey%2C_18_June_1941._CH2883.jpg','https://c1.staticflickr.com/9/8231/8554082058_11af66cc54.jpg'...]"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_med_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "dest_large = '/mnt/Data/datasets/legacy_large/'\n",
    "dest_med = '/mnt/Data/datasets/legacy_med/'\n",
    "dest_SR = '/mnt/Data/datasets/legacy_SR/'\n",
    "dest_damaged = '/mnt/Data/datasets/legacy_damaged/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download from the collected URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "download_images(dest_med, urls=res_med_unique)\n",
    "download_images(dest_large, urls=res_large_unique)\n",
    "download_images(dest_damaged, urls=res_damaged_unique)\n",
    "download_images(dest_SR, urls=res_SR_unique)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}