Macrostrat Pull Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Connect to Macrostrat API and pull multiple data sources\n",
    "\n",
    "import csv, codecs\n",
    "from urllib.request import urlopen\n",
    "from pprint import pprint\n",
    "\n",
    "def download_csv(url):\n",
    "    '''INPUT: string url address\n",
    "       RETRUNS: dictionary with headers'''\n",
    "    \n",
    "    # return variable\n",
    "    data_dict = {}\n",
    "    \n",
    "    # get url from user\n",
    "    #url = input('Gimme a url to process:')\n",
    "    \n",
    "    data = urlopen(url)\n",
    "    csv_file = csv.reader(codecs.iterdecode(data, 'utf-8'))\n",
    "    \n",
    "    # create list of tuples with list(zip)\n",
    "    # unpack csv as list of tuples\n",
    "    # length of VARIABLE == number of fields\n",
    "    # length of VARIABLE[i] == number of rows\n",
    "    # VARIABLE[i][0] == header name\n",
    "    columns = list(zip(*csv_file))\n",
    "    \n",
    "    # set key names to field names (headers in the csv)\n",
    "    for field in columns:\n",
    "        data_dict[field[0]] = field[1:]\n",
    "    \n",
    "    data_dict['headers'] = sorted(data_dict.keys())\n",
    "    \n",
    "    return data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done with strat\n",
      "done with intervals\n",
      "done with environments\n"
     ]
    }
   ],
   "source": [
    "strat_dict = download_csv('https://macrostrat.org/api/defs/strat_names?all&format=csv')\n",
    "print('done with strat')\n",
    "interval_dict = download_csv('https://macrostrat.org/api/defs/intervals?all&format=csv')\n",
    "print('done with intervals')\n",
    "environment_dict = download_csv('https://macrostrat.org/api/defs/environments?all&format=csv')\n",
    "print('done with environments')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STRAT DICT:  ['b_age', 'b_period', 'bed', 'bed_id', 'c_interval', 'concept_id', 'fm', 'fm_id', 'gp', 'gp_id', 'mbr', 'mbr_id', 'rank', 'ref_id', 'sgp', 'sgp_id', 'strat_name', 'strat_name_id', 'strat_name_long', 'subgp', 'subgp_id', 't_age', 't_period', 't_units']\n",
      "INTERVAL DICT:  ['abbrev', 'b_age', 'color', 'int_id', 'int_type', 'name', 't_age', 'timescales']\n",
      "ENVIRON DICT:  ['class', 'color', 'environ_id', 'name', 't_units', 'type']\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "print('STRAT DICT: ', strat_dict['headers'])\n",
    "print('INTERVAL DICT: ', interval_dict['headers'])\n",
    "print('ENVIRON DICT: ', environment_dict['headers'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b_age              (0.0029, 323.2, 323.2, 323.2, 323.2, 323.2, 32...\n",
       "b_period           (Quaternary, Carboniferous, Carboniferous, Car...\n",
       "bed                (1980 St Helens Ash, 22 Yard Coal, 27 Yard Coa...\n",
       "bed_id             (6808, 92986, 92987, 92988, 92989, 92990, 9299...\n",
       "c_interval         (Holocene, Bashkirian, Pennsylvanian, Bashkiri...\n",
       "concept_id         (0, 36374, 36819, 32110, 36465, 30792, 30871, ...\n",
       "fm                 (, Pennine Lower Coal Measures, Pennine Middle...\n",
       "fm_id              (0, 98660, 98661, 98660, 98660, 98660, 98660, ...\n",
       "gp                 (, Pennine Coal Measures, Pennine Coal Measure...\n",
       "gp_id              (0, 98659, 98659, 98659, 98659, 98659, 98659, ...\n",
       "mbr                (, , , , , , , , A'Mharagach, , , , Aagot, Aar...\n",
       "mbr_id             (0, 0, 0, 0, 0, 0, 0, 0, 92993, 0, 0, 0, 90930...\n",
       "rank               (Bed, Bed, Bed, Bed, Bed, Bed, Bed, Bed, Mbr, ...\n",
       "ref_id             (1, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 36...\n",
       "sgp                (, , , , , , , , New Red Sandstone, Moine, , ,...\n",
       "sgp_id             (0, 0, 0, 0, 0, 0, 0, 0, 98288, 98044, 0, 0, 0...\n",
       "strat_name         (1980 St Helens Ash, 22 Yard Coal, 27 Yard Coa...\n",
       "strat_name_id      (6808, 92986, 92987, 92988, 92989, 92990, 9299...\n",
       "strat_name_long    (1980 St Helens Ash Bed, 22 Yard Coal Bed, 27 ...\n",
       "subgp              (, , , , , , , , , , , , , , , , , , , , Busta...\n",
       "subgp_id           (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "t_age              (0, 315.2, 307, 315.2, 315.2, 315.2, 315.2, 31...\n",
       "t_period           (Quaternary, Carboniferous, Carboniferous, Car...\n",
       "t_units            (1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# as a pandsa series\n",
    "my_panda_series = pd.Series(data=strat_dict, index=strat_dict['headers'])\n",
    "my_panda_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>b_age</th>\n",
       "      <td>(0.0029, 323.2, 323.2, 323.2, 323.2, 323.2, 32...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b_period</th>\n",
       "      <td>(Quaternary, Carboniferous, Carboniferous, Car...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed</th>\n",
       "      <td>(1980 St Helens Ash, 22 Yard Coal, 27 Yard Coa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_id</th>\n",
       "      <td>(6808, 92986, 92987, 92988, 92989, 92990, 9299...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c_interval</th>\n",
       "      <td>(Holocene, Bashkirian, Pennsylvanian, Bashkiri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>concept_id</th>\n",
       "      <td>(0, 36374, 36819, 32110, 36465, 30792, 30871, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fm</th>\n",
       "      <td>(, Pennine Lower Coal Measures, Pennine Middle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fm_id</th>\n",
       "      <td>(0, 98660, 98661, 98660, 98660, 98660, 98660, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gp</th>\n",
       "      <td>(, Pennine Coal Measures, Pennine Coal Measure...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gp_id</th>\n",
       "      <td>(0, 98659, 98659, 98659, 98659, 98659, 98659, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mbr</th>\n",
       "      <td>(, , , , , , , , A'Mharagach, , , , Aagot, Aar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mbr_id</th>\n",
       "      <td>(0, 0, 0, 0, 0, 0, 0, 0, 92993, 0, 0, 0, 90930...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank</th>\n",
       "      <td>(Bed, Bed, Bed, Bed, Bed, Bed, Bed, Bed, Mbr, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ref_id</th>\n",
       "      <td>(1, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 36...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sgp</th>\n",
       "      <td>(, , , , , , , , New Red Sandstone, Moine, , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sgp_id</th>\n",
       "      <td>(0, 0, 0, 0, 0, 0, 0, 0, 98288, 98044, 0, 0, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strat_name</th>\n",
       "      <td>(1980 St Helens Ash, 22 Yard Coal, 27 Yard Coa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strat_name_id</th>\n",
       "      <td>(6808, 92986, 92987, 92988, 92989, 92990, 9299...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strat_name_long</th>\n",
       "      <td>(1980 St Helens Ash Bed, 22 Yard Coal Bed, 27 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subgp</th>\n",
       "      <td>(, , , , , , , , , , , , , , , , , , , , Busta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subgp_id</th>\n",
       "      <td>(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_age</th>\n",
       "      <td>(0, 315.2, 307, 315.2, 315.2, 315.2, 315.2, 31...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_period</th>\n",
       "      <td>(Quaternary, Carboniferous, Carboniferous, Car...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_units</th>\n",
       "      <td>(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                 0\n",
       "b_age            (0.0029, 323.2, 323.2, 323.2, 323.2, 323.2, 32...\n",
       "b_period         (Quaternary, Carboniferous, Carboniferous, Car...\n",
       "bed              (1980 St Helens Ash, 22 Yard Coal, 27 Yard Coa...\n",
       "bed_id           (6808, 92986, 92987, 92988, 92989, 92990, 9299...\n",
       "c_interval       (Holocene, Bashkirian, Pennsylvanian, Bashkiri...\n",
       "concept_id       (0, 36374, 36819, 32110, 36465, 30792, 30871, ...\n",
       "fm               (, Pennine Lower Coal Measures, Pennine Middle...\n",
       "fm_id            (0, 98660, 98661, 98660, 98660, 98660, 98660, ...\n",
       "gp               (, Pennine Coal Measures, Pennine Coal Measure...\n",
       "gp_id            (0, 98659, 98659, 98659, 98659, 98659, 98659, ...\n",
       "mbr              (, , , , , , , , A'Mharagach, , , , Aagot, Aar...\n",
       "mbr_id           (0, 0, 0, 0, 0, 0, 0, 0, 92993, 0, 0, 0, 90930...\n",
       "rank             (Bed, Bed, Bed, Bed, Bed, Bed, Bed, Bed, Mbr, ...\n",
       "ref_id           (1, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 36...\n",
       "sgp              (, , , , , , , , New Red Sandstone, Moine, , ,...\n",
       "sgp_id           (0, 0, 0, 0, 0, 0, 0, 0, 98288, 98044, 0, 0, 0...\n",
       "strat_name       (1980 St Helens Ash, 22 Yard Coal, 27 Yard Coa...\n",
       "strat_name_id    (6808, 92986, 92987, 92988, 92989, 92990, 9299...\n",
       "strat_name_long  (1980 St Helens Ash Bed, 22 Yard Coal Bed, 27 ...\n",
       "subgp            (, , , , , , , , , , , , , , , , , , , , Busta...\n",
       "subgp_id         (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "t_age            (0, 315.2, 307, 315.2, 315.2, 315.2, 315.2, 31...\n",
       "t_period         (Quaternary, Carboniferous, Carboniferous, Car...\n",
       "t_units          (1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_panda_dataframe = pd.DataFrame(data=my_panda_series)\n",
    "my_panda_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}