worms_ents.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import all necessary libraries\n",
    "import glob\n",
    "import pandas as pd\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from spacy_lookup import Entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in the taxon data as a dataframe\n",
    "taxon = pd.read_csv('WoRMS/taxon.txt', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of                                            taxonID  \\\n",
       "0             urn:lsid:marinespecies.org:taxname:1   \n",
       "1             urn:lsid:marinespecies.org:taxname:2   \n",
       "2             urn:lsid:marinespecies.org:taxname:3   \n",
       "3             urn:lsid:marinespecies.org:taxname:4   \n",
       "4             urn:lsid:marinespecies.org:taxname:5   \n",
       "5             urn:lsid:marinespecies.org:taxname:6   \n",
       "6             urn:lsid:marinespecies.org:taxname:7   \n",
       "7             urn:lsid:marinespecies.org:taxname:8   \n",
       "8             urn:lsid:marinespecies.org:taxname:9   \n",
       "9            urn:lsid:marinespecies.org:taxname:10   \n",
       "10           urn:lsid:marinespecies.org:taxname:11   \n",
       "11           urn:lsid:marinespecies.org:taxname:12   \n",
       "12           urn:lsid:marinespecies.org:taxname:13   \n",
       "13           urn:lsid:marinespecies.org:taxname:24   \n",
       "14           urn:lsid:marinespecies.org:taxname:25   \n",
       "15           urn:lsid:marinespecies.org:taxname:26   \n",
       "16           urn:lsid:marinespecies.org:taxname:51   \n",
       "17           urn:lsid:marinespecies.org:taxname:55   \n",
       "18           urn:lsid:marinespecies.org:taxname:57   \n",
       "19           urn:lsid:marinespecies.org:taxname:58   \n",
       "20           urn:lsid:marinespecies.org:taxname:59   \n",
       "21           urn:lsid:marinespecies.org:taxname:63   \n",
       "22           urn:lsid:marinespecies.org:taxname:64   \n",
       "23           urn:lsid:marinespecies.org:taxname:69   \n",
       "24           urn:lsid:marinespecies.org:taxname:90   \n",
       "25           urn:lsid:marinespecies.org:taxname:91   \n",
       "26          urn:lsid:marinespecies.org:taxname:101   \n",
       "27          urn:lsid:marinespecies.org:taxname:102   \n",
       "28          urn:lsid:marinespecies.org:taxname:103   \n",
       "29          urn:lsid:marinespecies.org:taxname:104   \n",
       "...                                            ...   \n",
       "597884  urn:lsid:marinespecies.org:taxname:1338683   \n",
       "597885  urn:lsid:marinespecies.org:taxname:1338684   \n",
       "597886  urn:lsid:marinespecies.org:taxname:1338685   \n",
       "597887  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597888  urn:lsid:marinespecies.org:taxname:1338693   \n",
       "597889  urn:lsid:marinespecies.org:taxname:1338694   \n",
       "597890  urn:lsid:marinespecies.org:taxname:1338695   \n",
       "597891  urn:lsid:marinespecies.org:taxname:1338696   \n",
       "597892  urn:lsid:marinespecies.org:taxname:1338697   \n",
       "597893  urn:lsid:marinespecies.org:taxname:1338698   \n",
       "597894  urn:lsid:marinespecies.org:taxname:1338699   \n",
       "597895  urn:lsid:marinespecies.org:taxname:1338700   \n",
       "597896  urn:lsid:marinespecies.org:taxname:1338701   \n",
       "597897  urn:lsid:marinespecies.org:taxname:1338702   \n",
       "597898  urn:lsid:marinespecies.org:taxname:1338703   \n",
       "597899  urn:lsid:marinespecies.org:taxname:1338704   \n",
       "597900  urn:lsid:marinespecies.org:taxname:1338705   \n",
       "597901  urn:lsid:marinespecies.org:taxname:1338723   \n",
       "597902  urn:lsid:marinespecies.org:taxname:1338724   \n",
       "597903  urn:lsid:marinespecies.org:taxname:1338725   \n",
       "597904  urn:lsid:marinespecies.org:taxname:1338726   \n",
       "597905  urn:lsid:marinespecies.org:taxname:1338727   \n",
       "597906  urn:lsid:marinespecies.org:taxname:1338728   \n",
       "597907  urn:lsid:marinespecies.org:taxname:1338729   \n",
       "597908  urn:lsid:marinespecies.org:taxname:1338730   \n",
       "597909  urn:lsid:marinespecies.org:taxname:1338731   \n",
       "597910  urn:lsid:marinespecies.org:taxname:1338732   \n",
       "597911  urn:lsid:marinespecies.org:taxname:1338734   \n",
       "597912  urn:lsid:marinespecies.org:taxname:1338745   \n",
       "597913  urn:lsid:marinespecies.org:taxname:1338746   \n",
       "\n",
       "                                  scientificNameID  \\\n",
       "0             urn:lsid:marinespecies.org:taxname:1   \n",
       "1             urn:lsid:marinespecies.org:taxname:2   \n",
       "2             urn:lsid:marinespecies.org:taxname:3   \n",
       "3             urn:lsid:marinespecies.org:taxname:4   \n",
       "4             urn:lsid:marinespecies.org:taxname:5   \n",
       "5             urn:lsid:marinespecies.org:taxname:6   \n",
       "6             urn:lsid:marinespecies.org:taxname:7   \n",
       "7             urn:lsid:marinespecies.org:taxname:8   \n",
       "8             urn:lsid:marinespecies.org:taxname:9   \n",
       "9            urn:lsid:marinespecies.org:taxname:10   \n",
       "10           urn:lsid:marinespecies.org:taxname:11   \n",
       "11           urn:lsid:marinespecies.org:taxname:12   \n",
       "12           urn:lsid:marinespecies.org:taxname:13   \n",
       "13           urn:lsid:marinespecies.org:taxname:24   \n",
       "14           urn:lsid:marinespecies.org:taxname:25   \n",
       "15           urn:lsid:marinespecies.org:taxname:26   \n",
       "16           urn:lsid:marinespecies.org:taxname:51   \n",
       "17           urn:lsid:marinespecies.org:taxname:55   \n",
       "18           urn:lsid:marinespecies.org:taxname:57   \n",
       "19           urn:lsid:marinespecies.org:taxname:58   \n",
       "20           urn:lsid:marinespecies.org:taxname:59   \n",
       "21           urn:lsid:marinespecies.org:taxname:63   \n",
       "22           urn:lsid:marinespecies.org:taxname:64   \n",
       "23           urn:lsid:marinespecies.org:taxname:69   \n",
       "24           urn:lsid:marinespecies.org:taxname:90   \n",
       "25           urn:lsid:marinespecies.org:taxname:91   \n",
       "26          urn:lsid:marinespecies.org:taxname:101   \n",
       "27          urn:lsid:marinespecies.org:taxname:102   \n",
       "28          urn:lsid:marinespecies.org:taxname:103   \n",
       "29          urn:lsid:marinespecies.org:taxname:104   \n",
       "...                                            ...   \n",
       "597884  urn:lsid:marinespecies.org:taxname:1338683   \n",
       "597885  urn:lsid:marinespecies.org:taxname:1338684   \n",
       "597886  urn:lsid:marinespecies.org:taxname:1338685   \n",
       "597887  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597888  urn:lsid:marinespecies.org:taxname:1338693   \n",
       "597889  urn:lsid:marinespecies.org:taxname:1338694   \n",
       "597890  urn:lsid:marinespecies.org:taxname:1338695   \n",
       "597891  urn:lsid:marinespecies.org:taxname:1338696   \n",
       "597892  urn:lsid:marinespecies.org:taxname:1338697   \n",
       "597893  urn:lsid:marinespecies.org:taxname:1338698   \n",
       "597894  urn:lsid:marinespecies.org:taxname:1338699   \n",
       "597895  urn:lsid:marinespecies.org:taxname:1338700   \n",
       "597896  urn:lsid:marinespecies.org:taxname:1338701   \n",
       "597897  urn:lsid:marinespecies.org:taxname:1338702   \n",
       "597898  urn:lsid:marinespecies.org:taxname:1338703   \n",
       "597899  urn:lsid:marinespecies.org:taxname:1338704   \n",
       "597900  urn:lsid:marinespecies.org:taxname:1338705   \n",
       "597901  urn:lsid:marinespecies.org:taxname:1338723   \n",
       "597902  urn:lsid:marinespecies.org:taxname:1338724   \n",
       "597903  urn:lsid:marinespecies.org:taxname:1338725   \n",
       "597904  urn:lsid:marinespecies.org:taxname:1338726   \n",
       "597905  urn:lsid:marinespecies.org:taxname:1338727   \n",
       "597906  urn:lsid:marinespecies.org:taxname:1338728   \n",
       "597907  urn:lsid:marinespecies.org:taxname:1338729   \n",
       "597908  urn:lsid:marinespecies.org:taxname:1338730   \n",
       "597909  urn:lsid:marinespecies.org:taxname:1338731   \n",
       "597910  urn:lsid:marinespecies.org:taxname:1338732   \n",
       "597911  urn:lsid:marinespecies.org:taxname:1338734   \n",
       "597912  urn:lsid:marinespecies.org:taxname:1338745   \n",
       "597913  urn:lsid:marinespecies.org:taxname:1338746   \n",
       "\n",
       "                               acceptedNameUsageID  \\\n",
       "0             urn:lsid:marinespecies.org:taxname:1   \n",
       "1             urn:lsid:marinespecies.org:taxname:2   \n",
       "2             urn:lsid:marinespecies.org:taxname:3   \n",
       "3             urn:lsid:marinespecies.org:taxname:4   \n",
       "4             urn:lsid:marinespecies.org:taxname:5   \n",
       "5             urn:lsid:marinespecies.org:taxname:6   \n",
       "6             urn:lsid:marinespecies.org:taxname:7   \n",
       "7             urn:lsid:marinespecies.org:taxname:8   \n",
       "8                                              NaN   \n",
       "9            urn:lsid:marinespecies.org:taxname:10   \n",
       "10           urn:lsid:marinespecies.org:taxname:11   \n",
       "11           urn:lsid:marinespecies.org:taxname:12   \n",
       "12       urn:lsid:marinespecies.org:taxname:450030   \n",
       "13                                             NaN   \n",
       "14           urn:lsid:marinespecies.org:taxname:25   \n",
       "15       urn:lsid:marinespecies.org:taxname:562565   \n",
       "16           urn:lsid:marinespecies.org:taxname:51   \n",
       "17           urn:lsid:marinespecies.org:taxname:55   \n",
       "18           urn:lsid:marinespecies.org:taxname:57   \n",
       "19           urn:lsid:marinespecies.org:taxname:58   \n",
       "20           urn:lsid:marinespecies.org:taxname:59   \n",
       "21       urn:lsid:marinespecies.org:taxname:382004   \n",
       "22           urn:lsid:marinespecies.org:taxname:64   \n",
       "23           urn:lsid:marinespecies.org:taxname:69   \n",
       "24           urn:lsid:marinespecies.org:taxname:90   \n",
       "25           urn:lsid:marinespecies.org:taxname:91   \n",
       "26          urn:lsid:marinespecies.org:taxname:101   \n",
       "27          urn:lsid:marinespecies.org:taxname:102   \n",
       "28                                             NaN   \n",
       "29          urn:lsid:marinespecies.org:taxname:104   \n",
       "...                                            ...   \n",
       "597884  urn:lsid:marinespecies.org:taxname:1338683   \n",
       "597885  urn:lsid:marinespecies.org:taxname:1338684   \n",
       "597886  urn:lsid:marinespecies.org:taxname:1338685   \n",
       "597887  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597888  urn:lsid:marinespecies.org:taxname:1338694   \n",
       "597889  urn:lsid:marinespecies.org:taxname:1338694   \n",
       "597890  urn:lsid:marinespecies.org:taxname:1338695   \n",
       "597891  urn:lsid:marinespecies.org:taxname:1338696   \n",
       "597892  urn:lsid:marinespecies.org:taxname:1338696   \n",
       "597893  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597894  urn:lsid:marinespecies.org:taxname:1338700   \n",
       "597895  urn:lsid:marinespecies.org:taxname:1338700   \n",
       "597896  urn:lsid:marinespecies.org:taxname:1338700   \n",
       "597897  urn:lsid:marinespecies.org:taxname:1338702   \n",
       "597898  urn:lsid:marinespecies.org:taxname:1338703   \n",
       "597899  urn:lsid:marinespecies.org:taxname:1338704   \n",
       "597900  urn:lsid:marinespecies.org:taxname:1338703   \n",
       "597901  urn:lsid:marinespecies.org:taxname:1338723   \n",
       "597902  urn:lsid:marinespecies.org:taxname:1338724   \n",
       "597903  urn:lsid:marinespecies.org:taxname:1338725   \n",
       "597904  urn:lsid:marinespecies.org:taxname:1338726   \n",
       "597905   urn:lsid:marinespecies.org:taxname:238058   \n",
       "597906   urn:lsid:marinespecies.org:taxname:238056   \n",
       "597907  urn:lsid:marinespecies.org:taxname:1338729   \n",
       "597908  urn:lsid:marinespecies.org:taxname:1338442   \n",
       "597909  urn:lsid:marinespecies.org:taxname:1338731   \n",
       "597910  urn:lsid:marinespecies.org:taxname:1338732   \n",
       "597911  urn:lsid:marinespecies.org:taxname:1338734   \n",
       "597912  urn:lsid:marinespecies.org:taxname:1338745   \n",
       "597913  urn:lsid:marinespecies.org:taxname:1338746   \n",
       "\n",
       "                                 parentNameUsageID  \\\n",
       "0                                              NaN   \n",
       "1             urn:lsid:marinespecies.org:taxname:1   \n",
       "2             urn:lsid:marinespecies.org:taxname:1   \n",
       "3             urn:lsid:marinespecies.org:taxname:1   \n",
       "4             urn:lsid:marinespecies.org:taxname:1   \n",
       "5             urn:lsid:marinespecies.org:taxname:1   \n",
       "6             urn:lsid:marinespecies.org:taxname:1   \n",
       "7             urn:lsid:marinespecies.org:taxname:1   \n",
       "8             urn:lsid:marinespecies.org:taxname:5   \n",
       "9             urn:lsid:marinespecies.org:taxname:1   \n",
       "10       urn:lsid:marinespecies.org:taxname:536209   \n",
       "11            urn:lsid:marinespecies.org:taxname:1   \n",
       "12            urn:lsid:marinespecies.org:taxname:9   \n",
       "13           urn:lsid:marinespecies.org:taxname:13   \n",
       "14           urn:lsid:marinespecies.org:taxname:24   \n",
       "15       urn:lsid:marinespecies.org:taxname:146221   \n",
       "16            urn:lsid:marinespecies.org:taxname:2   \n",
       "17           urn:lsid:marinespecies.org:taxname:51   \n",
       "18           urn:lsid:marinespecies.org:taxname:55   \n",
       "19           urn:lsid:marinespecies.org:taxname:57   \n",
       "20       urn:lsid:marinespecies.org:taxname:385383   \n",
       "21       urn:lsid:marinespecies.org:taxname:382003   \n",
       "22       urn:lsid:marinespecies.org:taxname:385404   \n",
       "23       urn:lsid:marinespecies.org:taxname:385404   \n",
       "24       urn:lsid:marinespecies.org:taxname:382003   \n",
       "25       urn:lsid:marinespecies.org:taxname:385402   \n",
       "26           urn:lsid:marinespecies.org:taxname:51   \n",
       "27          urn:lsid:marinespecies.org:taxname:101   \n",
       "28        urn:lsid:marinespecies.org:taxname:14712   \n",
       "29           urn:lsid:marinespecies.org:taxname:51   \n",
       "...                                            ...   \n",
       "597884   urn:lsid:marinespecies.org:taxname:138011   \n",
       "597885   urn:lsid:marinespecies.org:taxname:104219   \n",
       "597886   urn:lsid:marinespecies.org:taxname:743922   \n",
       "597887  urn:lsid:marinespecies.org:taxname:1338243   \n",
       "597888  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597889  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597890  urn:lsid:marinespecies.org:taxname:1338243   \n",
       "597891  urn:lsid:marinespecies.org:taxname:1338695   \n",
       "597892   urn:lsid:marinespecies.org:taxname:827571   \n",
       "597893   urn:lsid:marinespecies.org:taxname:414784   \n",
       "597894  urn:lsid:marinespecies.org:taxname:1338698   \n",
       "597895  urn:lsid:marinespecies.org:taxname:1338692   \n",
       "597896  urn:lsid:marinespecies.org:taxname:1338115   \n",
       "597897  urn:lsid:marinespecies.org:taxname:1338243   \n",
       "597898  urn:lsid:marinespecies.org:taxname:1338702   \n",
       "597899  urn:lsid:marinespecies.org:taxname:1338055   \n",
       "597900  urn:lsid:marinespecies.org:taxname:1338704   \n",
       "597901  urn:lsid:marinespecies.org:taxname:1338243   \n",
       "597902  urn:lsid:marinespecies.org:taxname:1338723   \n",
       "597903  urn:lsid:marinespecies.org:taxname:1338243   \n",
       "597904  urn:lsid:marinespecies.org:taxname:1338725   \n",
       "597905   urn:lsid:marinespecies.org:taxname:827571   \n",
       "597906  urn:lsid:marinespecies.org:taxname:1338038   \n",
       "597907  urn:lsid:marinespecies.org:taxname:1338442   \n",
       "597908   urn:lsid:marinespecies.org:taxname:414784   \n",
       "597909  urn:lsid:marinespecies.org:taxname:1328287   \n",
       "597910   urn:lsid:marinespecies.org:taxname:101810   \n",
       "597911  urn:lsid:marinespecies.org:taxname:1337535   \n",
       "597912   urn:lsid:marinespecies.org:taxname:100651   \n",
       "597913  urn:lsid:marinespecies.org:taxname:1338745   \n",
       "\n",
       "                                namePublishedInID  \\\n",
       "0                                             NaN   \n",
       "1                                             NaN   \n",
       "2                                             NaN   \n",
       "3                                             NaN   \n",
       "4                                             NaN   \n",
       "5                                             NaN   \n",
       "6                                             NaN   \n",
       "7                                             NaN   \n",
       "8                                             NaN   \n",
       "9                                             NaN   \n",
       "10                                            NaN   \n",
       "11                                            NaN   \n",
       "12                                            NaN   \n",
       "13                                            NaN   \n",
       "14                                            NaN   \n",
       "15                                            NaN   \n",
       "16                                            NaN   \n",
       "17                                            NaN   \n",
       "18                                            NaN   \n",
       "19                                            NaN   \n",
       "20                                            NaN   \n",
       "21                                            NaN   \n",
       "22                                            NaN   \n",
       "23                                            NaN   \n",
       "24                                            NaN   \n",
       "25                                            NaN   \n",
       "26                                            NaN   \n",
       "27                                            NaN   \n",
       "28                                            NaN   \n",
       "29                                            NaN   \n",
       "...                                           ...   \n",
       "597884                                        NaN   \n",
       "597885                                        NaN   \n",
       "597886                                        NaN   \n",
       "597887                                        NaN   \n",
       "597888                                        NaN   \n",
       "597889                                        NaN   \n",
       "597890                                        NaN   \n",
       "597891                                        NaN   \n",
       "597892                                        NaN   \n",
       "597893                                        NaN   \n",
       "597894                                        NaN   \n",
       "597895                                        NaN   \n",
       "597896                                        NaN   \n",
       "597897                                        NaN   \n",
       "597898                                        NaN   \n",
       "597899                                        NaN   \n",
       "597900                                        NaN   \n",
       "597901                                        NaN   \n",
       "597902                                        NaN   \n",
       "597903                                        NaN   \n",
       "597904                                        NaN   \n",
       "597905                                        NaN   \n",
       "597906                                        NaN   \n",
       "597907                                        NaN   \n",
       "597908                                        NaN   \n",
       "597909                                        NaN   \n",
       "597910                                        NaN   \n",
       "597911  https://doi.org/10.2478/s11687-011-0029-z   \n",
       "597912                                        NaN   \n",
       "597913                                        NaN   \n",
       "\n",
       "                         scientificName                acceptedNameUsage  \\\n",
       "0                                 Biota                            Biota   \n",
       "1                              Animalia                         Animalia   \n",
       "2                               Plantae                          Plantae   \n",
       "3                                 Fungi                            Fungi   \n",
       "4                              Protozoa                         Protozoa   \n",
       "5                              Bacteria                         Bacteria   \n",
       "6                             Chromista                        Chromista   \n",
       "7                               Archaea                          Archaea   \n",
       "8                     Sarcomastigophora                              NaN   \n",
       "9                               Viruses                          Viruses   \n",
       "10                           Ciliophora                       Ciliophora   \n",
       "11                 Biota incertae sedis             Biota incertae sedis   \n",
       "12                         Mastigophora                          Myzozoa   \n",
       "13                      Zoomastigophora                              NaN   \n",
       "14                     Choanoflagellida                 Choanoflagellida   \n",
       "15                       Kinetoplastida                    Kinetoplastea   \n",
       "16                             Mollusca                         Mollusca   \n",
       "17                       Polyplacophora                   Polyplacophora   \n",
       "18                          Neoloricata                      Neoloricata   \n",
       "19                       Lepidopleurida                   Lepidopleurida   \n",
       "20                      Leptochitonidae                  Leptochitonidae   \n",
       "21                      Ischnochitonina                        Chitonina   \n",
       "22                           Chitonidae                       Chitonidae   \n",
       "23                     Ischnochitonidae                 Ischnochitonidae   \n",
       "24                     Acanthochitonina                 Acanthochitonina   \n",
       "25                    Acanthochitonidae                Acanthochitonidae   \n",
       "26                           Gastropoda                       Gastropoda   \n",
       "27                        Prosobranchia                    Prosobranchia   \n",
       "28                            Pulmonata                              NaN   \n",
       "29                           Scaphopoda                       Scaphopoda   \n",
       "...                                 ...                              ...   \n",
       "597884                  Diodora terezae                  Diodora terezae   \n",
       "597885             Ryocalanus squamatus             Ryocalanus squamatus   \n",
       "597886            Yrocalanus kurilensis            Yrocalanus kurilensis   \n",
       "597887                        Halltrema                        Halltrema   \n",
       "597888             Halltrema avitellina            Halltrema avitellinum   \n",
       "597889            Halltrema avitellinum            Halltrema avitellinum   \n",
       "597890                      Nematophila                      Nematophila   \n",
       "597891              Nematophila grandis              Nematophila grandis   \n",
       "597892                Amphistoma grande              Nematophila grandis   \n",
       "597893                Pseudallossostoma                        Halltrema   \n",
       "597894    Pseudallossostoma heteroxenum            Halltrema heteroxenum   \n",
       "597895            Halltrema heteroxenum            Halltrema heteroxenum   \n",
       "597896           Cladorchis heteroxenus            Halltrema heteroxenum   \n",
       "597897               Pseudallastomoides               Pseudallastomoides   \n",
       "597898         Pseudallastomoides rarus         Pseudallastomoides rarus   \n",
       "597899                  Chiostichorchis                  Chiostichorchis   \n",
       "597900            Chiostichorchis rarus         Pseudallastomoides rarus   \n",
       "597901                    Parachiorchis                    Parachiorchis   \n",
       "597902  Parachiorchis parviacetabulatus  Parachiorchis parviacetabulatus   \n",
       "597903               Pseudocleptodiscus               Pseudocleptodiscus   \n",
       "597904    Pseudocleptodiscus margaritae    Pseudocleptodiscus margaritae   \n",
       "597905           Amphistoma scleroporum     Schizamphistomum scleroporum   \n",
       "597906           Amphistomum spinulosum   Schizamphistomoides spinulosus   \n",
       "597907              Allassostoma magnum              Allassostoma magnum   \n",
       "597908                      Alassostoma                     Allassostoma   \n",
       "597909             Ophioxenos dienteros             Ophioxenos dienteros   \n",
       "597910               Scina parasetigera               Scina parasetigera   \n",
       "597911               Dadayius puruensis               Dadayius puruensis   \n",
       "597912                      Relicanthus                      Relicanthus   \n",
       "597913             Relicanthus daphneae             Relicanthus daphneae   \n",
       "\n",
       "                  parentNameUsage  \\\n",
       "0                           Biota   \n",
       "1                           Biota   \n",
       "2                           Biota   \n",
       "3                           Biota   \n",
       "4                           Biota   \n",
       "5                           Biota   \n",
       "6                           Biota   \n",
       "7                           Biota   \n",
       "8                        Protozoa   \n",
       "9                           Biota   \n",
       "10                      Alveolata   \n",
       "11                          Biota   \n",
       "12              Sarcomastigophora   \n",
       "13                   Mastigophora   \n",
       "14                Zoomastigophora   \n",
       "15              Discomitochondria   \n",
       "16                       Animalia   \n",
       "17                       Mollusca   \n",
       "18                 Polyplacophora   \n",
       "19                    Neoloricata   \n",
       "20                 Lepidopleurina   \n",
       "21                      Chitonida   \n",
       "22                    Chitonoidea   \n",
       "23                    Chitonoidea   \n",
       "24                      Chitonida   \n",
       "25                Cryptoplacoidea   \n",
       "26                       Mollusca   \n",
       "27                     Gastropoda   \n",
       "28                 Heterobranchia   \n",
       "29                       Mollusca   \n",
       "...                           ...   \n",
       "597884                    Diodora   \n",
       "597885                 Ryocalanus   \n",
       "597886                 Yrocalanus   \n",
       "597887         Schizamphistominae   \n",
       "597888                  Halltrema   \n",
       "597889                  Halltrema   \n",
       "597890         Schizamphistominae   \n",
       "597891                Nematophila   \n",
       "597892                 Amphistoma   \n",
       "597893              Cladorchiidae   \n",
       "597894          Pseudallossostoma   \n",
       "597895                  Halltrema   \n",
       "597896                 Cladorchis   \n",
       "597897         Schizamphistominae   \n",
       "597898         Pseudallastomoides   \n",
       "597899              Cladorchiinae   \n",
       "597900            Chiostichorchis   \n",
       "597901         Schizamphistominae   \n",
       "597902              Parachiorchis   \n",
       "597903         Schizamphistominae   \n",
       "597904         Pseudocleptodiscus   \n",
       "597905                 Amphistoma   \n",
       "597906                Amphistomum   \n",
       "597907               Allassostoma   \n",
       "597908              Cladorchiidae   \n",
       "597909                 Ophioxenos   \n",
       "597910                      Scina   \n",
       "597911                   Dadayius   \n",
       "597912  Actiniaria incertae sedis   \n",
       "597913                Relicanthus   \n",
       "\n",
       "                                          namePublishedIn namePublishedInYear  \\\n",
       "0                                                     NaN                 NaN   \n",
       "1                                                     NaN                 NaN   \n",
       "2                                                     NaN                 NaN   \n",
       "3                                                     NaN                 NaN   \n",
       "4                                                     NaN                 NaN   \n",
       "5                                                     NaN                 NaN   \n",
       "6                                                     NaN                 NaN   \n",
       "7                                                     NaN                 NaN   \n",
       "8                                                     NaN                 NaN   \n",
       "9                                                     NaN                 NaN   \n",
       "10                                                    NaN                 NaN   \n",
       "11                                                    NaN                 NaN   \n",
       "12                                                    NaN                 NaN   \n",
       "13                                                    NaN                 NaN   \n",
       "14                                                    NaN                 NaN   \n",
       "15                                                    NaN                 NaN   \n",
       "16                                                    NaN                 NaN   \n",
       "17                                                    NaN                 NaN   \n",
       "18      Bergenhayn J. R. M. (1955). Die fossilen schwe...                1955   \n",
       "19                                                    NaN                 NaN   \n",
       "20                                                    NaN                 NaN   \n",
       "21                                                    NaN                 NaN   \n",
       "22                                                    NaN                 NaN   \n",
       "23                                                    NaN                 NaN   \n",
       "24                                                    NaN                 NaN   \n",
       "25                                                    NaN                 NaN   \n",
       "26      Cuvier, G. 1795. Second Mémoire sur l'organisa...                1795   \n",
       "27                                                    NaN                 NaN   \n",
       "28                                                    NaN                 NaN   \n",
       "29                                                    NaN                 NaN   \n",
       "...                                                   ...                 ...   \n",
       "597884  Neves K., Castillo S. & Ramil F. (2019). A new...                2019   \n",
       "597885                                                NaN                 NaN   \n",
       "597886  Renz, J., E.L. Markhaseva & S. Laakmann. (2018...                2018   \n",
       "597887                                                NaN                 NaN   \n",
       "597888                                                NaN                 NaN   \n",
       "597889                                                NaN                 NaN   \n",
       "597890                                                NaN                 NaN   \n",
       "597891                                                NaN                 NaN   \n",
       "597892                                                NaN                 NaN   \n",
       "597893                                                NaN                 NaN   \n",
       "597894                                                NaN                 NaN   \n",
       "597895                                                NaN                 NaN   \n",
       "597896                                                NaN                 NaN   \n",
       "597897                                                NaN                 NaN   \n",
       "597898                                                NaN                 NaN   \n",
       "597899                                                NaN                 NaN   \n",
       "597900                                                NaN                 NaN   \n",
       "597901                                                NaN                 NaN   \n",
       "597902                                                NaN                 NaN   \n",
       "597903                                                NaN                 NaN   \n",
       "597904                                                NaN                 NaN   \n",
       "597905                                                NaN                 NaN   \n",
       "597906                                                NaN                 NaN   \n",
       "597907                                                NaN                 NaN   \n",
       "597908                                                NaN                 NaN   \n",
       "597909                                                NaN                 NaN   \n",
       "597910  Zeidler, W. (1990). Pelagic Amphipods, Infraor...                1990   \n",
       "597911  Lopes, L.; Karling, L.; Takemoto, R.; Rossoni,...                2011   \n",
       "597912                                                NaN                 NaN   \n",
       "597913                                                NaN                 NaN   \n",
       "\n",
       "        ... nomenclaturalCode           taxonomicStatus    modified  \\\n",
       "0       ...               NaN                  accepted  2004-12-21   \n",
       "1       ...               NaN                  accepted  2004-12-21   \n",
       "2       ...               NaN                  accepted  2011-07-29   \n",
       "3       ...               NaN                  accepted  2004-12-21   \n",
       "4       ...               NaN                  accepted  2011-07-29   \n",
       "5       ...               NaN                  accepted  2010-01-19   \n",
       "6       ...               NaN                  accepted  2008-10-22   \n",
       "7       ...               NaN                  accepted  2009-09-24   \n",
       "8       ...               NaN                unaccepted  2004-12-21   \n",
       "9       ...               NaN                  accepted  2015-10-30   \n",
       "10      ...               ICN                  accepted  2011-10-17   \n",
       "11      ...               NaN            temporary name  2012-06-19   \n",
       "12      ...               NaN                unaccepted  2011-02-28   \n",
       "13      ...               NaN                unaccepted  2004-12-21   \n",
       "14      ...               NaN                  accepted  2013-11-26   \n",
       "15      ...               NaN                unaccepted  2011-02-28   \n",
       "16      ...              ICZN                  accepted  2004-12-21   \n",
       "17      ...              ICZN                  accepted  2009-01-30   \n",
       "18      ...              ICZN                  accepted  2009-01-30   \n",
       "19      ...              ICZN                  accepted  2009-03-17   \n",
       "20      ...              ICZN                  accepted  2009-03-17   \n",
       "21      ...              ICZN                unaccepted  2010-07-12   \n",
       "22      ...              ICZN                  accepted  2009-03-19   \n",
       "23      ...              ICZN                  accepted  2009-03-19   \n",
       "24      ...              ICZN                  accepted  2009-01-30   \n",
       "25      ...              ICZN                  accepted  2009-03-19   \n",
       "26      ...              ICZN                  accepted  2009-03-17   \n",
       "27      ...              ICZN  alternate representation  2009-03-27   \n",
       "28      ...              ICZN                unaccepted  2018-02-08   \n",
       "29      ...              ICZN                  accepted  2011-11-23   \n",
       "...     ...               ...                       ...         ...   \n",
       "597884  ...              ICZN                  accepted  2019-04-04   \n",
       "597885  ...              ICZN                  accepted  2019-04-04   \n",
       "597886  ...              ICZN                  accepted  2019-04-04   \n",
       "597887  ...              ICZN                  accepted  2019-04-04   \n",
       "597888  ...              ICZN                unaccepted  2019-04-04   \n",
       "597889  ...              ICZN                  accepted  2019-04-04   \n",
       "597890  ...              ICZN                  accepted  2019-04-04   \n",
       "597891  ...              ICZN                  accepted  2019-04-04   \n",
       "597892  ...              ICZN                unaccepted  2019-04-04   \n",
       "597893  ...              ICZN                unaccepted  2019-04-04   \n",
       "597894  ...              ICZN                unaccepted  2019-04-04   \n",
       "597895  ...              ICZN                  accepted  2019-04-04   \n",
       "597896  ...              ICZN                unaccepted  2019-04-04   \n",
       "597897  ...              ICZN         taxon inquirendum  2019-04-04   \n",
       "597898  ...              ICZN                  accepted  2019-04-04   \n",
       "597899  ...              ICZN                  accepted  2019-04-04   \n",
       "597900  ...              ICZN                unaccepted  2019-04-04   \n",
       "597901  ...              ICZN                  accepted  2019-04-04   \n",
       "597902  ...              ICZN                  accepted  2019-04-04   \n",
       "597903  ...              ICZN                  accepted  2019-04-04   \n",
       "597904  ...              ICZN                  accepted  2019-04-04   \n",
       "597905  ...              ICZN                unaccepted  2019-04-04   \n",
       "597906  ...              ICZN                unaccepted  2019-04-04   \n",
       "597907  ...              ICZN                  accepted  2019-04-04   \n",
       "597908  ...              ICZN                unaccepted  2019-04-04   \n",
       "597909  ...              ICZN                  accepted  2019-04-04   \n",
       "597910  ...              ICZN                  accepted  2019-04-04   \n",
       "597911  ...              ICZN                  accepted  2019-04-04   \n",
       "597912  ...              ICZN                  accepted  2019-04-04   \n",
       "597913  ...              ICZN                  accepted  2019-04-04   \n",
       "\n",
       "                                    bibliographicCitation  \\\n",
       "0       WoRMS (2019). Biota. Accessed at: http://www.m...   \n",
       "1       WoRMS (2019). Animalia. Accessed at: http://ww...   \n",
       "2       WoRMS (2019). Plantae. Accessed at: http://www...   \n",
       "3       WoRMS (2019). Fungi. Accessed at: http://www.m...   \n",
       "4       WoRMS (2019). Protozoa. Accessed at: http://ww...   \n",
       "5       WoRMS (2019). Bacteria. Accessed at: http://ww...   \n",
       "6       WoRMS (2019). Chromista. Accessed at: http://w...   \n",
       "7       WoRMS (2019). Archaea. Accessed at: http://www...   \n",
       "8       Guiry, M.D. & Guiry, G.M. (2019). AlgaeBase. W...   \n",
       "9       WoRMS (2019). Viruses. Accessed at: http://www...   \n",
       "10      WoRMS (2019). Ciliophora. Accessed at: http://...   \n",
       "11      Guiry, M.D. & Guiry, G.M. (2019). AlgaeBase. W...   \n",
       "12      Guiry, M.D. & Guiry, G.M. (2019). AlgaeBase. W...   \n",
       "13      Guiry, M.D. & Guiry, G.M. (2019). AlgaeBase. W...   \n",
       "14      Guiry, M.D. & Guiry, G.M. (2019). AlgaeBase. W...   \n",
       "15      Guiry, M.D. & Guiry, G.M. (2019). AlgaeBase. W...   \n",
       "16      WoRMS (2019). Mollusca. Accessed at: http://ww...   \n",
       "17      MolluscaBase (2019). MolluscaBase. Polyplacoph...   \n",
       "18      MolluscaBase (2019). MolluscaBase. Neoloricata...   \n",
       "19      MolluscaBase (2019). MolluscaBase. Lepidopleur...   \n",
       "20      MolluscaBase (2019). MolluscaBase. Leptochiton...   \n",
       "21      MolluscaBase (2019). MolluscaBase. Ischnochito...   \n",
       "22      MolluscaBase (2019). MolluscaBase. Chitonidae ...   \n",
       "23      MolluscaBase (2019). MolluscaBase. Ischnochito...   \n",
       "24      MolluscaBase (2019). MolluscaBase. Acanthochit...   \n",
       "25      MolluscaBase (2019). MolluscaBase. Acanthochit...   \n",
       "26      WoRMS (2019). Gastropoda. Accessed at: http://...   \n",
       "27      MolluscaBase (2019). MolluscaBase. Prosobranch...   \n",
       "28      MolluscaBase (2019). MolluscaBase. Pulmonata. ...   \n",
       "29      MolluscaBase (2019). MolluscaBase. Scaphopoda....   \n",
       "...                                                   ...   \n",
       "597884  MolluscaBase (2019). MolluscaBase. Diodora ter...   \n",
       "597885  Walter, T.C. & Boxshall, G. (2019). World of C...   \n",
       "597886  Walter, T.C. & Boxshall, G. (2019). World of C...   \n",
       "597887  WoRMS (2019). Halltrema Lent & Freitas, 1939. ...   \n",
       "597888  WoRMS (2019). Halltrema avitellina Lent & Frei...   \n",
       "597889  WoRMS (2019). Halltrema avitellinum Lent & Fre...   \n",
       "597890  WoRMS (2019). Nematophila Travassos, 1934. Acc...   \n",
       "597891  WoRMS (2019). Nematophila grandis (Diesing, 18...   \n",
       "597892  WoRMS (2019). Amphistoma grande Diesing, 1839....   \n",
       "597893  WoRMS (2019). Pseudallossostoma Yamaguti, 1958...   \n",
       "597894  WoRMS (2019). Pseudallossostoma heteroxenum (C...   \n",
       "597895  WoRMS (2019). Halltrema heteroxenum (Cordero &...   \n",
       "597896  WoRMS (2019). Cladorchis heteroxenus Cordero &...   \n",
       "597897  WoRMS (2019). Pseudallastomoides Yamaguti, 197...   \n",
       "597898  WoRMS (2019). Pseudallastomoides rarus (Rohde,...   \n",
       "597899  WoRMS (2019). Chiostichorchis Artigas & Pachec...   \n",
       "597900  WoRMS (2019). Chiostichorchis rarus Rohde, 196...   \n",
       "597901  WoRMS (2019). Parachiorchis Caballero, 1943. A...   \n",
       "597902  WoRMS (2019). Parachiorchis parviacetabulatus ...   \n",
       "597903  WoRMS (2019). Pseudocleptodiscus Caballero, 19...   \n",
       "597904  WoRMS (2019). Pseudocleptodiscus margaritae Ca...   \n",
       "597905  WoRMS (2019). Amphistoma scleroporum Creplin, ...   \n",
       "597906  WoRMS (2019). Amphistomum spinulosum Looss, 19...   \n",
       "597907  WoRMS (2019). Allassostoma magnum Stunkard, 19...   \n",
       "597908  WoRMS (2019). Alassostoma Stunkard, 1917. Acce...   \n",
       "597909  WoRMS (2019). Ophioxenos dienteros Sumwalt, 19...   \n",
       "597910  WoRMS (2019). Scina parasetigera Zeidler, 1990...   \n",
       "597911  WoRMS (2019). Dadayius puruensis Lopes, Karlin...   \n",
       "597912  WoRMS (2019). Relicanthus Rodriguez & Daly 201...   \n",
       "597913  WoRMS (2019). Relicanthus daphneae (Daly 2006)...   \n",
       "\n",
       "                                               references license  \\\n",
       "0       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "1       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "2       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "3       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "4       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "5       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "6       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "7       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "8       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "9       http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "10      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "11      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "12      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "13      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "14      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "15      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "16      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "17      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "18      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "19      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "20      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "21      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "22      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "23      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "24      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "25      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "26      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "27      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "28      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "29      http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "...                                                   ...     ...   \n",
       "597884  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597885  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597886  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597887  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597888  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597889  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597890  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597891  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597892  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597893  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597894  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597895  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597896  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597897  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597898  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597899  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597900  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597901  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597902  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597903  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597904  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597905  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597906  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597907  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597908  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597909  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597910  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597911  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597912  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "597913  http://www.marinespecies.org/aphia.php?p=taxde...     NaN   \n",
       "\n",
       "       rightsHolder datasetName institutionCode datasetID  \n",
       "0               NaN         NaN             NaN       NaN  \n",
       "1               NaN         NaN             NaN       NaN  \n",
       "2               NaN         NaN             NaN       NaN  \n",
       "3               NaN         NaN             NaN       NaN  \n",
       "4               NaN         NaN             NaN       NaN  \n",
       "5               NaN         NaN             NaN       NaN  \n",
       "6               NaN         NaN             NaN       NaN  \n",
       "7               NaN         NaN             NaN       NaN  \n",
       "8               NaN         NaN             NaN       NaN  \n",
       "9               NaN         NaN             NaN       NaN  \n",
       "10              NaN         NaN             NaN       NaN  \n",
       "11              NaN         NaN             NaN       NaN  \n",
       "12              NaN         NaN             NaN       NaN  \n",
       "13              NaN         NaN             NaN       NaN  \n",
       "14              NaN         NaN             NaN       NaN  \n",
       "15              NaN         NaN             NaN       NaN  \n",
       "16              NaN         NaN             NaN       NaN  \n",
       "17              NaN         NaN             NaN       NaN  \n",
       "18              NaN         NaN             NaN       NaN  \n",
       "19              NaN         NaN             NaN       NaN  \n",
       "20              NaN         NaN             NaN       NaN  \n",
       "21              NaN         NaN             NaN       NaN  \n",
       "22              NaN         NaN             NaN       NaN  \n",
       "23              NaN         NaN             NaN       NaN  \n",
       "24              NaN         NaN             NaN       NaN  \n",
       "25              NaN         NaN             NaN       NaN  \n",
       "26              NaN         NaN             NaN       NaN  \n",
       "27              NaN         NaN             NaN       NaN  \n",
       "28              NaN         NaN             NaN       NaN  \n",
       "29              NaN         NaN             NaN       NaN  \n",
       "...             ...         ...             ...       ...  \n",
       "597884          NaN         NaN             NaN       NaN  \n",
       "597885          NaN         NaN             NaN       NaN  \n",
       "597886          NaN         NaN             NaN       NaN  \n",
       "597887          NaN         NaN             NaN       NaN  \n",
       "597888          NaN         NaN             NaN       NaN  \n",
       "597889          NaN         NaN             NaN       NaN  \n",
       "597890          NaN         NaN             NaN       NaN  \n",
       "597891          NaN         NaN             NaN       NaN  \n",
       "597892          NaN         NaN             NaN       NaN  \n",
       "597893          NaN         NaN             NaN       NaN  \n",
       "597894          NaN         NaN             NaN       NaN  \n",
       "597895          NaN         NaN             NaN       NaN  \n",
       "597896          NaN         NaN             NaN       NaN  \n",
       "597897          NaN         NaN             NaN       NaN  \n",
       "597898          NaN         NaN             NaN       NaN  \n",
       "597899          NaN         NaN             NaN       NaN  \n",
       "597900          NaN         NaN             NaN       NaN  \n",
       "597901          NaN         NaN             NaN       NaN  \n",
       "597902          NaN         NaN             NaN       NaN  \n",
       "597903          NaN         NaN             NaN       NaN  \n",
       "597904          NaN         NaN             NaN       NaN  \n",
       "597905          NaN         NaN             NaN       NaN  \n",
       "597906          NaN         NaN             NaN       NaN  \n",
       "597907          NaN         NaN             NaN       NaN  \n",
       "597908          NaN         NaN             NaN       NaN  \n",
       "597909          NaN         NaN             NaN       NaN  \n",
       "597910          NaN         NaN             NaN       NaN  \n",
       "597911          NaN         NaN             NaN       NaN  \n",
       "597912          NaN         NaN             NaN       NaN  \n",
       "597913          NaN         NaN             NaN       NaN  \n",
       "\n",
       "[597914 rows x 31 columns]>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# See what the taxon data looks like\n",
    "taxon.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract just the scientific names to a list\n",
    "sci_names = taxon[\"scientificName\"].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "597914"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# See how many scientific names we have\n",
    "len(sci_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Spacy's medium size English model\n",
    "nlp = spacy.load(\"en_core_web_md\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new pipeline step using spacy_lookup.\n",
    "# We use the scientific names as the keyword dictionary, and use MARINE as the entity label\n",
    "entity = Entity(keywords_list=sci_names, label=\"MARINE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add our new pipeline step to our pipeline\n",
    "nlp.add_pipe(entity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('ner', <spacy.pipeline.pipes.EntityRecognizer at 0x1209b2168>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We have to remove the normal NER step since entities can only be in one ent type, and it causes conflicts\n",
    "nlp.remove_pipe(\"ner\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identify the directory containing all the papers we want to process\n",
    "text_dir = \"allpapers\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creates lists of filenames and read in texts from each file in the text_dir\n",
    "fns = []\n",
    "texts = []\n",
    "for fn in glob.glob(text_dir + \"/*.txt\"):\n",
    "    fns.append(fn.split(\"/\")[1])\n",
    "    with open(fn, 'r') as f:\n",
    "        texts.append(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stream the full texts into the NLP model with the additional entity id step and return a list\n",
    "docs= list(nlp.pipe(texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a list of sets of entities for every doc\n",
    "# We have to get the text representation of each ent for set to identify unique values\n",
    "ent_sets = [set([ent.text.lower() for ent in doc.ents]) for doc in docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a dictionary with the filenames and corresponding entity sets\n",
    "data = {\n",
    "    \"filename\": fns,\n",
    "    \"ents\": ent_sets\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe with filenames and entities\n",
    "ent_df = pd.DataFrame(data = data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>ents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ucla_1989_MarKelvin.txt</td>\n",
       "      <td>{astropecten brasiliensis armatus, polyacanthu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>usc_1998_Sp_Carr.txt</td>\n",
       "      <td>{julia, monica, sheila, boca, arbor, bacteria}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ucla_1981_WuestehubeLindaJ.txt</td>\n",
       "      <td>{spirobranchus spinosus, here, halichoeres}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hms_1989_MillerStephenE.txt</td>\n",
       "      <td>{anthopleura, clinocottus analis, oxylebius pi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ucla_1975_ChapmanJohn_Asculpta.txt</td>\n",
       "      <td>{acanthomysis sculpta, mysis, macrocystis, aca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>usc_1974_SpFa_Lissner.txt</td>\n",
       "      <td>{centrostephanus coronatus, io, corona, strong...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>usc_1978_Butsumyo_etal.txt</td>\n",
       "      <td>{perca, afer, lo, io, wak, nu, ino, pao, uva}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>usc_1991_Sp_Hentschke.txt</td>\n",
       "      <td>{callianassa, callianassidae, upogebiidae, che...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ucla_1981_HughesBrian.txt</td>\n",
       "      <td>{dictyota, paralabrax clathratus, io, loligo, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>usc_1971_Coyer.txt</td>\n",
       "      <td>{spirobranchus, lo, nereis diversicolor, hydro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ucla_1989_DaleyShawn.txt</td>\n",
       "      <td>{octopus bimaculoides, uca, pachygrapsus crass...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>usc_1979_Su_Bergen.txt</td>\n",
       "      <td>{centrostephanus coronatus, macrocystis, stron...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>usc_1982_Su_Yourke.txt</td>\n",
       "      <td>{corona}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ucla_1978_SkollerDebbie_Sspinosus.txt</td>\n",
       "      <td>{pisaster, echinodermata, strongylocentrotus p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>usc_1976_Su_Baez_2.txt</td>\n",
       "      <td>{callianassa, emerita analoga, uca, amphipoda,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>carl_1986_SpringIndex.txt</td>\n",
       "      <td>{eisenia, nuttalina, tetraclita rubescens, pha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>hms_1964_CoxAnnLouise.txt</td>\n",
       "      <td>{ammonia, io, vera, planaxis, littorina planax...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ucla_1981_INDEX.txt</td>\n",
       "      <td>{gelidium, spirobranchus, paralabrax clathratu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ucla_1975_BuckleyGary.txt</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>usc_1979_Sp_Smithwich.txt</td>\n",
       "      <td>{arbor, io, arca}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>carl_1978_SpornThomasA.txt</td>\n",
       "      <td>{pisaster, pisaster giganteus, astraea undosa,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>carl_1983_McCuskeyCharles.txt</td>\n",
       "      <td>{asa, fragile, cystoseira, norrisia, serpulorb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>usc_1977_Su_Hageman.txt</td>\n",
       "      <td>{aor, lo, ova, io, des, arca}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>usc_1990_V2_Modean.txt</td>\n",
       "      <td>{halidrys, radula, laminaria, trochidae, norri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ucla_1975_StarczakVictoria.txt</td>\n",
       "      <td>{pisaster ochraceus, pisaster, pisaster gigant...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>hms_1964_FarnumCornelia.txt</td>\n",
       "      <td>{beta, littorina scutulata, planaxis, enteromo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>usc_1979_Su_Payne.txt</td>\n",
       "      <td>{scorpaena guttata, stereolepis gigas, paralab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>usc_1976_Su_Baez.txt</td>\n",
       "      <td>{radius, tegula funebralis, paguristes ulreyi,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>usc_1982_Su_INDEX.txt</td>\n",
       "      <td>{coryphopterus nicholsii, centrostephanus coro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>usc_1994_Sp_Nedved.txt</td>\n",
       "      <td>{radius, lar, lo, nais, bacteria}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>ucla_1978_FirpoJanine_DensityDiversity.txt</td>\n",
       "      <td>{cnidaria, tegula, bryozoa, phyla, phaeophyta,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>usc_1990_V2_Chavez.txt</td>\n",
       "      <td>{mytilus, mytilidae, mollusca, byssus, extra}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>usc_2000_Hornick.txt</td>\n",
       "      <td>{ammonia, cylinder, capitella, bacteria, atrac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>usc_1976_Su_Green.txt</td>\n",
       "      <td>{lo, uca, orchestoidea, amphipoda, orchestoide...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>ucla_1988_MillarRenee.txt</td>\n",
       "      <td>{gracilaria, aplysia californica, rhodophyta, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>usc_1990_V1_Chavez_Villalba.txt</td>\n",
       "      <td>{io, johnson, artemis, nauplius, here}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>usc_1979_Sp_Sullivan.txt</td>\n",
       "      <td>{halidrys, eisenia, tegula funebralis, macrocy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>ucla_1988_SanchezShelley.txt</td>\n",
       "      <td>{lytechinus variegatus, pelvetia, centrostepha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>usc_1990_V1_Mensa.txt</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>ucla_1989_BaghdasarianGaren.txt</td>\n",
       "      <td>{ammonia, lates, laminaria, macrocystis, chrom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>410</th>\n",
       "      <td>ucla_1983_BoyajianLisaS.txt</td>\n",
       "      <td>{membranipora, pisaster, pisaster giganteus, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>usc_1990_V2_Silva.txt</td>\n",
       "      <td>{pisaster, mytilus, io, armatus, here, limatul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>ucla_1987_HuangMay.txt</td>\n",
       "      <td>{box, pisaster, chama, astraea undosa, bryozoa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>hms_1964_MiyamotoAlan.txt</td>\n",
       "      <td>{box, io, planaxis, littorina planaxis, here}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>414</th>\n",
       "      <td>usc_1978_Coyne_etal.txt</td>\n",
       "      <td>{lo, iva}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415</th>\n",
       "      <td>usc_1993_Sp_Howard.txt</td>\n",
       "      <td>{here}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416</th>\n",
       "      <td>usc_1990_V1_Modean.txt</td>\n",
       "      <td>{bacteria, macrocystis}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>hms_1989_DasGuptaHindola.txt</td>\n",
       "      <td>{here, sigma}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>hms_1964_NealeJR.txt</td>\n",
       "      <td>{osaka, ser, planaxis, littorina planaxis, here}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>usc_2000_Ahmed.txt</td>\n",
       "      <td>{here, io, capea, atriplex}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420</th>\n",
       "      <td>ucla_1981_DemetriouStevenJ.txt</td>\n",
       "      <td>{io, parastichopus, here, macrocystis, aquaria...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>hms_1964_TittleKennethM.txt</td>\n",
       "      <td>{littorina, acanthina spirata, tegula, littori...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>ucla_1975_RoquemoreFrank.txt</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>carl_1986_Bailey_and_Onello.txt</td>\n",
       "      <td>{laminariales, macrocystis}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>ucla_1981_DaparRhysV.txt</td>\n",
       "      <td>{oxyjulis californica, embiotocidae, scorpaeni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>ucla_1987_INDEX_Volume2.txt</td>\n",
       "      <td>{pelvetia, cnidaria, astraea undosa, eisenia, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>426</th>\n",
       "      <td>ucla_1987_ReichenbergerPatrick.txt</td>\n",
       "      <td>{pisaster, hypsypops rubicunda, spirobranchus,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427</th>\n",
       "      <td>hms_1964_DahlArthurLyon.txt</td>\n",
       "      <td>{prasiola, pelvetia, fucus, laminaria, cladoph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>428</th>\n",
       "      <td>ucla_1975_BrownGarry.txt</td>\n",
       "      <td>{lima, norrissia, tegula, odostomia, io, calli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>ucla_1989_INDEX.txt</td>\n",
       "      <td>{octopus bimaculoides, paralabrax clathratus, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>430 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       filename  \\\n",
       "0                       ucla_1989_MarKelvin.txt   \n",
       "1                          usc_1998_Sp_Carr.txt   \n",
       "2                ucla_1981_WuestehubeLindaJ.txt   \n",
       "3                   hms_1989_MillerStephenE.txt   \n",
       "4            ucla_1975_ChapmanJohn_Asculpta.txt   \n",
       "5                     usc_1974_SpFa_Lissner.txt   \n",
       "6                    usc_1978_Butsumyo_etal.txt   \n",
       "7                     usc_1991_Sp_Hentschke.txt   \n",
       "8                     ucla_1981_HughesBrian.txt   \n",
       "9                            usc_1971_Coyer.txt   \n",
       "10                     ucla_1989_DaleyShawn.txt   \n",
       "11                       usc_1979_Su_Bergen.txt   \n",
       "12                       usc_1982_Su_Yourke.txt   \n",
       "13        ucla_1978_SkollerDebbie_Sspinosus.txt   \n",
       "14                       usc_1976_Su_Baez_2.txt   \n",
       "15                    carl_1986_SpringIndex.txt   \n",
       "16                    hms_1964_CoxAnnLouise.txt   \n",
       "17                          ucla_1981_INDEX.txt   \n",
       "18                    ucla_1975_BuckleyGary.txt   \n",
       "19                    usc_1979_Sp_Smithwich.txt   \n",
       "20                   carl_1978_SpornThomasA.txt   \n",
       "21                carl_1983_McCuskeyCharles.txt   \n",
       "22                      usc_1977_Su_Hageman.txt   \n",
       "23                       usc_1990_V2_Modean.txt   \n",
       "24               ucla_1975_StarczakVictoria.txt   \n",
       "25                  hms_1964_FarnumCornelia.txt   \n",
       "26                        usc_1979_Su_Payne.txt   \n",
       "27                         usc_1976_Su_Baez.txt   \n",
       "28                        usc_1982_Su_INDEX.txt   \n",
       "29                       usc_1994_Sp_Nedved.txt   \n",
       "..                                          ...   \n",
       "400  ucla_1978_FirpoJanine_DensityDiversity.txt   \n",
       "401                      usc_1990_V2_Chavez.txt   \n",
       "402                        usc_2000_Hornick.txt   \n",
       "403                       usc_1976_Su_Green.txt   \n",
       "404                   ucla_1988_MillarRenee.txt   \n",
       "405             usc_1990_V1_Chavez_Villalba.txt   \n",
       "406                    usc_1979_Sp_Sullivan.txt   \n",
       "407                ucla_1988_SanchezShelley.txt   \n",
       "408                       usc_1990_V1_Mensa.txt   \n",
       "409             ucla_1989_BaghdasarianGaren.txt   \n",
       "410                 ucla_1983_BoyajianLisaS.txt   \n",
       "411                       usc_1990_V2_Silva.txt   \n",
       "412                      ucla_1987_HuangMay.txt   \n",
       "413                   hms_1964_MiyamotoAlan.txt   \n",
       "414                     usc_1978_Coyne_etal.txt   \n",
       "415                      usc_1993_Sp_Howard.txt   \n",
       "416                      usc_1990_V1_Modean.txt   \n",
       "417                hms_1989_DasGuptaHindola.txt   \n",
       "418                        hms_1964_NealeJR.txt   \n",
       "419                          usc_2000_Ahmed.txt   \n",
       "420              ucla_1981_DemetriouStevenJ.txt   \n",
       "421                 hms_1964_TittleKennethM.txt   \n",
       "422                ucla_1975_RoquemoreFrank.txt   \n",
       "423             carl_1986_Bailey_and_Onello.txt   \n",
       "424                    ucla_1981_DaparRhysV.txt   \n",
       "425                 ucla_1987_INDEX_Volume2.txt   \n",
       "426          ucla_1987_ReichenbergerPatrick.txt   \n",
       "427                 hms_1964_DahlArthurLyon.txt   \n",
       "428                    ucla_1975_BrownGarry.txt   \n",
       "429                         ucla_1989_INDEX.txt   \n",
       "\n",
       "                                                  ents  \n",
       "0    {astropecten brasiliensis armatus, polyacanthu...  \n",
       "1       {julia, monica, sheila, boca, arbor, bacteria}  \n",
       "2          {spirobranchus spinosus, here, halichoeres}  \n",
       "3    {anthopleura, clinocottus analis, oxylebius pi...  \n",
       "4    {acanthomysis sculpta, mysis, macrocystis, aca...  \n",
       "5    {centrostephanus coronatus, io, corona, strong...  \n",
       "6        {perca, afer, lo, io, wak, nu, ino, pao, uva}  \n",
       "7    {callianassa, callianassidae, upogebiidae, che...  \n",
       "8    {dictyota, paralabrax clathratus, io, loligo, ...  \n",
       "9    {spirobranchus, lo, nereis diversicolor, hydro...  \n",
       "10   {octopus bimaculoides, uca, pachygrapsus crass...  \n",
       "11   {centrostephanus coronatus, macrocystis, stron...  \n",
       "12                                            {corona}  \n",
       "13   {pisaster, echinodermata, strongylocentrotus p...  \n",
       "14   {callianassa, emerita analoga, uca, amphipoda,...  \n",
       "15   {eisenia, nuttalina, tetraclita rubescens, pha...  \n",
       "16   {ammonia, io, vera, planaxis, littorina planax...  \n",
       "17   {gelidium, spirobranchus, paralabrax clathratu...  \n",
       "18                                                  {}  \n",
       "19                                   {arbor, io, arca}  \n",
       "20   {pisaster, pisaster giganteus, astraea undosa,...  \n",
       "21   {asa, fragile, cystoseira, norrisia, serpulorb...  \n",
       "22                       {aor, lo, ova, io, des, arca}  \n",
       "23   {halidrys, radula, laminaria, trochidae, norri...  \n",
       "24   {pisaster ochraceus, pisaster, pisaster gigant...  \n",
       "25   {beta, littorina scutulata, planaxis, enteromo...  \n",
       "26   {scorpaena guttata, stereolepis gigas, paralab...  \n",
       "27   {radius, tegula funebralis, paguristes ulreyi,...  \n",
       "28   {coryphopterus nicholsii, centrostephanus coro...  \n",
       "29                   {radius, lar, lo, nais, bacteria}  \n",
       "..                                                 ...  \n",
       "400  {cnidaria, tegula, bryozoa, phyla, phaeophyta,...  \n",
       "401      {mytilus, mytilidae, mollusca, byssus, extra}  \n",
       "402  {ammonia, cylinder, capitella, bacteria, atrac...  \n",
       "403  {lo, uca, orchestoidea, amphipoda, orchestoide...  \n",
       "404  {gracilaria, aplysia californica, rhodophyta, ...  \n",
       "405             {io, johnson, artemis, nauplius, here}  \n",
       "406  {halidrys, eisenia, tegula funebralis, macrocy...  \n",
       "407  {lytechinus variegatus, pelvetia, centrostepha...  \n",
       "408                                                 {}  \n",
       "409  {ammonia, lates, laminaria, macrocystis, chrom...  \n",
       "410  {membranipora, pisaster, pisaster giganteus, a...  \n",
       "411  {pisaster, mytilus, io, armatus, here, limatul...  \n",
       "412  {box, pisaster, chama, astraea undosa, bryozoa...  \n",
       "413      {box, io, planaxis, littorina planaxis, here}  \n",
       "414                                          {lo, iva}  \n",
       "415                                             {here}  \n",
       "416                            {bacteria, macrocystis}  \n",
       "417                                      {here, sigma}  \n",
       "418   {osaka, ser, planaxis, littorina planaxis, here}  \n",
       "419                        {here, io, capea, atriplex}  \n",
       "420  {io, parastichopus, here, macrocystis, aquaria...  \n",
       "421  {littorina, acanthina spirata, tegula, littori...  \n",
       "422                                                 {}  \n",
       "423                        {laminariales, macrocystis}  \n",
       "424  {oxyjulis californica, embiotocidae, scorpaeni...  \n",
       "425  {pelvetia, cnidaria, astraea undosa, eisenia, ...  \n",
       "426  {pisaster, hypsypops rubicunda, spirobranchus,...  \n",
       "427  {prasiola, pelvetia, fucus, laminaria, cladoph...  \n",
       "428  {lima, norrissia, tegula, odostomia, io, calli...  \n",
       "429  {octopus bimaculoides, paralabrax clathratus, ...  \n",
       "\n",
       "[430 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ent_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the dataframe to a csv file\n",
    "ent_df.to_csv('all_papers_ents.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make this a pure python script with a cli interface by way of click\n",
    "# parameters would be the taxon file and the dir name for the directory of full texts in plain text format"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}