06_export_ihmp_data.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export iHMP datasets\n",
    "> Just take the get_diffs function and output to directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from src.ihmp import get_diffs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for name in [\"ibd\", \"moms\", \"t2d\"]:\n",
    "for name in [\"t2d\"]:\n",
    "    data, metadata = get_diffs(\n",
    "        name,\n",
    "        top_n=None,\n",
    "        get_abundances=True,\n",
    "        include_metadata=True,\n",
    "        log=False,\n",
    "        normalize=True,\n",
    "    )\n",
    "    data = data.drop_duplicates()\n",
    "    metadata = metadata.set_index(\n",
    "        [\"sample_body_site\", \"subject_id\", \"visit_number\", \"sample_id\"]\n",
    "    ).drop_duplicates()\n",
    "    metadata.index.names = [\"site\", \"patient\", \"visit\", \"sample\"]\n",
    "    data.to_csv(f\"ihmp/{name}_data.csv.gz\", compression=\"gzip\")\n",
    "    metadata.to_csv(f\"ihmp/{name}_metadata.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set()\n",
      "set()\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# Check that data and metadata mathc on [site, patient, visit] only:\n",
    "\n",
    "data_idx = data.index\n",
    "metadata_idx = metadata.index\n",
    "\n",
    "# Drop sample ID from both\n",
    "data_idx = data_idx.droplevel(\"sample\")\n",
    "metadata_idx = metadata_idx.droplevel(\"sample\")\n",
    "\n",
    "print(set(data_idx) - set(metadata_idx))\n",
    "print(set(metadata_idx) - set(data_idx))\n",
    "print(set(data_idx) == set(metadata_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ZN9YTFN-01', 'ZN9YTFN-01', 'ZN9YTFN-1011', 'ZN9YTFN-1011',\n",
       "       'ZN9YTFN-1012', 'ZN9YTFN-1012', 'ZN9YTFN-1013', 'ZN9YTFN-1013',\n",
       "       'ZN9YTFN-1014', 'ZN9YTFN-1014',\n",
       "       ...\n",
       "       'ZL63I8R-1014', 'ZL63I8R-1014', 'ZL63I8R-1014', 'ZLYEM2C-07',\n",
       "       'ZKFV71L-2013', 'ZKFV71L-2013', 'ZKFV71L-2013', 'ZN9YTFN-08',\n",
       "       'ZJXC41N-02', 'ZLTUJTN-05'],\n",
       "      dtype='object', name='VisitID', length=1528)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2d_metadata = pd.read_csv(\"/home/phil/phylosig/ihmp/t2d_metadata_ihmp.csv\", index_col=0)\n",
    "t2d_metadata.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>subject_uuid</th>\n",
       "      <th>subject_gender</th>\n",
       "      <th>subject_race</th>\n",
       "      <th>study_full_name</th>\n",
       "      <th>project_name</th>\n",
       "      <th>sample</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>site</th>\n",
       "      <th>patient</th>\n",
       "      <th>visit</th>\n",
       "      <th>sample</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">nasal cavity</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">ZVBQY1N</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">11</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c669dfd</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J43833_1_NS_T0_B0_0120_ZVBQY1N-1024_ANAV8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>932d8fbc70ae8f856028b3f67c669dfd</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J29034_1_NS_T0_B0_0120_ZVBQY1N-1024_AN77Y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">feces</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">ZVBQY1N</th>\n",
       "      <th>14</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c66f71a</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J43836_1_ST_T0_B0_0120_ZVBQY1N-6012_ANBEN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c66824d</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J43831_1_ST_T0_B0_0120_ZVBQY1N-1022_ANBEN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <th>d57eb430d669de8329be1769d4f9dec4</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J45529_1_ST_T0_B0_0120_ZVBQY1N-04_APA62.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nasal cavity</th>\n",
       "      <th>ZXSDNAK</th>\n",
       "      <th>1</th>\n",
       "      <th>d57eb430d669de8329be1769d4e8304e</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc345ca3</td>\n",
       "      <td>unknown</td>\n",
       "      <td>ethnic_other</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J35346_1_NS_T0_B0_0120_ZXSDNAK-01_APHU6.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">feces</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">ZK112BX</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>634416dfd8b630dcc6bba6aec4a336ac</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc379db4</td>\n",
       "      <td>female</td>\n",
       "      <td>asian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J00849_1_NS_T0_B0_0120_ZLGD9M0-1014_AAH7B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634416dfd8b630dcc6bba6aec4a336ac</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc379db4</td>\n",
       "      <td>female</td>\n",
       "      <td>asian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>otu_table.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">nasal cavity</th>\n",
       "      <th>ZV1ZA6Z</th>\n",
       "      <th>2</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c65d66e</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34afd9</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J34963_1_NS_T0_B0_0120_ZV1ZA6Z-05_ANAV8.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZIWTAHN</th>\n",
       "      <th>1</th>\n",
       "      <th>634416dfd8b630dcc6bba6aec4afecfc</th>\n",
       "      <td>634416dfd8b630dcc6bba6aec4af9ae8</td>\n",
       "      <td>unknown</td>\n",
       "      <td>ethnic_other</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J34982_1_NS_T0_B0_0120_ZIWTAHN-01_ANAV8.biom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2226 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                 subject_uuid  \\\n",
       "site         patient visit sample                                                               \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "...                                                                                       ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  88af6472fb03642dd5eaf8cddc345ca3   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac  88af6472fb03642dd5eaf8cddc379db4   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac  88af6472fb03642dd5eaf8cddc379db4   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e  88af6472fb03642dd5eaf8cddc34afd9   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  634416dfd8b630dcc6bba6aec4af9ae8   \n",
       "\n",
       "                                                            subject_gender  \\\n",
       "site         patient visit sample                                            \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd           male   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd           male   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a           male   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d           male   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4           male   \n",
       "...                                                                    ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e        unknown   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac         female   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac         female   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e           male   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc        unknown   \n",
       "\n",
       "                                                             subject_race  \\\n",
       "site         patient visit sample                                           \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd     caucasian   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd     caucasian   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a     caucasian   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d     caucasian   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4     caucasian   \n",
       "...                                                                   ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  ethnic_other   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac         asian   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac         asian   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e     caucasian   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  ethnic_other   \n",
       "\n",
       "                                                            study_full_name  \\\n",
       "site         patient visit sample                                             \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd     prediabetes   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd     prediabetes   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a     prediabetes   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d     prediabetes   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4     prediabetes   \n",
       "...                                                                     ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e     prediabetes   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac     prediabetes   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac     prediabetes   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e     prediabetes   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc     prediabetes   \n",
       "\n",
       "                                                                                     project_name  \\\n",
       "site         patient visit sample                                                                   \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd  Integrative Human Microbiome Project   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd  Integrative Human Microbiome Project   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a  Integrative Human Microbiome Project   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d  Integrative Human Microbiome Project   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4  Integrative Human Microbiome Project   \n",
       "...                                                                                           ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  Integrative Human Microbiome Project   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac  Integrative Human Microbiome Project   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac  Integrative Human Microbiome Project   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e  Integrative Human Microbiome Project   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  Integrative Human Microbiome Project   \n",
       "\n",
       "                                                                                                        sample  \n",
       "site         patient visit sample                                                                               \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd  HMP2_J43833_1_NS_T0_B0_0120_ZVBQY1N-1024_ANAV8...  \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd  HMP2_J29034_1_NS_T0_B0_0120_ZVBQY1N-1024_AN77Y...  \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a  HMP2_J43836_1_ST_T0_B0_0120_ZVBQY1N-6012_ANBEN...  \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d  HMP2_J43831_1_ST_T0_B0_0120_ZVBQY1N-1022_ANBEN...  \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4  HMP2_J45529_1_ST_T0_B0_0120_ZVBQY1N-04_APA62.biom  \n",
       "...                                                                                                        ...  \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  HMP2_J35346_1_NS_T0_B0_0120_ZXSDNAK-01_APHU6.biom  \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac  HMP2_J00849_1_NS_T0_B0_0120_ZLGD9M0-1014_AAH7B...  \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac                                     otu_table.biom  \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e  HMP2_J34963_1_NS_T0_B0_0120_ZV1ZA6Z-05_ANAV8.biom  \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  HMP2_J34982_1_NS_T0_B0_0120_ZIWTAHN-01_ANAV8.biom  \n",
       "\n",
       "[2226 rows x 6 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get rich IBD metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ibd_data, ibd_metadata = get_diffs(\n",
    "    \"ibd\", top_n=None, get_abundances=True, include_metadata=True\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/phil/miniconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py:3398: DtypeWarning: Columns (5) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Project</th>\n",
       "      <th>External ID</th>\n",
       "      <th>Participant ID</th>\n",
       "      <th>site_sub_coll</th>\n",
       "      <th>data_type</th>\n",
       "      <th>week_num</th>\n",
       "      <th>date_of_receipt</th>\n",
       "      <th>interval_days</th>\n",
       "      <th>visit_num</th>\n",
       "      <th>Research Project</th>\n",
       "      <th>...</th>\n",
       "      <th>FecalCal received at MGH:</th>\n",
       "      <th>Proteomics received at LBNL:</th>\n",
       "      <th>Stool Sample ID:  Tube A  (EtOH)</th>\n",
       "      <th>Sample ID: Tube B (No preservative)</th>\n",
       "      <th>Tube A and B received at Broad:</th>\n",
       "      <th>stool_id</th>\n",
       "      <th>smoking status</th>\n",
       "      <th>Number years smoked</th>\n",
       "      <th>Age when started smoking</th>\n",
       "      <th>How many cigarettes/cigars/etc. do you smoke per day?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C3001CSC1_BP</td>\n",
       "      <td>206615</td>\n",
       "      <td>C3001</td>\n",
       "      <td>C3001CSC1</td>\n",
       "      <td>biopsy_16S</td>\n",
       "      <td>2.0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C3001CSC2_BP</td>\n",
       "      <td>206614</td>\n",
       "      <td>C3001</td>\n",
       "      <td>C3001CSC2</td>\n",
       "      <td>biopsy_16S</td>\n",
       "      <td>2.0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C3002CSC1_BP</td>\n",
       "      <td>206617</td>\n",
       "      <td>C3002</td>\n",
       "      <td>C3002CSC1</td>\n",
       "      <td>biopsy_16S</td>\n",
       "      <td>0.0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C3002CSC2_BP</td>\n",
       "      <td>206619</td>\n",
       "      <td>C3002</td>\n",
       "      <td>C3002CSC2</td>\n",
       "      <td>biopsy_16S</td>\n",
       "      <td>0.0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C3002CSC3_BP</td>\n",
       "      <td>206616</td>\n",
       "      <td>C3002</td>\n",
       "      <td>C3002CSC3</td>\n",
       "      <td>biopsy_16S</td>\n",
       "      <td>0.0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5528</th>\n",
       "      <td>P6038C7_MVX</td>\n",
       "      <td>PSMA2675</td>\n",
       "      <td>P6038</td>\n",
       "      <td>P6038C7</td>\n",
       "      <td>viromics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-02-17</td>\n",
       "      <td>14.0</td>\n",
       "      <td>11</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>SM-A2675</td>\n",
       "      <td>SM-A2676</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5529</th>\n",
       "      <td>P6038C8_MVX</td>\n",
       "      <td>PSMA269S</td>\n",
       "      <td>P6038</td>\n",
       "      <td>P6038C8</td>\n",
       "      <td>viromics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-03-01</td>\n",
       "      <td>13.0</td>\n",
       "      <td>12</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>SM-A269S</td>\n",
       "      <td>SM-A269T</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5530</th>\n",
       "      <td>P6038C10_MVX</td>\n",
       "      <td>PSMA269W</td>\n",
       "      <td>P6038</td>\n",
       "      <td>P6038C10</td>\n",
       "      <td>viromics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-04-01</td>\n",
       "      <td>11.0</td>\n",
       "      <td>14</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>SM-A269W</td>\n",
       "      <td>SM-A269X</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5531</th>\n",
       "      <td>P6038C13_MVX</td>\n",
       "      <td>PSMA26A3</td>\n",
       "      <td>P6038</td>\n",
       "      <td>P6038C13</td>\n",
       "      <td>viromics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>18</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>SM-A26A3</td>\n",
       "      <td>SM-A26A4</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5532</th>\n",
       "      <td>P6038C19_MVX</td>\n",
       "      <td>PSMB4MC5</td>\n",
       "      <td>P6038</td>\n",
       "      <td>P6038C19</td>\n",
       "      <td>viromics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-08-03</td>\n",
       "      <td>14.0</td>\n",
       "      <td>25</td>\n",
       "      <td>ibdmdb</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>SM-B4MC5</td>\n",
       "      <td>SM-B4MC6</td>\n",
       "      <td>No</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5533 rows × 490 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Project External ID Participant ID site_sub_coll   data_type  \\\n",
       "0     C3001CSC1_BP      206615          C3001     C3001CSC1  biopsy_16S   \n",
       "1     C3001CSC2_BP      206614          C3001     C3001CSC2  biopsy_16S   \n",
       "2     C3002CSC1_BP      206617          C3002     C3002CSC1  biopsy_16S   \n",
       "3     C3002CSC2_BP      206619          C3002     C3002CSC2  biopsy_16S   \n",
       "4     C3002CSC3_BP      206616          C3002     C3002CSC3  biopsy_16S   \n",
       "...            ...         ...            ...           ...         ...   \n",
       "5528   P6038C7_MVX    PSMA2675          P6038       P6038C7    viromics   \n",
       "5529   P6038C8_MVX    PSMA269S          P6038       P6038C8    viromics   \n",
       "5530  P6038C10_MVX    PSMA269W          P6038      P6038C10    viromics   \n",
       "5531  P6038C13_MVX    PSMA26A3          P6038      P6038C13    viromics   \n",
       "5532  P6038C19_MVX    PSMB4MC5          P6038      P6038C19    viromics   \n",
       "\n",
       "     week_num date_of_receipt interval_days  visit_num Research Project  ...  \\\n",
       "0         2.0                                        1           ibdmdb  ...   \n",
       "1         2.0                                        1           ibdmdb  ...   \n",
       "2         0.0                                        1           ibdmdb  ...   \n",
       "3         0.0                                        1           ibdmdb  ...   \n",
       "4         0.0                                        1           ibdmdb  ...   \n",
       "...       ...             ...           ...        ...              ...  ...   \n",
       "5528      NaN      2016-02-17          14.0         11           ibdmdb  ...   \n",
       "5529      NaN      2016-03-01          13.0         12           ibdmdb  ...   \n",
       "5530      NaN      2016-04-01          11.0         14           ibdmdb  ...   \n",
       "5531      NaN      2016-05-12          14.0         18           ibdmdb  ...   \n",
       "5532      NaN      2016-08-03          14.0         25           ibdmdb  ...   \n",
       "\n",
       "     FecalCal received at MGH: Proteomics received at LBNL:  \\\n",
       "0                           No                           No   \n",
       "1                           No                           No   \n",
       "2                           No                           No   \n",
       "3                           No                           No   \n",
       "4                           No                           No   \n",
       "...                        ...                          ...   \n",
       "5528                        No                           No   \n",
       "5529                        No                           No   \n",
       "5530                        No                           No   \n",
       "5531                        No                           No   \n",
       "5532                        No                           No   \n",
       "\n",
       "     Stool Sample ID:  Tube A  (EtOH) Sample ID: Tube B (No preservative)  \\\n",
       "0                                                                           \n",
       "1                                                                           \n",
       "2                                                                           \n",
       "3                                                                           \n",
       "4                                                                           \n",
       "...                               ...                                 ...   \n",
       "5528                         SM-A2675                            SM-A2676   \n",
       "5529                         SM-A269S                            SM-A269T   \n",
       "5530                         SM-A269W                            SM-A269X   \n",
       "5531                         SM-A26A3                            SM-A26A4   \n",
       "5532                         SM-B4MC5                            SM-B4MC6   \n",
       "\n",
       "     Tube A and B received at Broad: stool_id smoking status  \\\n",
       "0                                 No                           \n",
       "1                                 No                           \n",
       "2                                 No                           \n",
       "3                                 No                           \n",
       "4                                 No                           \n",
       "...                              ...      ...            ...   \n",
       "5528                              No                           \n",
       "5529                              No                           \n",
       "5530                              No                           \n",
       "5531                              No                           \n",
       "5532                              No                           \n",
       "\n",
       "     Number years smoked Age when started smoking  \\\n",
       "0                                                   \n",
       "1                                                   \n",
       "2                                                   \n",
       "3                                                   \n",
       "4                                                   \n",
       "...                  ...                      ...   \n",
       "5528                                                \n",
       "5529                                                \n",
       "5530                                                \n",
       "5531                                                \n",
       "5532                                                \n",
       "\n",
       "     How many cigarettes/cigars/etc. do you smoke per day?  \n",
       "0                                                   NaN     \n",
       "1                                                   NaN     \n",
       "2                                                   NaN     \n",
       "3                                                   NaN     \n",
       "4                                                   NaN     \n",
       "...                                                 ...     \n",
       "5528                                                NaN     \n",
       "5529                                                NaN     \n",
       "5530                                                NaN     \n",
       "5531                                                NaN     \n",
       "5532                                                NaN     \n",
       "\n",
       "[5533 rows x 490 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# metadata2 = pd.read_table(\"/home/phil/DATA/ihmp/IBD/hmp2_ibd_metadata.csv\")\n",
    "\n",
    "# Read table ignoring trailing whitespace\n",
    "ibd_metadata2 = pd.read_csv(\"/home/phil/DATA/ihmp/IBD/hmp2_ibd_metadata.csv\")\n",
    "\n",
    "# Strip whitespace from column names\n",
    "ibd_metadata2.columns = ibd_metadata2.columns.str.strip()\n",
    "\n",
    "# Values\n",
    "for col in ibd_metadata2.columns:\n",
    "    if ibd_metadata2[col].dtype == \"object\":\n",
    "        ibd_metadata2[col] = ibd_metadata2[col].str.strip()\n",
    "\n",
    "ibd_metadata2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'metadata2' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/home/phil/phylosig/06_export_ihmp_data.ipynb Cell 7\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m ibd_merged \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mmerge(\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m     ibd_data\u001b[39m.\u001b[39mreset_index()\u001b[39m.\u001b[39mset_index([\u001b[39m\"\u001b[39m\u001b[39msite\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mpatient\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mvisit\u001b[39m\u001b[39m\"\u001b[39m]),\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m     metadata2[metadata2[\u001b[39m\"\u001b[39m\u001b[39mdata_type\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mstool_16S\u001b[39m\u001b[39m\"\u001b[39m],\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m     left_on\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39msample\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=4'>5</a>\u001b[0m     right_on\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mExternal ID\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=5'>6</a>\u001b[0m     how\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39minner\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=6'>7</a>\u001b[0m )\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m ibd_merged \u001b[39m=\u001b[39m ibd_merged\u001b[39m.\u001b[39mset_index(\u001b[39m\"\u001b[39m\u001b[39msample\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m ibd_merged\n",
      "\u001b[0;31mNameError\u001b[0m: name 'metadata2' is not defined"
     ]
    }
   ],
   "source": [
    "ibd_merged = pd.merge(\n",
    "    ibd_data.reset_index().set_index([\"site\", \"patient\", \"visit\"]),\n",
    "    metadata2[metadata2[\"data_type\"] == \"stool_16S\"],\n",
    "    left_on=\"sample\",\n",
    "    right_on=\"External ID\",\n",
    "    how=\"inner\",\n",
    ")\n",
    "ibd_merged = ibd_merged.set_index(\"sample\")\n",
    "\n",
    "ibd_merged\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ibd_otus = ibd_merged.iloc[:, :1370]\n",
    "ibd_otus\n",
    "\n",
    "ibd_metadata = ibd_merged.iloc[:, 1370:]\n",
    "\n",
    "# ibd_otus\n",
    "# ibd_metadata\n",
    "\n",
    "ibd_otus.to_csv(\"ihmp/ibd_data.csv.gz\", compression=\"gzip\")\n",
    "ibd_metadata.to_csv(\"ihmp/ibd_metadata.csv.gz\", compression=\"gzip\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get rich T2D metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t2d_data, t2d_metadata = get_diffs(\n",
    "    \"t2d\", top_n=None, get_abundances=True, include_metadata=True\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t2d_subject_metadata = pd.read_table(\n",
    "    \"/home/phil/DATA/ihmp/T2D/t2d_subject_metadata.tsv\"\n",
    ")\n",
    "\n",
    "# Strip whitespace from column names\n",
    "t2d_subject_metadata.columns = t2d_subject_metadata.columns.str.strip()\n",
    "\n",
    "# Strip whitespace from values\n",
    "for col in t2d_subject_metadata.columns:\n",
    "    if t2d_subject_metadata[col].dtype == \"object\":\n",
    "        t2d_subject_metadata[col] = t2d_subject_metadata[col].str.strip()\n",
    "\n",
    "t2d_metadata_with_subjects = pd.merge(\n",
    "    t2d_metadata,\n",
    "    t2d_subject_metadata,\n",
    "    left_on=\"subject_id\",\n",
    "    right_on=\"SubjectID\",\n",
    ")\n",
    "\n",
    "t2d_metadata_with_subjects.to_csv(\n",
    "    \"ihmp/t2d_metadata.csv.gz\", compression=\"gzip\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "# Make df with VisitID column for merge\n",
    "t2d_data2 = t2d_data.reset_index().set_index([\"site\", \"patient\", \"visit\"])\n",
    "\n",
    "\n",
    "def get_visit_id(x):\n",
    "    try:\n",
    "        visit_id = x.split(\"_\")[7]\n",
    "    except IndexError:\n",
    "        visit_id = None\n",
    "    return visit_id\n",
    "\n",
    "\n",
    "t2d_data2[\"VisitID\"] = t2d_data2[\"sample\"].apply(get_visit_id)\n",
    "t2d_data2.drop(\"sample\", axis=1, inplace=True)\n",
    "t2d_data2 = t2d_data2.drop_duplicates()\n",
    "\n",
    "# Merge\n",
    "t2d_merged = pd.merge(t2d_data2, t2d_metadata2, on=\"VisitID\")\n",
    "t2d_merged = t2d_merged.set_index(\"VisitID\")\n",
    "\n",
    "# Split into OTU and metadata\n",
    "t2d_otus = t2d_merged.iloc[:, : t2d_data2.shape[1] - 1]\n",
    "t2d_metadata = t2d_merged.iloc[:, t2d_data2.shape[1] - 1 :]\n",
    "\n",
    "# Export\n",
    "t2d_otus.to_csv(\"ihmp/t2d_data.csv.gz\", compression=\"gzip\")\n",
    "t2d_metadata.to_csv(\"ihmp/t2d_metadata.csv.gz\", compression=\"gzip\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 't2d_otus' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/home/phil/phylosig/06_export_ihmp_data.ipynb Cell 13\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#X21sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m t2d_otus\n",
      "\u001b[0;31mNameError\u001b[0m: name 't2d_otus' is not defined"
     ]
    }
   ],
   "source": [
    "t2d_otus\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export clustered OTU tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/phil/phylosig/06_export_ihmp_data.ipynb Cell 14\u001b[0m in \u001b[0;36m<cell line: 10>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#X15sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m get_ipython()\u001b[39m.\u001b[39msystem(\u001b[39m'\u001b[39m\u001b[39m mkdir -p ihmp/clustered/moms\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#X15sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39mfor\u001b[39;00m name \u001b[39min\u001b[39;00m [\u001b[39m\"\u001b[39m\u001b[39mibd\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mt2d\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mmoms\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#X15sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m     data \u001b[39m=\u001b[39m get_diffs(name, get_abundances\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, log\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#X15sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m     data\u001b[39m.\u001b[39mto_csv(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mihmp/clustered/\u001b[39m\u001b[39m{\u001b[39;00mname\u001b[39m}\u001b[39;00m\u001b[39m/otus_99.csv\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Beumaeus/home/phil/phylosig/06_export_ihmp_data.ipynb#X15sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m     similarity_map99 \u001b[39m=\u001b[39m parse_similarity_map(\u001b[39m99\u001b[39m)\n",
      "File \u001b[0;32m~/phylosig/src/ihmp.py:44\u001b[0m, in \u001b[0;36mget_diffs\u001b[0;34m(dataset, top_n, get_abundances, include_metadata, log, normalize)\u001b[0m\n\u001b[1;32m     41\u001b[0m manifest_path \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mprefix\u001b[39m}\u001b[39;00m\u001b[39m/16s_manifest.tsv\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     43\u001b[0m \u001b[39m# Load data\u001b[39;00m\n\u001b[0;32m---> 44\u001b[0m otus \u001b[39m=\u001b[39m strip_df(pd\u001b[39m.\u001b[39;49mread_csv(otu_path, dtype\u001b[39m=\u001b[39;49m{\u001b[39m0\u001b[39;49m: \u001b[39mstr\u001b[39;49m}))\n\u001b[1;32m     45\u001b[0m otus \u001b[39m=\u001b[39m otus\u001b[39m.\u001b[39mset_index(otus\u001b[39m.\u001b[39mcolumns[\u001b[39m0\u001b[39m])\u001b[39m.\u001b[39mT\n\u001b[1;32m     46\u001b[0m metadata \u001b[39m=\u001b[39m strip_df(pd\u001b[39m.\u001b[39mread_table(metadata_path, skipinitialspace\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m))\n",
      "File \u001b[0;32m~/phylosig/src/ihmp.py:16\u001b[0m, in \u001b[0;36mstrip_df\u001b[0;34m(df)\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[39m# Values\u001b[39;00m\n\u001b[1;32m     15\u001b[0m \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m df\u001b[39m.\u001b[39mcolumns:\n\u001b[0;32m---> 16\u001b[0m     \u001b[39mif\u001b[39;00m df[col]\u001b[39m.\u001b[39mdtype \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mobject\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m     17\u001b[0m         df[col] \u001b[39m=\u001b[39m df[col]\u001b[39m.\u001b[39mstr\u001b[39m.\u001b[39mstrip()\n\u001b[1;32m     18\u001b[0m \u001b[39mreturn\u001b[39;00m df\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.9/site-packages/pandas/core/frame.py:3427\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3425\u001b[0m         \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcolumns, MultiIndex):\n\u001b[1;32m   3426\u001b[0m             \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getitem_multilevel(key)\n\u001b[0;32m-> 3427\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get_item_cache(key)\n\u001b[1;32m   3429\u001b[0m \u001b[39m# Do we have a slicer (on rows)?\u001b[39;00m\n\u001b[1;32m   3430\u001b[0m indexer \u001b[39m=\u001b[39m convert_to_index_sliceable(\u001b[39mself\u001b[39m, key)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.9/site-packages/pandas/core/frame.py:3881\u001b[0m, in \u001b[0;36mDataFrame._get_item_cache\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m   3879\u001b[0m loc \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcolumns\u001b[39m.\u001b[39mget_loc(item)\n\u001b[1;32m   3880\u001b[0m values \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mgr\u001b[39m.\u001b[39miget(loc)\n\u001b[0;32m-> 3881\u001b[0m res \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_box_col_values(values, loc)\u001b[39m.\u001b[39m__finalize__(\u001b[39mself\u001b[39m)\n\u001b[1;32m   3883\u001b[0m cache[item] \u001b[39m=\u001b[39m res\n\u001b[1;32m   3884\u001b[0m res\u001b[39m.\u001b[39m_set_as_cached(item, \u001b[39mself\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.9/site-packages/pandas/core/frame.py:3863\u001b[0m, in \u001b[0;36mDataFrame._box_col_values\u001b[0;34m(self, values, loc)\u001b[0m\n\u001b[1;32m   3861\u001b[0m name \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcolumns[loc]\n\u001b[1;32m   3862\u001b[0m klass \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_constructor_sliced\n\u001b[0;32m-> 3863\u001b[0m \u001b[39mreturn\u001b[39;00m klass(values, index\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mindex, name\u001b[39m=\u001b[39;49mname, fastpath\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.9/site-packages/pandas/core/series.py:447\u001b[0m, in \u001b[0;36mSeries.__init__\u001b[0;34m(self, data, index, dtype, name, copy, fastpath)\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[39melif\u001b[39;00m manager \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39marray\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m    445\u001b[0m             data \u001b[39m=\u001b[39m SingleArrayManager\u001b[39m.\u001b[39mfrom_array(data, index)\n\u001b[0;32m--> 447\u001b[0m generic\u001b[39m.\u001b[39;49mNDFrame\u001b[39m.\u001b[39;49m\u001b[39m__init__\u001b[39;49m(\u001b[39mself\u001b[39;49m, data)\n\u001b[1;32m    448\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mname \u001b[39m=\u001b[39m name\n\u001b[1;32m    449\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_set_axis(\u001b[39m0\u001b[39m, index, fastpath\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:240\u001b[0m, in \u001b[0;36mNDFrame.__init__\u001b[0;34m(self, data, copy, attrs)\u001b[0m\n\u001b[1;32m    238\u001b[0m \u001b[39mobject\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__setattr__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m_is_copy\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m    239\u001b[0m \u001b[39mobject\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__setattr__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m_mgr\u001b[39m\u001b[39m\"\u001b[39m, data)\n\u001b[0;32m--> 240\u001b[0m \u001b[39mobject\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__setattr__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m_item_cache\u001b[39m\u001b[39m\"\u001b[39m, {})\n\u001b[1;32m    241\u001b[0m \u001b[39mif\u001b[39;00m attrs \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    242\u001b[0m     attrs \u001b[39m=\u001b[39m {}\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# from src.greengenes import (\n",
    "#     parse_similarity_map, merge_otu_table, combine_similarity_maps\n",
    "# )\n",
    "# from src.ihmp import get_diffs\n",
    "\n",
    "# ! mkdir -p ihmp/clustered/ibd\n",
    "# ! mkdir -p ihmp/clustered/t2d\n",
    "# ! mkdir -p ihmp/clustered/moms\n",
    "\n",
    "# for name in [\"ibd\", \"t2d\", \"moms\"]:\n",
    "#     data = get_diffs(name, get_abundances=True, log=False)\n",
    "#     data.to_csv(f\"ihmp/clustered/{name}/otus_99.csv\")\n",
    "\n",
    "#     similarity_map99 = parse_similarity_map(99)\n",
    "#     similarity_map = similarity_map99\n",
    "\n",
    "#     for x in [99]:# 97, 94, 91, 88, 85, 82, 79, 76, 73, 70, 67, 64, 61]:\n",
    "#         similarity_map = combine_similarity_maps(\n",
    "#             [similarity_map, parse_similarity_map(x)]\n",
    "#         )\n",
    "#         clustered = merge_otu_table(data, similarity_map)\n",
    "#         s = clustered.sum(axis=1)\n",
    "#         print(f\"{x} {clustered.shape} {s.min():.3f} {s.mean():.3f} {s.max():.3f}\")\n",
    "\n",
    "#         clustered.to_csv(f\"ihmp/clustered/{name}/otus_{x}.csv\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final attempt to make sense of iHMP metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107 subjects\n",
      "1416 visits\n"
     ]
    }
   ],
   "source": [
    "t2d_subject = pd.read_table(\"ihmp/metadata/t2d_subject.tsv\")\n",
    "t2d_visit = pd.read_table(\"ihmp/metadata/t2d_visit.tsv\")\n",
    "\n",
    "print(len(t2d_subject), \"subjects\")\n",
    "print(len(t2d_visit), \"visits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1057 stool samples\n",
      "1169 nasal samples\n"
     ]
    }
   ],
   "source": [
    "# Site is 'nasal cavity' or 'feces' - first level of multiindex\n",
    "\n",
    "t2d_ihmp_metadata = metadata\n",
    "# Rename 'sample' column to 'filename'\n",
    "t2d_ihmp_metadata = t2d_ihmp_metadata.rename(columns={\"sample\": \"filename\"})\n",
    "t2d_ihmp_metadata = t2d_ihmp_metadata.reset_index()\n",
    "\n",
    "t2d_ihmp_stool = t2d_ihmp_metadata[t2d_ihmp_metadata[\"site\"] == \"feces\"]\n",
    "t2d_ihmp_nasal = t2d_ihmp_metadata[t2d_ihmp_metadata[\"site\"] == \"nasal cavity\"]\n",
    "\n",
    "print(len(t2d_ihmp_stool), \"stool samples\")\n",
    "print(len(t2d_ihmp_nasal), \"nasal samples\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "884 stool IDs\n",
      "828 nasal IDs\n",
      "1037 total IDs\n"
     ]
    }
   ],
   "source": [
    "# Great, now we need to recocile visit IDs with filenames\n",
    "\n",
    "stool_ids = [(row[\"patient\"], row[\"visit\"]) for i, row in t2d_ihmp_stool.iterrows()]\n",
    "nasal_ids = [(row[\"patient\"], row[\"visit\"]) for i, row in t2d_ihmp_nasal.iterrows()]\n",
    "\n",
    "stool_ids = set(stool_ids)\n",
    "nasal_ids = set(nasal_ids)\n",
    "\n",
    "print(len(stool_ids), \"stool IDs\")\n",
    "print(len(nasal_ids), \"nasal IDs\")\n",
    "print(len(stool_ids.union(nasal_ids)), \"total IDs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>subject_uuid</th>\n",
       "      <th>subject_gender</th>\n",
       "      <th>subject_race</th>\n",
       "      <th>study_full_name</th>\n",
       "      <th>project_name</th>\n",
       "      <th>sample</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>site</th>\n",
       "      <th>patient</th>\n",
       "      <th>visit</th>\n",
       "      <th>sample</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">nasal cavity</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">ZVBQY1N</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">11</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c669dfd</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J43833_1_NS_T0_B0_0120_ZVBQY1N-1024_ANAV8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>932d8fbc70ae8f856028b3f67c669dfd</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J29034_1_NS_T0_B0_0120_ZVBQY1N-1024_AN77Y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">feces</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">ZVBQY1N</th>\n",
       "      <th>14</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c66f71a</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J43836_1_ST_T0_B0_0120_ZVBQY1N-6012_ANBEN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c66824d</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J43831_1_ST_T0_B0_0120_ZVBQY1N-1022_ANBEN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <th>d57eb430d669de8329be1769d4f9dec4</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34ae9e</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J45529_1_ST_T0_B0_0120_ZVBQY1N-04_APA62.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nasal cavity</th>\n",
       "      <th>ZXSDNAK</th>\n",
       "      <th>1</th>\n",
       "      <th>d57eb430d669de8329be1769d4e8304e</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc345ca3</td>\n",
       "      <td>unknown</td>\n",
       "      <td>ethnic_other</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J35346_1_NS_T0_B0_0120_ZXSDNAK-01_APHU6.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">feces</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">ZK112BX</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>634416dfd8b630dcc6bba6aec4a336ac</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc379db4</td>\n",
       "      <td>female</td>\n",
       "      <td>asian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J00849_1_NS_T0_B0_0120_ZLGD9M0-1014_AAH7B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634416dfd8b630dcc6bba6aec4a336ac</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc379db4</td>\n",
       "      <td>female</td>\n",
       "      <td>asian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>otu_table.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">nasal cavity</th>\n",
       "      <th>ZV1ZA6Z</th>\n",
       "      <th>2</th>\n",
       "      <th>932d8fbc70ae8f856028b3f67c65d66e</th>\n",
       "      <td>88af6472fb03642dd5eaf8cddc34afd9</td>\n",
       "      <td>male</td>\n",
       "      <td>caucasian</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J34963_1_NS_T0_B0_0120_ZV1ZA6Z-05_ANAV8.biom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZIWTAHN</th>\n",
       "      <th>1</th>\n",
       "      <th>634416dfd8b630dcc6bba6aec4afecfc</th>\n",
       "      <td>634416dfd8b630dcc6bba6aec4af9ae8</td>\n",
       "      <td>unknown</td>\n",
       "      <td>ethnic_other</td>\n",
       "      <td>prediabetes</td>\n",
       "      <td>Integrative Human Microbiome Project</td>\n",
       "      <td>HMP2_J34982_1_NS_T0_B0_0120_ZIWTAHN-01_ANAV8.biom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2226 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                 subject_uuid  \\\n",
       "site         patient visit sample                                                               \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4  88af6472fb03642dd5eaf8cddc34ae9e   \n",
       "...                                                                                       ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  88af6472fb03642dd5eaf8cddc345ca3   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac  88af6472fb03642dd5eaf8cddc379db4   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac  88af6472fb03642dd5eaf8cddc379db4   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e  88af6472fb03642dd5eaf8cddc34afd9   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  634416dfd8b630dcc6bba6aec4af9ae8   \n",
       "\n",
       "                                                            subject_gender  \\\n",
       "site         patient visit sample                                            \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd           male   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd           male   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a           male   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d           male   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4           male   \n",
       "...                                                                    ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e        unknown   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac         female   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac         female   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e           male   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc        unknown   \n",
       "\n",
       "                                                             subject_race  \\\n",
       "site         patient visit sample                                           \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd     caucasian   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd     caucasian   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a     caucasian   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d     caucasian   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4     caucasian   \n",
       "...                                                                   ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  ethnic_other   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac         asian   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac         asian   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e     caucasian   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  ethnic_other   \n",
       "\n",
       "                                                            study_full_name  \\\n",
       "site         patient visit sample                                             \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd     prediabetes   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd     prediabetes   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a     prediabetes   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d     prediabetes   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4     prediabetes   \n",
       "...                                                                     ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e     prediabetes   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac     prediabetes   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac     prediabetes   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e     prediabetes   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc     prediabetes   \n",
       "\n",
       "                                                                                     project_name  \\\n",
       "site         patient visit sample                                                                   \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd  Integrative Human Microbiome Project   \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd  Integrative Human Microbiome Project   \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a  Integrative Human Microbiome Project   \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d  Integrative Human Microbiome Project   \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4  Integrative Human Microbiome Project   \n",
       "...                                                                                           ...   \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  Integrative Human Microbiome Project   \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac  Integrative Human Microbiome Project   \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac  Integrative Human Microbiome Project   \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e  Integrative Human Microbiome Project   \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  Integrative Human Microbiome Project   \n",
       "\n",
       "                                                                                                        sample  \n",
       "site         patient visit sample                                                                               \n",
       "nasal cavity ZVBQY1N 11    932d8fbc70ae8f856028b3f67c669dfd  HMP2_J43833_1_NS_T0_B0_0120_ZVBQY1N-1024_ANAV8...  \n",
       "                           932d8fbc70ae8f856028b3f67c669dfd  HMP2_J29034_1_NS_T0_B0_0120_ZVBQY1N-1024_AN77Y...  \n",
       "feces        ZVBQY1N 14    932d8fbc70ae8f856028b3f67c66f71a  HMP2_J43836_1_ST_T0_B0_0120_ZVBQY1N-6012_ANBEN...  \n",
       "                     9     932d8fbc70ae8f856028b3f67c66824d  HMP2_J43831_1_ST_T0_B0_0120_ZVBQY1N-1022_ANBEN...  \n",
       "                     7     d57eb430d669de8329be1769d4f9dec4  HMP2_J45529_1_ST_T0_B0_0120_ZVBQY1N-04_APA62.biom  \n",
       "...                                                                                                        ...  \n",
       "nasal cavity ZXSDNAK 1     d57eb430d669de8329be1769d4e8304e  HMP2_J35346_1_NS_T0_B0_0120_ZXSDNAK-01_APHU6.biom  \n",
       "feces        ZK112BX 2     634416dfd8b630dcc6bba6aec4a336ac  HMP2_J00849_1_NS_T0_B0_0120_ZLGD9M0-1014_AAH7B...  \n",
       "                           634416dfd8b630dcc6bba6aec4a336ac                                     otu_table.biom  \n",
       "nasal cavity ZV1ZA6Z 2     932d8fbc70ae8f856028b3f67c65d66e  HMP2_J34963_1_NS_T0_B0_0120_ZV1ZA6Z-05_ANAV8.biom  \n",
       "             ZIWTAHN 1     634416dfd8b630dcc6bba6aec4afecfc  HMP2_J34982_1_NS_T0_B0_0120_ZIWTAHN-01_ANAV8.biom  \n",
       "\n",
       "[2226 rows x 6 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "site   patient  visit  sample                                             \n",
       "feces  ZN9YTFN  1      HMP2_J00825_1_ST_T0_B0_0120_ZN9YTFN-01_AA31J.biom      1.0\n",
       "                2      HMP2_J00826_1_ST_T0_B0_0120_ZN9YTFN-1011_AA31J.biom    1.0\n",
       "                3      HMP2_J00827_1_ST_T0_B0_0120_ZN9YTFN-1012_AA31J.biom    1.0\n",
       "                4      HMP2_J00828_1_ST_T0_B0_0120_ZN9YTFN-1013_AA31J.biom    1.0\n",
       "                5      HMP2_J00829_1_ST_T0_B0_0120_ZN9YTFN-1014_AA31J.biom    1.0\n",
       "                                                                             ... \n",
       "       ZLZNCLZ  12     HMP2_J83832_1_ST_T0_B0_0120_ZLZNCLZ-12_B86HB.biom      1.0\n",
       "       ZYHHR4Z  8025   HMP2_J83833_1_ST_T0_B0_0120_ZYHHR4Z-8025_B86HB.biom    1.0\n",
       "       ZVBQY1N  999    HMP2_J83836_1_ST_T0_B0_0120_ZVBQY1N-1012_B86HB.biom    1.0\n",
       "       ZUF48YS  999    HMP2_J83837_1_ST_T0_B0_0120_ZUF48YS-05_B86HB.biom      1.0\n",
       "       ZPEL6L3  1014   HMP2_J83838_1_ST_T0_B0_0120_ZPEL6L3-1014_B86HB.biom    1.0\n",
       "Length: 2204, dtype: float64"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VisitID</th>\n",
       "      <th>SubjectID</th>\n",
       "      <th>CollectionDate</th>\n",
       "      <th>Event</th>\n",
       "      <th>Event_Note1</th>\n",
       "      <th>Event_Note2</th>\n",
       "      <th>Event_Note3</th>\n",
       "      <th>SubStudy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ZIS22OE-02</td>\n",
       "      <td>ZIS22OE</td>\n",
       "      <td>132.0</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZJBOZ2E11-E11</td>\n",
       "      <td>ZJBOZ2X</td>\n",
       "      <td>1481.0</td>\n",
       "      <td>Exercise</td>\n",
       "      <td>Baseline</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ZJBOZ2E12-E12</td>\n",
       "      <td>ZJBOZ2X</td>\n",
       "      <td>1481.0</td>\n",
       "      <td>Exercise</td>\n",
       "      <td>2 min</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ZJBOZ2E13-E13</td>\n",
       "      <td>ZJBOZ2X</td>\n",
       "      <td>1481.0</td>\n",
       "      <td>Exercise</td>\n",
       "      <td>15 min</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ZJBOZ2E14-E14</td>\n",
       "      <td>ZJBOZ2X</td>\n",
       "      <td>1481.0</td>\n",
       "      <td>Exercise</td>\n",
       "      <td>30 min</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1411</th>\n",
       "      <td>ZYXQKWY-05</td>\n",
       "      <td>ZYXQKWY</td>\n",
       "      <td>957.0</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1412</th>\n",
       "      <td>ZYXQKWY-06</td>\n",
       "      <td>ZYXQKWY</td>\n",
       "      <td>1062.0</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413</th>\n",
       "      <td>ZYXQKWY-07</td>\n",
       "      <td>ZYXQKWY</td>\n",
       "      <td>1153.0</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1414</th>\n",
       "      <td>ZYXQKWY-08</td>\n",
       "      <td>ZYXQKWY</td>\n",
       "      <td>1271.0</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1415</th>\n",
       "      <td>ZYXQKWY-09</td>\n",
       "      <td>ZYXQKWY</td>\n",
       "      <td>1375.0</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HMP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1416 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            VisitID SubjectID  CollectionDate     Event Event_Note1  \\\n",
       "0        ZIS22OE-02   ZIS22OE           132.0   Healthy         NaN   \n",
       "1     ZJBOZ2E11-E11   ZJBOZ2X          1481.0  Exercise    Baseline   \n",
       "2     ZJBOZ2E12-E12   ZJBOZ2X          1481.0  Exercise       2 min   \n",
       "3     ZJBOZ2E13-E13   ZJBOZ2X          1481.0  Exercise      15 min   \n",
       "4     ZJBOZ2E14-E14   ZJBOZ2X          1481.0  Exercise      30 min   \n",
       "...             ...       ...             ...       ...         ...   \n",
       "1411     ZYXQKWY-05   ZYXQKWY           957.0   Healthy         NaN   \n",
       "1412     ZYXQKWY-06   ZYXQKWY          1062.0   Healthy         NaN   \n",
       "1413     ZYXQKWY-07   ZYXQKWY          1153.0   Healthy         NaN   \n",
       "1414     ZYXQKWY-08   ZYXQKWY          1271.0   Healthy         NaN   \n",
       "1415     ZYXQKWY-09   ZYXQKWY          1375.0   Healthy         NaN   \n",
       "\n",
       "     Event_Note2 Event_Note3  SubStudy  \n",
       "0            NaN         NaN       HMP  \n",
       "1            NaN         NaN  Exercise  \n",
       "2            NaN         NaN  Exercise  \n",
       "3            NaN         NaN  Exercise  \n",
       "4            NaN         NaN  Exercise  \n",
       "...          ...         ...       ...  \n",
       "1411         NaN         NaN       HMP  \n",
       "1412         NaN         NaN       HMP  \n",
       "1413         NaN         NaN       HMP  \n",
       "1414         NaN         NaN       HMP  \n",
       "1415         NaN         NaN       HMP  \n",
       "\n",
       "[1416 rows x 8 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2d_visit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubjectID</th>\n",
       "      <th>Study</th>\n",
       "      <th>Race</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>BMI</th>\n",
       "      <th>SSPG</th>\n",
       "      <th>IR_IS_classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ZIS22OE</td>\n",
       "      <td>HMP</td>\n",
       "      <td>C</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZJBOZ2X</td>\n",
       "      <td>Exercise</td>\n",
       "      <td>C</td>\n",
       "      <td>F</td>\n",
       "      <td>65.30</td>\n",
       "      <td>19.82</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ZJOSZHK</td>\n",
       "      <td>HMP</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>41.43</td>\n",
       "      <td>19.42</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ZJTKAE3</td>\n",
       "      <td>HMP</td>\n",
       "      <td>C</td>\n",
       "      <td>F</td>\n",
       "      <td>58.65</td>\n",
       "      <td>31.24</td>\n",
       "      <td>162.0</td>\n",
       "      <td>IR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ZJXC41N</td>\n",
       "      <td>HMP, Exercise</td>\n",
       "      <td>B</td>\n",
       "      <td>F</td>\n",
       "      <td>49.69</td>\n",
       "      <td>28.24</td>\n",
       "      <td>75.0</td>\n",
       "      <td>IS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>ZYHHR4Z</td>\n",
       "      <td>HMP, Exercise</td>\n",
       "      <td>C</td>\n",
       "      <td>F</td>\n",
       "      <td>43.99</td>\n",
       "      <td>26.47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>ZYKZ9HQ</td>\n",
       "      <td>HMP</td>\n",
       "      <td>unknown</td>\n",
       "      <td>F</td>\n",
       "      <td>55.62</td>\n",
       "      <td>29.94</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>ZYLJ7R7</td>\n",
       "      <td>HMP</td>\n",
       "      <td>A</td>\n",
       "      <td>F</td>\n",
       "      <td>40.97</td>\n",
       "      <td>23.45</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>ZYU89T9</td>\n",
       "      <td>HMP</td>\n",
       "      <td>C</td>\n",
       "      <td>F</td>\n",
       "      <td>60.19</td>\n",
       "      <td>29.83</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>ZYXQKWY</td>\n",
       "      <td>HMP</td>\n",
       "      <td>C</td>\n",
       "      <td>F</td>\n",
       "      <td>50.70</td>\n",
       "      <td>32.59</td>\n",
       "      <td>158.5</td>\n",
       "      <td>IR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>107 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    SubjectID          Study     Race Sex    Age    BMI   SSPG  \\\n",
       "0     ZIS22OE            HMP        C   F    NaN    NaN    NaN   \n",
       "1     ZJBOZ2X       Exercise        C   F  65.30  19.82    NaN   \n",
       "2     ZJOSZHK            HMP        C   M  41.43  19.42    NaN   \n",
       "3     ZJTKAE3            HMP        C   F  58.65  31.24  162.0   \n",
       "4     ZJXC41N  HMP, Exercise        B   F  49.69  28.24   75.0   \n",
       "..        ...            ...      ...  ..    ...    ...    ...   \n",
       "102   ZYHHR4Z  HMP, Exercise        C   F  43.99  26.47    NaN   \n",
       "103   ZYKZ9HQ            HMP  unknown   F  55.62  29.94    NaN   \n",
       "104   ZYLJ7R7            HMP        A   F  40.97  23.45    NaN   \n",
       "105   ZYU89T9            HMP        C   F  60.19  29.83    NaN   \n",
       "106   ZYXQKWY            HMP        C   F  50.70  32.59  158.5   \n",
       "\n",
       "    IR_IS_classification  \n",
       "0                Unknown  \n",
       "1                Unknown  \n",
       "2                Unknown  \n",
       "3                     IR  \n",
       "4                     IS  \n",
       "..                   ...  \n",
       "102              Unknown  \n",
       "103              Unknown  \n",
       "104              Unknown  \n",
       "105              Unknown  \n",
       "106                   IR  \n",
       "\n",
       "[107 rows x 8 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2d_subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e4dc294415b14ea4c0d4d07894c031d87e01c25f832e475b95612eada6e667d4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}