10.gene-expression-signatures/1.separate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Detect Separation Between Two Phenotypes in TCGA, GTEx, and TARGET Data\n",
    "\n",
    "**Gregory Way, 2019**\n",
    "\n",
    "Perform a t-test between two distinct phenotypes. In TCGA and GTEx data, we perform a t-test on males and females while in TARGET data, we test MYCN amplified vs. MYCN not amplified neuroblastoma tumors. We track the t-test p values across k dimensions and algorithms to isolate the features that best distinguishes the two groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import ttest_ind, ttest_rel\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sys.path.append(\"../8.gtex-interpret\")\n",
    "from scripts.utils import (\n",
    "    load_weight_matrix,\n",
    "    apply_signature,\n",
    "    load_enrichment_results,\n",
    "    extract_feature,\n",
    ")\n",
    "\n",
    "sys.path.append(\"../9.tcga-classify/\")\n",
    "from scripts.tcga_util import build_feature_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ttest_difference(feature_series, group_a_ids, group_b_ids):\n",
    "    \"\"\"\n",
    "    To be applied to a pandas dataframe by column\n",
    "    \"\"\"\n",
    "    feature_name = feature_series.name\n",
    "    feature_algorithm, feature_num = feature_name.split('_')\n",
    "    \n",
    "    a_activation = feature_series[feature_series.index.isin(group_a_ids)]\n",
    "    b_activation = feature_series[feature_series.index.isin(group_b_ids)]\n",
    "    \n",
    "    # Perform t-test on two groups\n",
    "    t_stat, t_p = ttest_ind(a_activation, b_activation, equal_var=False)\n",
    "    \n",
    "    return([t_stat, t_p, feature_algorithm, feature_num])\n",
    "\n",
    "\n",
    "def get_ttest_results(z_matrix_dict, group_a_ids, group_b_ids, train_or_test='test'):\n",
    "    \"\"\"\n",
    "    Loop through z matrix performing t-test using the compressed feature scores.\n",
    "    Output full t-test results\n",
    "    \"\"\"\n",
    "    \n",
    "    # Perform t-test for all compressed features\n",
    "    full_results = []\n",
    "    for signal in z_matrix_dict.keys():\n",
    "        for z_dim in z_matrix_dict[signal].keys():\n",
    "            for seed in z_matrix_dict[signal][z_dim].keys():\n",
    "                if train_or_test == \"both\":\n",
    "                    z_train_df = z_matrix_dict[signal][z_dim][seed][\"train\"]\n",
    "                    z_test_df = z_matrix_dict[signal][z_dim][seed][\"test\"]\n",
    "                    z_df = pd.concat([z_train_df, z_test_df])\n",
    "                else:\n",
    "                    z_df = z_matrix_dict[signal][z_dim][seed][train_or_test]\n",
    "\n",
    "                result_df = pd.DataFrame(z_df.apply(lambda x:\n",
    "                                                    ttest_difference(feature_series=x,\n",
    "                                                                     group_a_ids=group_a_ids,\n",
    "                                                                     group_b_ids=group_b_ids)),\n",
    "                                         columns = ['result'])\n",
    "\n",
    "                result_df = (\n",
    "                    pd.DataFrame(result_df.result.values.tolist(),\n",
    "                                 columns=['t_stat', 't_p', 'algorithm', 'feature_num'])\n",
    "                ).fillna(1)\n",
    "\n",
    "                result_df = result_df.assign(\n",
    "                    z_dim=z_dim,\n",
    "                    signal=signal,\n",
    "                    seed=seed\n",
    "                )\n",
    "                full_results.append(result_df)\n",
    "    \n",
    "    full_results_df = pd.concat(full_results)\n",
    "    full_results_df = full_results_df.assign(neg_log_p=-np.log10(full_results_df.t_p + 1e-300))\n",
    "    full_results_df = full_results_df.sort_values(by='neg_log_p', ascending=False)\n",
    "    return full_results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. GTEx Sex Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUBJID</th>\n",
       "      <th>SEX</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DTHHRDY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEX-1117F</td>\n",
       "      <td>2</td>\n",
       "      <td>60-69</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEX-111CU</td>\n",
       "      <td>1</td>\n",
       "      <td>50-59</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEX-111FC</td>\n",
       "      <td>1</td>\n",
       "      <td>60-69</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEX-111VG</td>\n",
       "      <td>1</td>\n",
       "      <td>60-69</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEX-111YS</td>\n",
       "      <td>1</td>\n",
       "      <td>60-69</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       SUBJID  SEX    AGE  DTHHRDY\n",
       "0  GTEX-1117F    2  60-69      4.0\n",
       "1  GTEX-111CU    1  50-59      0.0\n",
       "2  GTEX-111FC    1  60-69      1.0\n",
       "3  GTEX-111VG    1  60-69      3.0\n",
       "4  GTEX-111YS    1  60-69      0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load GTEx phenotype data\n",
    "file = os.path.join(\"..\", \"0.expression-download\", \"download\", \"GTEx_v7_Annotations_SubjectPhenotypesDS.txt\")\n",
    "gtex_pheno_df = pd.read_table(file)\n",
    "gtex_pheno_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15598, 63)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAMPID</th>\n",
       "      <th>SMATSSCR</th>\n",
       "      <th>SMCENTER</th>\n",
       "      <th>SMPTHNTS</th>\n",
       "      <th>SMRIN</th>\n",
       "      <th>SMTS</th>\n",
       "      <th>SMTSD</th>\n",
       "      <th>SMUBRID</th>\n",
       "      <th>SMTSISCH</th>\n",
       "      <th>SMTSPAX</th>\n",
       "      <th>...</th>\n",
       "      <th>SME1ANTI</th>\n",
       "      <th>SMSPLTRD</th>\n",
       "      <th>SMBSMMRT</th>\n",
       "      <th>SME1SNSE</th>\n",
       "      <th>SME1PCTS</th>\n",
       "      <th>SMRRNART</th>\n",
       "      <th>SME1MPRT</th>\n",
       "      <th>SMNUM5CD</th>\n",
       "      <th>SMDPMPRT</th>\n",
       "      <th>SME2PCTS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEX-1117F-0003-SM-58Q7G</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Blood</td>\n",
       "      <td>Whole Blood</td>\n",
       "      <td>0013756</td>\n",
       "      <td>1188.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEX-1117F-0003-SM-5DWSB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Blood</td>\n",
       "      <td>Whole Blood</td>\n",
       "      <td>0013756</td>\n",
       "      <td>1188.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     SAMPID  SMATSSCR SMCENTER SMPTHNTS  SMRIN   SMTS  \\\n",
       "0  GTEX-1117F-0003-SM-58Q7G       NaN       B1      NaN    NaN  Blood   \n",
       "1  GTEX-1117F-0003-SM-5DWSB       NaN       B1      NaN    NaN  Blood   \n",
       "\n",
       "         SMTSD  SMUBRID  SMTSISCH  SMTSPAX    ...    SME1ANTI SMSPLTRD  \\\n",
       "0  Whole Blood  0013756    1188.0      NaN    ...         NaN      NaN   \n",
       "1  Whole Blood  0013756    1188.0      NaN    ...         NaN      NaN   \n",
       "\n",
       "  SMBSMMRT SME1SNSE SME1PCTS SMRRNART SME1MPRT SMNUM5CD  SMDPMPRT  SME2PCTS  \n",
       "0      NaN      NaN      NaN      NaN      NaN      NaN       NaN       NaN  \n",
       "1      NaN      NaN      NaN      NaN      NaN      NaN       NaN       NaN  \n",
       "\n",
       "[2 rows x 63 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load GTEx Sample Attribute Data\n",
    "file = os.path.join(\"..\", \"0.expression-download\", \"download\", \"GTEx_v7_Annotations_SampleAttributesDS.txt\")\n",
    "gtex_sample_df = pd.read_table(file)\n",
    "\n",
    "print(gtex_sample_df.shape)\n",
    "gtex_sample_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load compressed matrix\n",
    "gtex_z_matrix_dict = build_feature_dictionary(dataset=\"GTEX\",\n",
    "                                              load_data=True,\n",
    "                                              store_train_test='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1169, 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>SUBJID</th>\n",
       "      <th>SEX</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DTHHRDY</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>SAMPID</th>\n",
       "      <th>SMATSSCR</th>\n",
       "      <th>SMCENTER</th>\n",
       "      <th>SMPTHNTS</th>\n",
       "      <th>...</th>\n",
       "      <th>SME1ANTI</th>\n",
       "      <th>SMSPLTRD</th>\n",
       "      <th>SMBSMMRT</th>\n",
       "      <th>SME1SNSE</th>\n",
       "      <th>SME1PCTS</th>\n",
       "      <th>SMRRNART</th>\n",
       "      <th>SME1MPRT</th>\n",
       "      <th>SMNUM5CD</th>\n",
       "      <th>SMDPMPRT</th>\n",
       "      <th>SME2PCTS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEX-ZTX8</td>\n",
       "      <td>GTEX-ZTX8</td>\n",
       "      <td>1</td>\n",
       "      <td>20-29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX-ZTX8-1126-SM-51MRM</td>\n",
       "      <td>GTEX-ZTX8-1126-SM-51MRM</td>\n",
       "      <td>1.0</td>\n",
       "      <td>B1</td>\n",
       "      <td>2 pieces</td>\n",
       "      <td>...</td>\n",
       "      <td>21116618.0</td>\n",
       "      <td>15670572.0</td>\n",
       "      <td>0.004129</td>\n",
       "      <td>21302446.0</td>\n",
       "      <td>50.219040</td>\n",
       "      <td>0.014548</td>\n",
       "      <td>0.993089</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.90624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEX-Y3IK</td>\n",
       "      <td>GTEX-Y3IK</td>\n",
       "      <td>2</td>\n",
       "      <td>50-59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX-Y3IK-2426-SM-4WWDU</td>\n",
       "      <td>GTEX-Y3IK-2426-SM-4WWDU</td>\n",
       "      <td>0.0</td>\n",
       "      <td>B1</td>\n",
       "      <td>6 pieces, up to 9x2mm. Well trimmed, but with ...</td>\n",
       "      <td>...</td>\n",
       "      <td>21949011.0</td>\n",
       "      <td>19389955.0</td>\n",
       "      <td>0.003998</td>\n",
       "      <td>22107996.0</td>\n",
       "      <td>50.180430</td>\n",
       "      <td>0.005525</td>\n",
       "      <td>0.985788</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.02296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEX-X62O</td>\n",
       "      <td>GTEX-X62O</td>\n",
       "      <td>1</td>\n",
       "      <td>50-59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX-X62O-0826-SM-46MW8</td>\n",
       "      <td>GTEX-X62O-0826-SM-46MW8</td>\n",
       "      <td>2.0</td>\n",
       "      <td>C1</td>\n",
       "      <td>2 pieces, 8x7 &amp; 7x7mm;</td>\n",
       "      <td>...</td>\n",
       "      <td>12511646.0</td>\n",
       "      <td>10713700.0</td>\n",
       "      <td>0.002388</td>\n",
       "      <td>12536978.0</td>\n",
       "      <td>50.050564</td>\n",
       "      <td>0.009804</td>\n",
       "      <td>0.993809</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.04157</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 69 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  patient_id     SUBJID  SEX    AGE  DTHHRDY                sample_id  \\\n",
       "0  GTEX-ZTX8  GTEX-ZTX8    1  20-29      0.0  GTEX-ZTX8-1126-SM-51MRM   \n",
       "1  GTEX-Y3IK  GTEX-Y3IK    2  50-59      0.0  GTEX-Y3IK-2426-SM-4WWDU   \n",
       "2  GTEX-X62O  GTEX-X62O    1  50-59      0.0  GTEX-X62O-0826-SM-46MW8   \n",
       "\n",
       "                    SAMPID  SMATSSCR SMCENTER  \\\n",
       "0  GTEX-ZTX8-1126-SM-51MRM       1.0       B1   \n",
       "1  GTEX-Y3IK-2426-SM-4WWDU       0.0       B1   \n",
       "2  GTEX-X62O-0826-SM-46MW8       2.0       C1   \n",
       "\n",
       "                                            SMPTHNTS    ...       SME1ANTI  \\\n",
       "0                                           2 pieces    ...     21116618.0   \n",
       "1  6 pieces, up to 9x2mm. Well trimmed, but with ...    ...     21949011.0   \n",
       "2                             2 pieces, 8x7 & 7x7mm;    ...     12511646.0   \n",
       "\n",
       "     SMSPLTRD  SMBSMMRT    SME1SNSE   SME1PCTS  SMRRNART  SME1MPRT SMNUM5CD  \\\n",
       "0  15670572.0  0.004129  21302446.0  50.219040  0.014548  0.993089      NaN   \n",
       "1  19389955.0  0.003998  22107996.0  50.180430  0.005525  0.985788      NaN   \n",
       "2  10713700.0  0.002388  12536978.0  50.050564  0.009804  0.993809      NaN   \n",
       "\n",
       "  SMDPMPRT  SME2PCTS  \n",
       "0      0.0  49.90624  \n",
       "1      0.0  50.02296  \n",
       "2      0.0  50.04157  \n",
       "\n",
       "[3 rows x 69 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract male and female ids from the dataset using one matrix as an example\n",
    "# (All matrices are aligned with the same IDs)\n",
    "example_matrix_df = gtex_z_matrix_dict['signal']['8']['451283']['test']\n",
    "\n",
    "patient_id_df = pd.concat(\n",
    "    [\n",
    "    pd.DataFrame([\"{}-{}\".format(x[0], x[1]) for x in example_matrix_df.index.str.split('-')],\n",
    "                 columns=['patient_id'])\n",
    "        .merge(gtex_pheno_df,\n",
    "               how='left',\n",
    "               left_on='patient_id',\n",
    "               right_on='SUBJID'),\n",
    "    pd.DataFrame(example_matrix_df.index)\n",
    "    ],\n",
    "    axis='columns'\n",
    ")\n",
    "\n",
    "patient_id_df = (\n",
    "    patient_id_df.merge(\n",
    "        gtex_sample_df,\n",
    "        left_on=\"sample_id\",\n",
    "        right_on=\"SAMPID\",\n",
    "        how=\"left\"\n",
    "    )\n",
    ")\n",
    "\n",
    "print(patient_id_df.shape)\n",
    "patient_id_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>SEX</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>ratio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SMTS</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Adrenal Gland</th>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>0.909091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Blood Vessel</th>\n",
       "      <td>55</td>\n",
       "      <td>36</td>\n",
       "      <td>0.660714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brain</th>\n",
       "      <td>111</td>\n",
       "      <td>56</td>\n",
       "      <td>0.508929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breast</th>\n",
       "      <td>19</td>\n",
       "      <td>10</td>\n",
       "      <td>0.550000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colon</th>\n",
       "      <td>31</td>\n",
       "      <td>19</td>\n",
       "      <td>0.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esophagus</th>\n",
       "      <td>61</td>\n",
       "      <td>41</td>\n",
       "      <td>0.677419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Heart</th>\n",
       "      <td>40</td>\n",
       "      <td>20</td>\n",
       "      <td>0.512195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung</th>\n",
       "      <td>27</td>\n",
       "      <td>16</td>\n",
       "      <td>0.607143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Muscle</th>\n",
       "      <td>33</td>\n",
       "      <td>23</td>\n",
       "      <td>0.705882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pancreas</th>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.687500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Skin</th>\n",
       "      <td>75</td>\n",
       "      <td>45</td>\n",
       "      <td>0.605263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spleen</th>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>1.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stomach</th>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>0.647059</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "SEX              1   2     ratio\n",
       "SMTS                            \n",
       "Adrenal Gland   10   9  0.909091\n",
       "Blood Vessel    55  36  0.660714\n",
       "Brain          111  56  0.508929\n",
       "Breast          19  10  0.550000\n",
       "Colon           31  19  0.625000\n",
       "Esophagus       61  41  0.677419\n",
       "Heart           40  20  0.512195\n",
       "Lung            27  16  0.607143\n",
       "Muscle          33  23  0.705882\n",
       "Pancreas        15  10  0.687500\n",
       "Skin            75  45  0.605263\n",
       "Spleen           7   9  1.250000\n",
       "Stomach         16  10  0.647059"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get gender ratio across samples\n",
    "gender_type_ratio = pd.crosstab(patient_id_df.SMTS, patient_id_df.SEX)\n",
    "gender_type_ratio = (\n",
    "    gender_type_ratio\n",
    "    .assign(ratio=((gender_type_ratio.loc[:, 2] + 1) / (gender_type_ratio.loc[:, 1] + 1)))\n",
    ")\n",
    "\n",
    "fudge_factor = 0.5\n",
    "balanced= (\n",
    "    gender_type_ratio\n",
    "    .loc[\n",
    "        (gender_type_ratio.ratio < (1 + fudge_factor)) &\n",
    "        (gender_type_ratio.ratio > (1 - fudge_factor)),\n",
    "        :]\n",
    ")\n",
    "\n",
    "out_file = os.path.join(\"results\", \"balanced_gtex_tissues.tsv\")\n",
    "balanced.to_csv(out_file, sep='\\t')\n",
    "        \n",
    "balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "304\n"
     ]
    }
   ],
   "source": [
    "balanced_tissue_types = balanced.index.tolist()\n",
    "\n",
    "gtex_males = (\n",
    "    patient_id_df\n",
    "    .query(\"SEX == 1\")\n",
    "    .query(\"SMTS in @balanced_tissue_types\")\n",
    "    .sample_id.tolist()\n",
    ")\n",
    "gtex_females = (\n",
    "    patient_id_df\n",
    "    .query(\"SEX == 2\")\n",
    "    .query(\"SMTS in @balanced_tissue_types\")\n",
    "    .sample_id.tolist()\n",
    ")\n",
    "\n",
    "print(len(gtex_males))\n",
    "print(len(gtex_females))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform t-test for all compressed features\n",
    "gtex_full_results_df = get_ttest_results(z_matrix_dict=gtex_z_matrix_dict,\n",
    "                                         group_a_ids=gtex_males,\n",
    "                                         group_b_ids=gtex_females,\n",
    "                                         train_or_test=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61700, 8)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_stat</th>\n",
       "      <th>t_p</th>\n",
       "      <th>algorithm</th>\n",
       "      <th>feature_num</th>\n",
       "      <th>z_dim</th>\n",
       "      <th>signal</th>\n",
       "      <th>seed</th>\n",
       "      <th>neg_log_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>44.507291</td>\n",
       "      <td>7.302909e-176</td>\n",
       "      <td>nmf</td>\n",
       "      <td>111</td>\n",
       "      <td>200</td>\n",
       "      <td>signal</td>\n",
       "      <td>451283</td>\n",
       "      <td>175.136504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>43.596212</td>\n",
       "      <td>2.655225e-172</td>\n",
       "      <td>nmf</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>signal</td>\n",
       "      <td>486191</td>\n",
       "      <td>171.575899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>43.448300</td>\n",
       "      <td>1.021953e-171</td>\n",
       "      <td>nmf</td>\n",
       "      <td>111</td>\n",
       "      <td>125</td>\n",
       "      <td>signal</td>\n",
       "      <td>486191</td>\n",
       "      <td>170.990569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>43.297948</td>\n",
       "      <td>4.074371e-171</td>\n",
       "      <td>nmf</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>signal</td>\n",
       "      <td>165158</td>\n",
       "      <td>170.389939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>42.894468</td>\n",
       "      <td>1.614220e-169</td>\n",
       "      <td>nmf</td>\n",
       "      <td>111</td>\n",
       "      <td>150</td>\n",
       "      <td>signal</td>\n",
       "      <td>978124</td>\n",
       "      <td>168.792037</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        t_stat            t_p algorithm feature_num z_dim  signal    seed  \\\n",
       "511  44.507291  7.302909e-176       nmf         111   200  signal  451283   \n",
       "411  43.596212  2.655225e-172       nmf         111   150  signal  486191   \n",
       "361  43.448300  1.021953e-171       nmf         111   125  signal  486191   \n",
       "411  43.297948  4.074371e-171       nmf         111   150  signal  165158   \n",
       "411  42.894468  1.614220e-169       nmf         111   150  signal  978124   \n",
       "\n",
       "      neg_log_p  \n",
       "511  175.136504  \n",
       "411  171.575899  \n",
       "361  170.990569  \n",
       "411  170.389939  \n",
       "411  168.792037  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Output results\n",
    "file = os.path.join(\"results\", \"sex_separation_gtex_t_test.tsv\")\n",
    "gtex_full_results_df.to_csv(file, sep='\\t', index=False)\n",
    "\n",
    "print(gtex_full_results_df.shape)\n",
    "gtex_full_results_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. TCGA Sex Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bcr_patient_barcode</th>\n",
       "      <th>type</th>\n",
       "      <th>age_at_initial_pathologic_diagnosis</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>ajcc_pathologic_tumor_stage</th>\n",
       "      <th>clinical_stage</th>\n",
       "      <th>histological_type</th>\n",
       "      <th>histological_grade</th>\n",
       "      <th>initial_pathologic_dx_year</th>\n",
       "      <th>...</th>\n",
       "      <th>residual_tumor</th>\n",
       "      <th>OS</th>\n",
       "      <th>OS.time</th>\n",
       "      <th>DSS</th>\n",
       "      <th>DSS.time</th>\n",
       "      <th>DFI</th>\n",
       "      <th>DFI.time</th>\n",
       "      <th>PFI</th>\n",
       "      <th>PFI.time</th>\n",
       "      <th>Redaction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-OR-A5J1</td>\n",
       "      <td>ACC</td>\n",
       "      <td>58.0</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage II</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Adrenocortical carcinoma- Usual Type</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1355.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1355.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>754.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>754.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TCGA-OR-A5J2</td>\n",
       "      <td>ACC</td>\n",
       "      <td>44.0</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage IV</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Adrenocortical carcinoma- Usual Type</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1677.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1677.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>289.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TCGA-OR-A5J3</td>\n",
       "      <td>ACC</td>\n",
       "      <td>23.0</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage III</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Adrenocortical carcinoma- Usual Type</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2091.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2091.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TCGA-OR-A5J4</td>\n",
       "      <td>ACC</td>\n",
       "      <td>23.0</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage IV</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Adrenocortical carcinoma- Usual Type</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>423.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>423.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TCGA-OR-A5J5</td>\n",
       "      <td>ACC</td>\n",
       "      <td>30.0</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage III</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Adrenocortical carcinoma- Usual Type</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>365.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>365.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  bcr_patient_barcode type  age_at_initial_pathologic_diagnosis  gender  \\\n",
       "1        TCGA-OR-A5J1  ACC                                 58.0    MALE   \n",
       "2        TCGA-OR-A5J2  ACC                                 44.0  FEMALE   \n",
       "3        TCGA-OR-A5J3  ACC                                 23.0  FEMALE   \n",
       "4        TCGA-OR-A5J4  ACC                                 23.0  FEMALE   \n",
       "5        TCGA-OR-A5J5  ACC                                 30.0    MALE   \n",
       "\n",
       "    race ajcc_pathologic_tumor_stage    clinical_stage  \\\n",
       "1  WHITE                    Stage II  [Not Applicable]   \n",
       "2  WHITE                    Stage IV  [Not Applicable]   \n",
       "3  WHITE                   Stage III  [Not Applicable]   \n",
       "4  WHITE                    Stage IV  [Not Applicable]   \n",
       "5  WHITE                   Stage III  [Not Applicable]   \n",
       "\n",
       "                      histological_type histological_grade  \\\n",
       "1  Adrenocortical carcinoma- Usual Type    [Not Available]   \n",
       "2  Adrenocortical carcinoma- Usual Type    [Not Available]   \n",
       "3  Adrenocortical carcinoma- Usual Type    [Not Available]   \n",
       "4  Adrenocortical carcinoma- Usual Type    [Not Available]   \n",
       "5  Adrenocortical carcinoma- Usual Type    [Not Available]   \n",
       "\n",
       "   initial_pathologic_dx_year    ...    residual_tumor   OS OS.time  DSS  \\\n",
       "1                      2000.0    ...               NaN  1.0  1355.0  1.0   \n",
       "2                      2004.0    ...               NaN  1.0  1677.0  1.0   \n",
       "3                      2008.0    ...               NaN  0.0  2091.0  0.0   \n",
       "4                      2000.0    ...               NaN  1.0   423.0  1.0   \n",
       "5                      2000.0    ...               NaN  1.0   365.0  1.0   \n",
       "\n",
       "   DSS.time  DFI DFI.time  PFI PFI.time Redaction  \n",
       "1    1355.0  1.0    754.0  1.0    754.0       NaN  \n",
       "2    1677.0  NaN      NaN  1.0    289.0       NaN  \n",
       "3    2091.0  1.0     53.0  1.0     53.0       NaN  \n",
       "4     423.0  NaN      NaN  1.0    126.0       NaN  \n",
       "5     365.0  NaN      NaN  1.0     50.0       NaN  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load TCGA phenotype data\n",
    "file = os.path.join(\"..\", \"0.expression-download\", \"download\", \"TCGA-CDR-SupplementalTableS1.xlsx\")\n",
    "tcga_pheno_df = pd.read_excel(file)\n",
    "\n",
    "tcga_pheno_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_z_matrix_dict = build_feature_dictionary(dataset=\"TCGA\",\n",
    "                                              load_data=True,\n",
    "                                              store_train_test='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1106, 35)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>bcr_patient_barcode</th>\n",
       "      <th>type</th>\n",
       "      <th>age_at_initial_pathologic_diagnosis</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>ajcc_pathologic_tumor_stage</th>\n",
       "      <th>clinical_stage</th>\n",
       "      <th>histological_type</th>\n",
       "      <th>histological_grade</th>\n",
       "      <th>...</th>\n",
       "      <th>OS</th>\n",
       "      <th>OS.time</th>\n",
       "      <th>DSS</th>\n",
       "      <th>DSS.time</th>\n",
       "      <th>DFI</th>\n",
       "      <th>DFI.time</th>\n",
       "      <th>PFI</th>\n",
       "      <th>PFI.time</th>\n",
       "      <th>Redaction</th>\n",
       "      <th>sample_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-CN-5365</td>\n",
       "      <td>TCGA-CN-5365</td>\n",
       "      <td>HNSC</td>\n",
       "      <td>38.0</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage IVB</td>\n",
       "      <td>Stage IVC</td>\n",
       "      <td>Head &amp; Neck Squamous Cell Carcinoma</td>\n",
       "      <td>G2</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>351.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>351.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>351.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCGA-CN-5365-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-LP-A7HU</td>\n",
       "      <td>TCGA-LP-A7HU</td>\n",
       "      <td>CESC</td>\n",
       "      <td>53.0</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>ASIAN</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>Stage II</td>\n",
       "      <td>Endocervical Type of Adenocarcinoma</td>\n",
       "      <td>G3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>406.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>406.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>406.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>406.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCGA-LP-A7HU-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TCGA-22-5491</td>\n",
       "      <td>TCGA-22-5491</td>\n",
       "      <td>LUSC</td>\n",
       "      <td>74.0</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>Stage IA</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Lung Squamous Cell Carcinoma</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1713.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1713.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1713.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCGA-22-5491-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TCGA-CS-6667</td>\n",
       "      <td>TCGA-CS-6667</td>\n",
       "      <td>LGG</td>\n",
       "      <td>39.0</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>[Not Available]</td>\n",
       "      <td>Astrocytoma</td>\n",
       "      <td>G2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1469.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1469.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1469.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCGA-CS-6667-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TCGA-20-1684</td>\n",
       "      <td>TCGA-20-1684</td>\n",
       "      <td>OV</td>\n",
       "      <td>51.0</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>[Not Applicable]</td>\n",
       "      <td>Stage IIIC</td>\n",
       "      <td>Serous Cystadenocarcinoma</td>\n",
       "      <td>G3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>581.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>581.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>581.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>581.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCGA-20-1684-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     patient_id bcr_patient_barcode  type  \\\n",
       "0  TCGA-CN-5365        TCGA-CN-5365  HNSC   \n",
       "1  TCGA-LP-A7HU        TCGA-LP-A7HU  CESC   \n",
       "2  TCGA-22-5491        TCGA-22-5491  LUSC   \n",
       "3  TCGA-CS-6667        TCGA-CS-6667   LGG   \n",
       "4  TCGA-20-1684        TCGA-20-1684    OV   \n",
       "\n",
       "   age_at_initial_pathologic_diagnosis  gender   race  \\\n",
       "0                                 38.0    MALE  WHITE   \n",
       "1                                 53.0  FEMALE  ASIAN   \n",
       "2                                 74.0    MALE  WHITE   \n",
       "3                                 39.0  FEMALE  WHITE   \n",
       "4                                 51.0  FEMALE  WHITE   \n",
       "\n",
       "  ajcc_pathologic_tumor_stage    clinical_stage  \\\n",
       "0                   Stage IVB         Stage IVC   \n",
       "1             [Not Available]          Stage II   \n",
       "2                    Stage IA  [Not Applicable]   \n",
       "3             [Not Available]   [Not Available]   \n",
       "4            [Not Applicable]        Stage IIIC   \n",
       "\n",
       "                     histological_type histological_grade       ...         \\\n",
       "0  Head & Neck Squamous Cell Carcinoma                 G2       ...          \n",
       "1  Endocervical Type of Adenocarcinoma                 G3       ...          \n",
       "2         Lung Squamous Cell Carcinoma    [Not Available]       ...          \n",
       "3                          Astrocytoma                 G2       ...          \n",
       "4            Serous Cystadenocarcinoma                 G3       ...          \n",
       "\n",
       "    OS OS.time  DSS DSS.time  DFI  DFI.time  PFI PFI.time Redaction  \\\n",
       "0  1.0   351.0  1.0    351.0  NaN       NaN  1.0    351.0       NaN   \n",
       "1  0.0   406.0  0.0    406.0  0.0     406.0  0.0    406.0       NaN   \n",
       "2  1.0  1713.0  NaN   1713.0  NaN       NaN  0.0   1713.0       NaN   \n",
       "3  0.0  1469.0  0.0   1469.0  NaN       NaN  0.0   1469.0       NaN   \n",
       "4  0.0   581.0  0.0    581.0  0.0     581.0  0.0    581.0       NaN   \n",
       "\n",
       "         sample_id  \n",
       "0  TCGA-CN-5365-01  \n",
       "1  TCGA-LP-A7HU-01  \n",
       "2  TCGA-22-5491-11  \n",
       "3  TCGA-CS-6667-01  \n",
       "4  TCGA-20-1684-01  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract male and female ids from the dataset\n",
    "example_matrix_df = tcga_z_matrix_dict['signal']['2']['451283']['test']\n",
    "\n",
    "patient_id_df = pd.concat(\n",
    "    [\n",
    "    pd.DataFrame([\"{}-{}-{}\".format(x[0], x[1], x[2]) for x in example_matrix_df.index.str.split('-')],\n",
    "                 columns=['patient_id'])\n",
    "        .merge(tcga_pheno_df,\n",
    "               how='left',\n",
    "               left_on='patient_id',\n",
    "               right_on='bcr_patient_barcode'),\n",
    "    pd.DataFrame(example_matrix_df.index)\n",
    "    ],\n",
    "    axis='columns'\n",
    ")\n",
    "\n",
    "print(patient_id_df.shape)\n",
    "patient_id_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>gender</th>\n",
       "      <th>FEMALE</th>\n",
       "      <th>MALE</th>\n",
       "      <th>ratio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>COAD</th>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>1.318182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DLBC</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GBM</th>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.636364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HNSC</th>\n",
       "      <td>19</td>\n",
       "      <td>37</td>\n",
       "      <td>0.526316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KICH</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KIRC</th>\n",
       "      <td>21</td>\n",
       "      <td>39</td>\n",
       "      <td>0.550000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KIRP</th>\n",
       "      <td>11</td>\n",
       "      <td>22</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LAML</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>0.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LGG</th>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>0.964286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LUAD</th>\n",
       "      <td>32</td>\n",
       "      <td>25</td>\n",
       "      <td>1.269231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PCPG</th>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>1.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>READ</th>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>1.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SARC</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SKCM</th>\n",
       "      <td>18</td>\n",
       "      <td>29</td>\n",
       "      <td>0.633333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>THYM</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UVM</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "gender  FEMALE  MALE     ratio\n",
       "type                          \n",
       "COAD        28    21  1.318182\n",
       "DLBC         3     2  1.333333\n",
       "GBM          6    10  0.636364\n",
       "HNSC        19    37  0.526316\n",
       "KICH         5     5  1.000000\n",
       "KIRC        21    39  0.550000\n",
       "KIRP        11    22  0.521739\n",
       "LAML         8     9  0.900000\n",
       "LGG         26    27  0.964286\n",
       "LUAD        32    25  1.269231\n",
       "PCPG        10     9  1.100000\n",
       "READ         9     8  1.111111\n",
       "SARC        13    13  1.000000\n",
       "SKCM        18    29  0.633333\n",
       "THYM         6     6  1.000000\n",
       "UVM          3     5  0.666667"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gender_type_ratio = pd.crosstab(patient_id_df.loc[:, \"type\"], patient_id_df.gender)\n",
    "gender_type_ratio = (\n",
    "    gender_type_ratio\n",
    "    .assign(ratio=((gender_type_ratio.FEMALE + 1) / (gender_type_ratio.MALE + 1)))\n",
    ")\n",
    "\n",
    "fudge_factor = 0.5\n",
    "balanced= (\n",
    "    gender_type_ratio\n",
    "    .loc[\n",
    "        (gender_type_ratio.ratio < (1 + fudge_factor)) &\n",
    "        (gender_type_ratio.ratio > (1 - fudge_factor)),\n",
    "        :]\n",
    ")\n",
    "\n",
    "out_file = os.path.join(\"results\", \"balanced_tcga_tissues.tsv\")\n",
    "balanced.to_csv(out_file, sep='\\t')\n",
    "\n",
    "balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "267\n",
      "218\n"
     ]
    }
   ],
   "source": [
    "balanced_cancer_types = balanced.index.tolist()\n",
    "\n",
    "tcga_males = (\n",
    "    patient_id_df\n",
    "    .query(\"gender == 'MALE'\")\n",
    "    .query(\"type in @balanced_cancer_types\")\n",
    "    .sample_id.tolist()\n",
    ")\n",
    "tcga_females = (\n",
    "    patient_id_df\n",
    "    .query(\"gender == 'FEMALE'\")\n",
    "    .query(\"type in @balanced_cancer_types\")\n",
    "    .sample_id.tolist()\n",
    ")\n",
    "\n",
    "print(len(tcga_males))\n",
    "print(len(tcga_females))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform t-test for all compressed features\n",
    "tcga_full_results_df = get_ttest_results(z_matrix_dict=tcga_z_matrix_dict,\n",
    "                                         group_a_ids=tcga_males,\n",
    "                                         group_b_ids=tcga_females,\n",
    "                                         train_or_test=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61700, 8)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_stat</th>\n",
       "      <th>t_p</th>\n",
       "      <th>algorithm</th>\n",
       "      <th>feature_num</th>\n",
       "      <th>z_dim</th>\n",
       "      <th>signal</th>\n",
       "      <th>seed</th>\n",
       "      <th>neg_log_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>4.864065</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>ica</td>\n",
       "      <td>53</td>\n",
       "      <td>90</td>\n",
       "      <td>signal</td>\n",
       "      <td>165158</td>\n",
       "      <td>5.798404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>-4.857106</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>ica</td>\n",
       "      <td>44</td>\n",
       "      <td>90</td>\n",
       "      <td>signal</td>\n",
       "      <td>451283</td>\n",
       "      <td>5.779352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>-4.797379</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>ica</td>\n",
       "      <td>15</td>\n",
       "      <td>90</td>\n",
       "      <td>signal</td>\n",
       "      <td>908341</td>\n",
       "      <td>5.657098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>4.766643</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>ica</td>\n",
       "      <td>14</td>\n",
       "      <td>90</td>\n",
       "      <td>signal</td>\n",
       "      <td>486191</td>\n",
       "      <td>5.593012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>4.621749</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>ica</td>\n",
       "      <td>42</td>\n",
       "      <td>90</td>\n",
       "      <td>signal</td>\n",
       "      <td>978124</td>\n",
       "      <td>5.299861</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       t_stat       t_p algorithm feature_num z_dim  signal    seed  neg_log_p\n",
       "143  4.864065  0.000002       ica          53    90  signal  165158   5.798404\n",
       "134 -4.857106  0.000002       ica          44    90  signal  451283   5.779352\n",
       "105 -4.797379  0.000002       ica          15    90  signal  908341   5.657098\n",
       "104  4.766643  0.000003       ica          14    90  signal  486191   5.593012\n",
       "132  4.621749  0.000005       ica          42    90  signal  978124   5.299861"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Output results\n",
    "file = os.path.join(\"results\", \"sex_separation_tcga_t_test.tsv\")\n",
    "tcga_full_results_df.to_csv(file, sep='\\t', index=False)\n",
    "\n",
    "print(tcga_full_results_df.shape)\n",
    "tcga_full_results_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. TARGET NBL MYCN Status Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usi</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Race</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Age at Diagnosis in Days</th>\n",
       "      <th>First Event</th>\n",
       "      <th>Event Free Survival Time in Days</th>\n",
       "      <th>Vital Status</th>\n",
       "      <th>Overall Survival Time in Days</th>\n",
       "      <th>Year of Diagnosis</th>\n",
       "      <th>...</th>\n",
       "      <th>Histology</th>\n",
       "      <th>Grade</th>\n",
       "      <th>MKI</th>\n",
       "      <th>Diagnostic Category</th>\n",
       "      <th>ICDO</th>\n",
       "      <th>ICDO Description</th>\n",
       "      <th>COG Risk Group</th>\n",
       "      <th>Site Relapse</th>\n",
       "      <th>Comment</th>\n",
       "      <th>target_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PAAPFA</td>\n",
       "      <td>Male</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>1762</td>\n",
       "      <td>Event</td>\n",
       "      <td>444.0</td>\n",
       "      <td>Dead</td>\n",
       "      <td>487.0</td>\n",
       "      <td>1986.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Unfavorable</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PACLJN</td>\n",
       "      <td>Male</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>1475</td>\n",
       "      <td>Censored</td>\n",
       "      <td>5553.0</td>\n",
       "      <td>Alive</td>\n",
       "      <td>5553.0</td>\n",
       "      <td>1986.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PACPJG</td>\n",
       "      <td>Female</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>760</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PACRYY</td>\n",
       "      <td>Male</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Hispanic or Latino</td>\n",
       "      <td>1314</td>\n",
       "      <td>Censored</td>\n",
       "      <td>5296.0</td>\n",
       "      <td>Alive</td>\n",
       "      <td>5296.0</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Unfavorable</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Lge Right retroperitoneal mass extending thru ...</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PACRZM</td>\n",
       "      <td>Male</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>3686</td>\n",
       "      <td>Event</td>\n",
       "      <td>922.0</td>\n",
       "      <td>Dead</td>\n",
       "      <td>922.0</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Unfavorable</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      usi  Gender     Race               Ethnicity  Age at Diagnosis in Days  \\\n",
       "0  PAAPFA    Male    White  Not Hispanic or Latino                      1762   \n",
       "1  PACLJN    Male    White  Not Hispanic or Latino                      1475   \n",
       "2  PACPJG  Female    White  Not Hispanic or Latino                       760   \n",
       "3  PACRYY    Male  Unknown      Hispanic or Latino                      1314   \n",
       "4  PACRZM    Male    White  Not Hispanic or Latino                      3686   \n",
       "\n",
       "  First Event  Event Free Survival Time in Days Vital Status  \\\n",
       "0       Event                             444.0         Dead   \n",
       "1    Censored                            5553.0        Alive   \n",
       "2     Unknown                               NaN      Unknown   \n",
       "3    Censored                            5296.0        Alive   \n",
       "4       Event                             922.0         Dead   \n",
       "\n",
       "   Overall Survival Time in Days  Year of Diagnosis      ...       \\\n",
       "0                          487.0             1986.0      ...        \n",
       "1                         5553.0             1986.0      ...        \n",
       "2                            NaN             1987.0      ...        \n",
       "3                         5296.0             1987.0      ...        \n",
       "4                          922.0             1987.0      ...        \n",
       "\n",
       "     Histology    Grade      MKI Diagnostic Category     ICDO  \\\n",
       "0  Unfavorable  Unknown  Unknown             Unknown  Unknown   \n",
       "1      Unknown  Unknown  Unknown             Unknown  Unknown   \n",
       "2      Unknown  Unknown  Unknown             Unknown  Unknown   \n",
       "3  Unfavorable  Unknown  Unknown             Unknown  Unknown   \n",
       "4  Unfavorable  Unknown  Unknown             Unknown  Unknown   \n",
       "\n",
       "                                    ICDO Description COG Risk Group  \\\n",
       "0                                            Unknown      High Risk   \n",
       "1                                            Unknown      High Risk   \n",
       "2                                            Unknown        Unknown   \n",
       "3  Lge Right retroperitoneal mass extending thru ...      High Risk   \n",
       "4                                            Unknown      High Risk   \n",
       "\n",
       "  Site Relapse Comment target_update  \n",
       "0          NaN     NaN           old  \n",
       "1          NaN     NaN           old  \n",
       "2          NaN     NaN           old  \n",
       "3          NaN     NaN           old  \n",
       "4          NaN     NaN           old  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load TARGET phenotype data\n",
    "file = os.path.join(\"..\", \"0.expression-download\", \"data\", \"2017-09-30-TARGET update harmonized.txt\")\n",
    "nbl_pheno_df = pd.read_table(file)\n",
    "nbl_pheno_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load TARGET matrices\n",
    "target_z_matrix_dict = build_feature_dictionary(dataset=\"TARGET\",\n",
    "                                                load_data=True,\n",
    "                                                store_train_test='both')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(130, 28)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>usi</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Race</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Age at Diagnosis in Days</th>\n",
       "      <th>First Event</th>\n",
       "      <th>Event Free Survival Time in Days</th>\n",
       "      <th>Vital Status</th>\n",
       "      <th>Overall Survival Time in Days</th>\n",
       "      <th>...</th>\n",
       "      <th>Grade</th>\n",
       "      <th>MKI</th>\n",
       "      <th>Diagnostic Category</th>\n",
       "      <th>ICDO</th>\n",
       "      <th>ICDO Description</th>\n",
       "      <th>COG Risk Group</th>\n",
       "      <th>Site Relapse</th>\n",
       "      <th>Comment</th>\n",
       "      <th>target_update</th>\n",
       "      <th>sample_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PARSBI</td>\n",
       "      <td>PARSBI</td>\n",
       "      <td>Female</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>2390.0</td>\n",
       "      <td>Relapse</td>\n",
       "      <td>1377.0</td>\n",
       "      <td>Dead</td>\n",
       "      <td>1743.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Undifferentiated or Poorly Differentiated</td>\n",
       "      <td>Low</td>\n",
       "      <td>Neuroblastoma</td>\n",
       "      <td>C76.2</td>\n",
       "      <td>Abdomen, NOS  Abdominal wall, NOS  Intra-abdom...</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>Other metastatic sites</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "      <td>TARGET-30-PARSBI-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>PARNNC</td>\n",
       "      <td>PARNNC</td>\n",
       "      <td>Female</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>41.0</td>\n",
       "      <td>Relapse</td>\n",
       "      <td>735.0</td>\n",
       "      <td>Alive</td>\n",
       "      <td>2979.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Undifferentiated or Poorly Differentiated</td>\n",
       "      <td>Low</td>\n",
       "      <td>Neuroblastoma</td>\n",
       "      <td>C48.0</td>\n",
       "      <td>Retroperitoneum\\n\\nPeriadrenal tissue\\n\\nPerin...</td>\n",
       "      <td>Low Risk</td>\n",
       "      <td>Primary site;; Other metastatic sites</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "      <td>TARGET-30-PARNNC-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>PASVRU</td>\n",
       "      <td>PASVRU</td>\n",
       "      <td>Male</td>\n",
       "      <td>White</td>\n",
       "      <td>Hispanic or Latino</td>\n",
       "      <td>631.0</td>\n",
       "      <td>Event</td>\n",
       "      <td>254.0</td>\n",
       "      <td>Dead</td>\n",
       "      <td>440.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Undifferentiated or Poorly Differentiated</td>\n",
       "      <td>High</td>\n",
       "      <td>Neuroblastoma</td>\n",
       "      <td>C48.0</td>\n",
       "      <td>Retroperitoneum  Periadrenal tissue  Perinephr...</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "      <td>TARGET-30-PASVRU-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>PALTEG</td>\n",
       "      <td>PALTEG</td>\n",
       "      <td>Male</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>1367.0</td>\n",
       "      <td>Relapse</td>\n",
       "      <td>523.0</td>\n",
       "      <td>Dead</td>\n",
       "      <td>551.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Undifferentiated or Poorly Differentiated</td>\n",
       "      <td>Intermediate</td>\n",
       "      <td>Neuroblastoma</td>\n",
       "      <td>C76.2</td>\n",
       "      <td>Abdomen, NOS  Abdominal wall, NOS  Intra-abdom...</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>Other Metastatic Sites</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "      <td>TARGET-30-PALTEG-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>PASJZC</td>\n",
       "      <td>PASJZC</td>\n",
       "      <td>Female</td>\n",
       "      <td>White</td>\n",
       "      <td>Not Hispanic or Latino</td>\n",
       "      <td>74.0</td>\n",
       "      <td>Relapse</td>\n",
       "      <td>367.0</td>\n",
       "      <td>Alive</td>\n",
       "      <td>2631.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Undifferentiated or Poorly Differentiated</td>\n",
       "      <td>Low</td>\n",
       "      <td>Neuroblastoma</td>\n",
       "      <td>C22.0</td>\n",
       "      <td>Liver\\n\\nHepatic, NOS</td>\n",
       "      <td>High Risk</td>\n",
       "      <td>Primary site</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old</td>\n",
       "      <td>TARGET-30-PASJZC-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   patient_id     usi  Gender   Race               Ethnicity  \\\n",
       "0      PARSBI  PARSBI  Female  White  Not Hispanic or Latino   \n",
       "11     PARNNC  PARNNC  Female  White  Not Hispanic or Latino   \n",
       "18     PASVRU  PASVRU    Male  White      Hispanic or Latino   \n",
       "22     PALTEG  PALTEG    Male  White  Not Hispanic or Latino   \n",
       "24     PASJZC  PASJZC  Female  White  Not Hispanic or Latino   \n",
       "\n",
       "    Age at Diagnosis in Days First Event  Event Free Survival Time in Days  \\\n",
       "0                     2390.0     Relapse                            1377.0   \n",
       "11                      41.0     Relapse                             735.0   \n",
       "18                     631.0       Event                             254.0   \n",
       "22                    1367.0     Relapse                             523.0   \n",
       "24                      74.0     Relapse                             367.0   \n",
       "\n",
       "   Vital Status  Overall Survival Time in Days         ...           \\\n",
       "0          Dead                         1743.0         ...            \n",
       "11        Alive                         2979.0         ...            \n",
       "18         Dead                          440.0         ...            \n",
       "22         Dead                          551.0         ...            \n",
       "24        Alive                         2631.0         ...            \n",
       "\n",
       "                                        Grade           MKI  \\\n",
       "0   Undifferentiated or Poorly Differentiated           Low   \n",
       "11  Undifferentiated or Poorly Differentiated           Low   \n",
       "18  Undifferentiated or Poorly Differentiated          High   \n",
       "22  Undifferentiated or Poorly Differentiated  Intermediate   \n",
       "24  Undifferentiated or Poorly Differentiated           Low   \n",
       "\n",
       "   Diagnostic Category   ICDO  \\\n",
       "0        Neuroblastoma  C76.2   \n",
       "11       Neuroblastoma  C48.0   \n",
       "18       Neuroblastoma  C48.0   \n",
       "22       Neuroblastoma  C76.2   \n",
       "24       Neuroblastoma  C22.0   \n",
       "\n",
       "                                     ICDO Description COG Risk Group  \\\n",
       "0   Abdomen, NOS  Abdominal wall, NOS  Intra-abdom...      High Risk   \n",
       "11  Retroperitoneum\\n\\nPeriadrenal tissue\\n\\nPerin...       Low Risk   \n",
       "18  Retroperitoneum  Periadrenal tissue  Perinephr...      High Risk   \n",
       "22  Abdomen, NOS  Abdominal wall, NOS  Intra-abdom...      High Risk   \n",
       "24                              Liver\\n\\nHepatic, NOS      High Risk   \n",
       "\n",
       "                             Site Relapse Comment target_update  \\\n",
       "0                  Other metastatic sites     NaN           old   \n",
       "11  Primary site;; Other metastatic sites     NaN           old   \n",
       "18                                    NaN     NaN           old   \n",
       "22                 Other Metastatic Sites     NaN           old   \n",
       "24                           Primary site     NaN           old   \n",
       "\n",
       "              sample_id  \n",
       "0   TARGET-30-PARSBI-01  \n",
       "11  TARGET-30-PARNNC-01  \n",
       "18  TARGET-30-PASVRU-01  \n",
       "22  TARGET-30-PALTEG-01  \n",
       "24  TARGET-30-PASJZC-01  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract amplified and not amplified samples from the dataset\n",
    "example_train_matrix_df = target_z_matrix_dict['signal']['8']['451283']['train']\n",
    "example_test_matrix_df = target_z_matrix_dict['signal']['8']['451283']['test']\n",
    "example_matrix_df = pd.concat([example_train_matrix_df, example_test_matrix_df])\n",
    "\n",
    "patient_id_df = (\n",
    "    pd.concat(\n",
    "        [\n",
    "            pd.DataFrame([x[2] for x in example_matrix_df.index.str.split('-')],\n",
    "                         columns=['patient_id'])\n",
    "            .merge(nbl_pheno_df,\n",
    "                   how='left',\n",
    "                   left_on='patient_id',\n",
    "                   right_on='usi'),\n",
    "            pd.DataFrame(example_matrix_df.index)\n",
    "        ],\n",
    "        axis='columns'\n",
    "    )\n",
    "    .dropna(subset=['usi'])\n",
    ")\n",
    "\n",
    "patient_id_df = patient_id_df.loc[patient_id_df[\"Diagnostic Category\"] == \"Neuroblastoma\", :]\n",
    "\n",
    "mycn_amp = patient_id_df.loc[patient_id_df[\"MYCN status\"] == \"Amplified\", \"sample_id\"].tolist()\n",
    "mycn_nonamp = patient_id_df.loc[patient_id_df[\"MYCN status\"] == \"Not Amplified\", \"sample_id\"].tolist()\n",
    "\n",
    "print(patient_id_df.shape)\n",
    "patient_id_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform t-test for all compressed features\n",
    "target_full_results_df = get_ttest_results(z_matrix_dict=target_z_matrix_dict,\n",
    "                                           group_a_ids=mycn_amp,\n",
    "                                           group_b_ids=mycn_nonamp,\n",
    "                                           train_or_test='both')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61700, 8)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_stat</th>\n",
       "      <th>t_p</th>\n",
       "      <th>algorithm</th>\n",
       "      <th>feature_num</th>\n",
       "      <th>z_dim</th>\n",
       "      <th>signal</th>\n",
       "      <th>seed</th>\n",
       "      <th>neg_log_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>-18.539478</td>\n",
       "      <td>6.613869e-38</td>\n",
       "      <td>vae</td>\n",
       "      <td>12</td>\n",
       "      <td>50</td>\n",
       "      <td>signal</td>\n",
       "      <td>451283</td>\n",
       "      <td>37.179544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>-17.652445</td>\n",
       "      <td>7.166732e-36</td>\n",
       "      <td>dae</td>\n",
       "      <td>88</td>\n",
       "      <td>100</td>\n",
       "      <td>signal</td>\n",
       "      <td>451283</td>\n",
       "      <td>35.144679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>-17.324506</td>\n",
       "      <td>9.638913e-35</td>\n",
       "      <td>vae</td>\n",
       "      <td>5</td>\n",
       "      <td>50</td>\n",
       "      <td>signal</td>\n",
       "      <td>978124</td>\n",
       "      <td>34.015972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>-16.472087</td>\n",
       "      <td>6.465500e-32</td>\n",
       "      <td>vae</td>\n",
       "      <td>95</td>\n",
       "      <td>150</td>\n",
       "      <td>signal</td>\n",
       "      <td>908341</td>\n",
       "      <td>31.189398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>-16.650011</td>\n",
       "      <td>9.533733e-31</td>\n",
       "      <td>nmf</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>signal</td>\n",
       "      <td>486191</td>\n",
       "      <td>30.020737</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        t_stat           t_p algorithm feature_num z_dim  signal    seed  \\\n",
       "162 -18.539478  6.613869e-38       vae          12    50  signal  451283   \n",
       "488 -17.652445  7.166732e-36       dae          88   100  signal  451283   \n",
       "155 -17.324506  9.638913e-35       vae           5    50  signal  978124   \n",
       "545 -16.472087  6.465500e-32       vae          95   150  signal  908341   \n",
       "72  -16.650011  9.533733e-31       nmf           2    35  signal  486191   \n",
       "\n",
       "     neg_log_p  \n",
       "162  37.179544  \n",
       "488  35.144679  \n",
       "155  34.015972  \n",
       "545  31.189398  \n",
       "72   30.020737  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.join(\"results\", \"nbl_mycn_separation_target_t_test.tsv\")\n",
    "target_full_results_df.to_csv(file, sep='\\t', index=False)\n",
    "\n",
    "print(target_full_results_df.shape)\n",
    "target_full_results_df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biobombe]",
   "language": "python",
   "name": "conda-env-biobombe-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}