10.gene-expression-signatures/0.download-validation-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download and Process Neuroblastoma RNAseq Data\n",
    "\n",
    "**Gregory Way 2019**\n",
    "\n",
    "We are downloading the dataset associated with [Harenza et al. 2017](https://doi.org/10.1038/sdata.2017.33). The data profiles RNAseq data from 39 commonly used neuroblastoma (NBL) cell lines.\n",
    "\n",
    "We are interested in the MYCN amplification status of these cell lines. We will test if the MYCN amplification score learned through the BioBombe signature approach applied to TARGET data generalizes to this cell line dataset.\n",
    "\n",
    "MYCN Amplification refers to the number of copies of the _MYCN_ gene. MYCN amplification is used as a biomarker for poor prognosis in neuroblastoma patients ([Huang and Weiss 2013](https://doi.org/10.1101/cshperspect.a014415))."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "import pandas as pd\n",
    "from urllib.request import urlretrieve\n",
    "\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://ndownloader.figshare.com/files/14138792\"\n",
    "name = \"2019-01-22-CellLineSTAR-fpkm-2pass_matrix.txt\"\n",
    "path = os.path.join(\"download\", name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"download\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('download/2019-01-22-CellLineSTAR-fpkm-2pass_matrix.txt',\n",
       " <http.client.HTTPMessage at 0x7f2d72ae0080>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urlretrieve(url, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55ea0255d1aa7708eba2ebd0113eeb3f  download/2019-01-22-CellLineSTAR-fpkm-2pass_matrix.txt\r\n"
     ]
    }
   ],
   "source": [
    "! md5sum \"download/2019-01-22-CellLineSTAR-fpkm-2pass_matrix.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Phenotype Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.nature.com/articles/sdata201733/tables/4\"\n",
    "name = \"nbl_cellline_phenotype.txt\"\n",
    "path = os.path.join(\"download\", name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cell Line</th>\n",
       "      <th>MYCN status</th>\n",
       "      <th>1p36 del</th>\n",
       "      <th>3p26 del</th>\n",
       "      <th>11q23 del</th>\n",
       "      <th>17q21-qter unbal gain</th>\n",
       "      <th>ALK mutation</th>\n",
       "      <th>p53 mutation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CHP134</td>\n",
       "      <td>Amplified</td>\n",
       "      <td>LOH p32.3-pter; Gain p34.3-p36.22; Loss p36.22...</td>\n",
       "      <td>Gain/AI p26.3</td>\n",
       "      <td>None</td>\n",
       "      <td>Gain q12-qter</td>\n",
       "      <td>WT</td>\n",
       "      <td>WT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CHP212</td>\n",
       "      <td>Amplified</td>\n",
       "      <td>Loss p13.2-pter</td>\n",
       "      <td>Gain/AI p26.3</td>\n",
       "      <td>cnLOH 23.3</td>\n",
       "      <td>Gain q12-qter</td>\n",
       "      <td>WT</td>\n",
       "      <td>WT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>COGN415</td>\n",
       "      <td>Amplified</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>F1174L</td>\n",
       "      <td>WT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>COGN440</td>\n",
       "      <td>Amplified</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>WT</td>\n",
       "      <td>WT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>COGN453</td>\n",
       "      <td>Amplified</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>F1174L</td>\n",
       "      <td>WT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cell Line MYCN status                                           1p36 del  \\\n",
       "0    CHP134   Amplified  LOH p32.3-pter; Gain p34.3-p36.22; Loss p36.22...   \n",
       "1    CHP212   Amplified                                    Loss p13.2-pter   \n",
       "2   COGN415   Amplified                                            Unknown   \n",
       "3   COGN440   Amplified                                            Unknown   \n",
       "4   COGN453   Amplified                                            Unknown   \n",
       "\n",
       "        3p26 del   11q23 del 17q21-qter unbal gain ALK mutation p53 mutation  \n",
       "0  Gain/AI p26.3        None         Gain q12-qter           WT           WT  \n",
       "1  Gain/AI p26.3  cnLOH 23.3         Gain q12-qter           WT           WT  \n",
       "2        Unknown     Unknown               Unknown       F1174L           WT  \n",
       "3        Unknown     Unknown               Unknown           WT           WT  \n",
       "4        Unknown     Unknown               Unknown       F1174L           WT  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if not os.path.isfile(path):\n",
    "    html = requests.get(url).content\n",
    "\n",
    "    pheno_df = pd.read_html(html)[0]\n",
    "    pheno_df['Cell Line'] = pheno_df['Cell Line'].str.replace(\"-\", \"\")\n",
    "\n",
    "    pheno_df.to_csv(path, sep='\\t', index=False)\n",
    "\n",
    "else:\n",
    "    pheno_df = pd.read_csv(path, sep=\"\\t\")\n",
    "\n",
    "pheno_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b08854900f32c037b0d1f516ab6b99b5  download/nbl_cellline_phenotype.txt\r\n"
     ]
    }
   ],
   "source": [
    "! md5sum \"download/nbl_cellline_phenotype.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process RNAseq Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GeneID</th>\n",
       "      <th>CHP134</th>\n",
       "      <th>CHP212</th>\n",
       "      <th>COGN415</th>\n",
       "      <th>COGN440</th>\n",
       "      <th>COGN453</th>\n",
       "      <th>COGN471</th>\n",
       "      <th>COGN496</th>\n",
       "      <th>COGN519</th>\n",
       "      <th>COGN534</th>\n",
       "      <th>...</th>\n",
       "      <th>RPE1</th>\n",
       "      <th>SHSY5Y</th>\n",
       "      <th>SKNAS</th>\n",
       "      <th>SKNBE2</th>\n",
       "      <th>SKNBE2C</th>\n",
       "      <th>SKNDZ</th>\n",
       "      <th>SKNFI</th>\n",
       "      <th>SKNSH</th>\n",
       "      <th>SMSKAN</th>\n",
       "      <th>SMSSAN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>EAF1</th>\n",
       "      <td>EAF1</td>\n",
       "      <td>8.783210</td>\n",
       "      <td>6.863459</td>\n",
       "      <td>5.462931</td>\n",
       "      <td>10.104389</td>\n",
       "      <td>6.604241</td>\n",
       "      <td>4.866576</td>\n",
       "      <td>9.329809</td>\n",
       "      <td>4.401073</td>\n",
       "      <td>11.083115</td>\n",
       "      <td>...</td>\n",
       "      <td>4.363805</td>\n",
       "      <td>8.555519</td>\n",
       "      <td>6.260962</td>\n",
       "      <td>4.842320</td>\n",
       "      <td>4.028825</td>\n",
       "      <td>8.861676</td>\n",
       "      <td>19.390770</td>\n",
       "      <td>9.936771</td>\n",
       "      <td>6.871837</td>\n",
       "      <td>7.696442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SARNP</th>\n",
       "      <td>SARNP</td>\n",
       "      <td>12.752439</td>\n",
       "      <td>23.717824</td>\n",
       "      <td>22.162407</td>\n",
       "      <td>27.401760</td>\n",
       "      <td>19.256736</td>\n",
       "      <td>9.157380</td>\n",
       "      <td>23.000173</td>\n",
       "      <td>25.250549</td>\n",
       "      <td>23.622492</td>\n",
       "      <td>...</td>\n",
       "      <td>25.264710</td>\n",
       "      <td>31.705089</td>\n",
       "      <td>22.246654</td>\n",
       "      <td>24.115542</td>\n",
       "      <td>28.652249</td>\n",
       "      <td>32.976600</td>\n",
       "      <td>38.725041</td>\n",
       "      <td>15.290273</td>\n",
       "      <td>18.659111</td>\n",
       "      <td>31.636539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CXCR5</th>\n",
       "      <td>CXCR5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004727</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KRTAP4-2</th>\n",
       "      <td>KRTAP4-2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAPK7</th>\n",
       "      <td>MAPK7</td>\n",
       "      <td>2.778997</td>\n",
       "      <td>2.779451</td>\n",
       "      <td>2.718453</td>\n",
       "      <td>1.677380</td>\n",
       "      <td>3.709534</td>\n",
       "      <td>0.988977</td>\n",
       "      <td>1.575597</td>\n",
       "      <td>1.455281</td>\n",
       "      <td>2.519137</td>\n",
       "      <td>...</td>\n",
       "      <td>1.971220</td>\n",
       "      <td>5.174954</td>\n",
       "      <td>1.115360</td>\n",
       "      <td>1.679980</td>\n",
       "      <td>0.971465</td>\n",
       "      <td>3.334610</td>\n",
       "      <td>2.354082</td>\n",
       "      <td>2.788888</td>\n",
       "      <td>5.015511</td>\n",
       "      <td>3.019704</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            GeneID     CHP134     CHP212    COGN415    COGN440    COGN453  \\\n",
       "EAF1          EAF1   8.783210   6.863459   5.462931  10.104389   6.604241   \n",
       "SARNP        SARNP  12.752439  23.717824  22.162407  27.401760  19.256736   \n",
       "CXCR5        CXCR5   0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "KRTAP4-2  KRTAP4-2   0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "MAPK7        MAPK7   2.778997   2.779451   2.718453   1.677380   3.709534   \n",
       "\n",
       "           COGN471    COGN496    COGN519    COGN534    ...           RPE1  \\\n",
       "EAF1      4.866576   9.329809   4.401073  11.083115    ...       4.363805   \n",
       "SARNP     9.157380  23.000173  25.250549  23.622492    ...      25.264710   \n",
       "CXCR5     0.000000   0.000000   0.004727   0.000000    ...       0.000000   \n",
       "KRTAP4-2  0.000000   0.000000   0.000000   0.000000    ...       0.000000   \n",
       "MAPK7     0.988977   1.575597   1.455281   2.519137    ...       1.971220   \n",
       "\n",
       "             SHSY5Y      SKNAS     SKNBE2    SKNBE2C      SKNDZ      SKNFI  \\\n",
       "EAF1       8.555519   6.260962   4.842320   4.028825   8.861676  19.390770   \n",
       "SARNP     31.705089  22.246654  24.115542  28.652249  32.976600  38.725041   \n",
       "CXCR5      0.000000   0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "KRTAP4-2   0.000000   0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "MAPK7      5.174954   1.115360   1.679980   0.971465   3.334610   2.354082   \n",
       "\n",
       "              SKNSH     SMSKAN     SMSSAN  \n",
       "EAF1       9.936771   6.871837   7.696442  \n",
       "SARNP     15.290273  18.659111  31.636539  \n",
       "CXCR5      0.000000   0.000000   0.000000  \n",
       "KRTAP4-2   0.000000   0.000000   0.000000  \n",
       "MAPK7      2.788888   5.015511   3.019704  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_file = os.path.join(\"download\", \"2019-01-22-CellLineSTAR-fpkm-2pass_matrix.txt\")\n",
    "\n",
    "raw_df = pd.read_table(raw_file, sep='\\t')\n",
    "raw_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update Gene Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load curated gene names from versioned resource \n",
    "commit = '721204091a96e55de6dcad165d6d8265e67e2a48'\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/genes.tsv'.format(commit)\n",
    "gene_df = pd.read_table(url)\n",
    "\n",
    "# Only consider protein-coding genes\n",
    "gene_df = (\n",
    "    gene_df.query(\"gene_type == 'protein-coding'\")\n",
    ")\n",
    "\n",
    "symbol_to_entrez = dict(zip(gene_df.symbol,\n",
    "                            gene_df.entrez_gene_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add alternative symbols to entrez mapping dictionary\n",
    "gene_df = gene_df.dropna(axis='rows', subset=['synonyms'])\n",
    "gene_df.synonyms = gene_df.synonyms.str.split('|')\n",
    "\n",
    "all_syn = (\n",
    "    gene_df.apply(lambda x: pd.Series(x.synonyms), axis=1)\n",
    "    .stack()\n",
    "    .reset_index(level=1, drop=True)\n",
    ")\n",
    "\n",
    "# Name the synonym series and join with rest of genes\n",
    "all_syn.name = 'all_synonyms'\n",
    "gene_with_syn_df = gene_df.join(all_syn)\n",
    "\n",
    "# Remove rows that have redundant symbols in all_synonyms\n",
    "gene_with_syn_df = (\n",
    "    gene_with_syn_df\n",
    "    \n",
    "    # Drop synonyms that are duplicated - can't be sure of mapping\n",
    "    .drop_duplicates(['all_synonyms'], keep=False)\n",
    "\n",
    "    # Drop rows in which the symbol appears in the list of synonyms\n",
    "    .query('symbol not in all_synonyms')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a synonym to entrez mapping and add to dictionary\n",
    "synonym_to_entrez = dict(zip(gene_with_syn_df.all_synonyms,\n",
    "                             gene_with_syn_df.entrez_gene_id))\n",
    "\n",
    "symbol_to_entrez.update(synonym_to_entrez)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load gene updater\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/updater.tsv'.format(commit)\n",
    "updater_df = pd.read_table(url)\n",
    "old_to_new_entrez = dict(zip(updater_df.old_entrez_gene_id,\n",
    "                             updater_df.new_entrez_gene_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_map = raw_df.GeneID.replace(symbol_to_entrez)\n",
    "gene_map = gene_map.replace(old_to_new_entrez)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19287, 40)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CHP134</th>\n",
       "      <th>CHP212</th>\n",
       "      <th>COGN415</th>\n",
       "      <th>COGN440</th>\n",
       "      <th>COGN453</th>\n",
       "      <th>COGN471</th>\n",
       "      <th>COGN496</th>\n",
       "      <th>COGN519</th>\n",
       "      <th>COGN534</th>\n",
       "      <th>COGN549</th>\n",
       "      <th>...</th>\n",
       "      <th>RPE1</th>\n",
       "      <th>SHSY5Y</th>\n",
       "      <th>SKNAS</th>\n",
       "      <th>SKNBE2</th>\n",
       "      <th>SKNBE2C</th>\n",
       "      <th>SKNDZ</th>\n",
       "      <th>SKNFI</th>\n",
       "      <th>SKNSH</th>\n",
       "      <th>SMSKAN</th>\n",
       "      <th>SMSSAN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10847</th>\n",
       "      <td>8.783210</td>\n",
       "      <td>6.863459</td>\n",
       "      <td>5.462931</td>\n",
       "      <td>10.104389</td>\n",
       "      <td>6.604241</td>\n",
       "      <td>4.866576</td>\n",
       "      <td>9.329809</td>\n",
       "      <td>4.401073</td>\n",
       "      <td>11.083115</td>\n",
       "      <td>5.182270</td>\n",
       "      <td>...</td>\n",
       "      <td>4.363805</td>\n",
       "      <td>8.555519</td>\n",
       "      <td>6.260962</td>\n",
       "      <td>4.842320</td>\n",
       "      <td>4.028825</td>\n",
       "      <td>8.861676</td>\n",
       "      <td>19.390770</td>\n",
       "      <td>9.936771</td>\n",
       "      <td>6.871837</td>\n",
       "      <td>7.696442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84324</th>\n",
       "      <td>12.752439</td>\n",
       "      <td>23.717824</td>\n",
       "      <td>22.162407</td>\n",
       "      <td>27.401760</td>\n",
       "      <td>19.256736</td>\n",
       "      <td>9.157380</td>\n",
       "      <td>23.000173</td>\n",
       "      <td>25.250549</td>\n",
       "      <td>23.622492</td>\n",
       "      <td>12.517533</td>\n",
       "      <td>...</td>\n",
       "      <td>25.264710</td>\n",
       "      <td>31.705089</td>\n",
       "      <td>22.246654</td>\n",
       "      <td>24.115542</td>\n",
       "      <td>28.652249</td>\n",
       "      <td>32.976600</td>\n",
       "      <td>38.725041</td>\n",
       "      <td>15.290273</td>\n",
       "      <td>18.659111</td>\n",
       "      <td>31.636539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004727</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85291</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5598</th>\n",
       "      <td>2.778997</td>\n",
       "      <td>2.779451</td>\n",
       "      <td>2.718453</td>\n",
       "      <td>1.677380</td>\n",
       "      <td>3.709534</td>\n",
       "      <td>0.988977</td>\n",
       "      <td>1.575597</td>\n",
       "      <td>1.455281</td>\n",
       "      <td>2.519137</td>\n",
       "      <td>1.208765</td>\n",
       "      <td>...</td>\n",
       "      <td>1.971220</td>\n",
       "      <td>5.174954</td>\n",
       "      <td>1.115360</td>\n",
       "      <td>1.679980</td>\n",
       "      <td>0.971465</td>\n",
       "      <td>3.334610</td>\n",
       "      <td>2.354082</td>\n",
       "      <td>2.788888</td>\n",
       "      <td>5.015511</td>\n",
       "      <td>3.019704</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   CHP134     CHP212    COGN415    COGN440    COGN453  \\\n",
       "entrez_gene_id                                                          \n",
       "10847            8.783210   6.863459   5.462931  10.104389   6.604241   \n",
       "84324           12.752439  23.717824  22.162407  27.401760  19.256736   \n",
       "643              0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "85291            0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "5598             2.778997   2.779451   2.718453   1.677380   3.709534   \n",
       "\n",
       "                 COGN471    COGN496    COGN519    COGN534    COGN549  \\\n",
       "entrez_gene_id                                                         \n",
       "10847           4.866576   9.329809   4.401073  11.083115   5.182270   \n",
       "84324           9.157380  23.000173  25.250549  23.622492  12.517533   \n",
       "643             0.000000   0.000000   0.004727   0.000000   0.000000   \n",
       "85291           0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "5598            0.988977   1.575597   1.455281   2.519137   1.208765   \n",
       "\n",
       "                  ...           RPE1     SHSY5Y      SKNAS     SKNBE2  \\\n",
       "entrez_gene_id    ...                                                   \n",
       "10847             ...       4.363805   8.555519   6.260962   4.842320   \n",
       "84324             ...      25.264710  31.705089  22.246654  24.115542   \n",
       "643               ...       0.000000   0.000000   0.000000   0.000000   \n",
       "85291             ...       0.000000   0.000000   0.000000   0.000000   \n",
       "5598              ...       1.971220   5.174954   1.115360   1.679980   \n",
       "\n",
       "                  SKNBE2C      SKNDZ      SKNFI      SKNSH     SMSKAN  \\\n",
       "entrez_gene_id                                                          \n",
       "10847            4.028825   8.861676  19.390770   9.936771   6.871837   \n",
       "84324           28.652249  32.976600  38.725041  15.290273  18.659111   \n",
       "643              0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "85291            0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "5598             0.971465   3.334610   2.354082   2.788888   5.015511   \n",
       "\n",
       "                   SMSSAN  \n",
       "entrez_gene_id             \n",
       "10847            7.696442  \n",
       "84324           31.636539  \n",
       "643              0.000000  \n",
       "85291            0.000000  \n",
       "5598             3.019704  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.index = gene_map\n",
    "raw_df.index.name = 'entrez_gene_id'\n",
    "raw_df = raw_df.drop(['GeneID'], axis='columns')\n",
    "raw_df = raw_df.loc[raw_df.index.isin(symbol_to_entrez.values()), :]\n",
    "\n",
    "print(raw_df.shape)\n",
    "raw_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scale Data and Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>...</th>\n",
       "      <th>105375787</th>\n",
       "      <th>106707243</th>\n",
       "      <th>106821730</th>\n",
       "      <th>106865373</th>\n",
       "      <th>107080638</th>\n",
       "      <th>107080644</th>\n",
       "      <th>107181291</th>\n",
       "      <th>107984155</th>\n",
       "      <th>107984923</th>\n",
       "      <th>107987479</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CHP134</th>\n",
       "      <td>0.496142</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031914</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.335754</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.527104</td>\n",
       "      <td>0.123020</td>\n",
       "      <td>0.101682</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.202001</td>\n",
       "      <td>0.350250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.359538</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.398515</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHP212</th>\n",
       "      <td>0.130269</td>\n",
       "      <td>0.077607</td>\n",
       "      <td>0.384543</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.047631</td>\n",
       "      <td>0.483361</td>\n",
       "      <td>0.011138</td>\n",
       "      <td>0.680574</td>\n",
       "      <td>0.002162</td>\n",
       "      <td>0.136316</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030310</td>\n",
       "      <td>0.673424</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.086474</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.556493</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COGN415</th>\n",
       "      <td>0.356876</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.168576</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.303912</td>\n",
       "      <td>0.068080</td>\n",
       "      <td>0.153800</td>\n",
       "      <td>0.100531</td>\n",
       "      <td>0.033393</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.417738</td>\n",
       "      <td>0.199854</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.211183</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.726023</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COGN440</th>\n",
       "      <td>0.512493</td>\n",
       "      <td>0.000593</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001941</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.373208</td>\n",
       "      <td>0.102337</td>\n",
       "      <td>0.249516</td>\n",
       "      <td>0.217459</td>\n",
       "      <td>0.004284</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.230559</td>\n",
       "      <td>0.538564</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.228188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.607477</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COGN453</th>\n",
       "      <td>0.383474</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.086284</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.464890</td>\n",
       "      <td>0.113665</td>\n",
       "      <td>0.261963</td>\n",
       "      <td>0.125526</td>\n",
       "      <td>0.001720</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.301525</td>\n",
       "      <td>0.471939</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.272335</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.375054</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 18628 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "entrez_gene_id         1         2         9        12        13        14  \\\n",
       "CHP134          0.496142  0.000000  0.031914  0.000000  0.000000  0.335754   \n",
       "CHP212          0.130269  0.077607  0.384543  0.000000  0.047631  0.483361   \n",
       "COGN415         0.356876  0.000000  0.168576  0.000000  0.000000  0.303912   \n",
       "COGN440         0.512493  0.000593  1.000000  0.001941  0.000000  0.373208   \n",
       "COGN453         0.383474  0.000000  0.086284  0.000000  0.000000  0.464890   \n",
       "\n",
       "entrez_gene_id        15        16        18        19    ...      105375787  \\\n",
       "CHP134          1.000000  0.527104  0.123020  0.101682    ...            0.0   \n",
       "CHP212          0.011138  0.680574  0.002162  0.136316    ...            0.0   \n",
       "COGN415         0.068080  0.153800  0.100531  0.033393    ...            0.0   \n",
       "COGN440         0.102337  0.249516  0.217459  0.004284    ...            0.0   \n",
       "COGN453         0.113665  0.261963  0.125526  0.001720    ...            0.0   \n",
       "\n",
       "entrez_gene_id  106707243  106821730  106865373  107080638  107080644  \\\n",
       "CHP134           0.202001   0.350250        0.0   0.359538        0.0   \n",
       "CHP212           0.030310   0.673424        0.0   0.086474        0.0   \n",
       "COGN415          0.417738   0.199854        0.0   0.211183        0.0   \n",
       "COGN440          0.230559   0.538564        0.0   0.228188        0.0   \n",
       "COGN453          0.301525   0.471939        0.0   0.272335        0.0   \n",
       "\n",
       "entrez_gene_id  107181291  107984155  107984923  107987479  \n",
       "CHP134                0.0        0.0   0.398515        0.0  \n",
       "CHP212                0.0        0.0   0.556493        0.0  \n",
       "COGN415               0.0        0.0   0.726023        0.0  \n",
       "COGN440               0.0        0.0   0.607477        0.0  \n",
       "COGN453               0.0        0.0   0.375054        0.0  \n",
       "\n",
       "[5 rows x 18628 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_scaled_df = preprocessing.MinMaxScaler().fit_transform(raw_df.transpose())\n",
    "raw_scaled_df = (\n",
    "    pd.DataFrame(raw_scaled_df,\n",
    "                 columns=raw_df.index,\n",
    "                 index=raw_df.columns)\n",
    "    .sort_index(axis='columns')\n",
    "    .sort_index(axis='rows')\n",
    ")\n",
    "raw_scaled_df.columns = raw_scaled_df.columns.astype(str)\n",
    "raw_scaled_df = raw_scaled_df.loc[:, ~raw_scaled_df.columns.duplicated(keep='first')]\n",
    "\n",
    "raw_scaled_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('data', exist_ok=True)\n",
    "\n",
    "file = os.path.join('data', 'nbl_celllines_processed_matrix.tsv.gz')\n",
    "raw_scaled_df.to_csv(file, sep='\\t', compression='gzip')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biobombe]",
   "language": "python",
   "name": "conda-env-biobombe-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}