0.data-download/1.data_downloader.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Dependency Data\n",
    "\n",
    "Source: [Cancer Dependency Map resource](https://depmap.org/portal/download/).\n",
    "\n",
    "- `CRISPRGeneDependency.parquet`: The data in this document describes the probability that a gene knockdown has an effect on cell-inhibition or death. These probabilities are derived from the data contained in CRISPRGeneEffect.parquet using methods described [here](https://doi.org/10.1101/720243)\n",
    "- `Model.parquet`: Metadata for all of DepMap’s cancer models/cell lines.\n",
    "- `CRISPRGeneEffect.parquet`: The data in this document are the Gene Effect Scores obtained from CRISPR knockout screens conducted by the Broad Institute. Negative scores notate that cell growth inhibition and/or death occurred following a gene knockout. Information on how these Gene Effect Scores were determined can be found [here](https://genomebiology.biomedcentral.com/articles/10.1186/s13059-021-02540-7)\n",
    "- `depmap_gene_meta.tsv`: Genes that passed QC and were included in the training model for Pan et al. 2022. We use this data to filter genes as input to our models. The genes were filtered based 1) variance, 2) perturbation confidence, and 3) high on target predictions based on high correlation across other guides.\n",
    "\n",
    ">Tsherniak A, Vazquez F, Montgomery PG, Weir BA, Kryukov G, Cowley GS, Gill S, Harrington WF, Pantel S, Krill-Burger JM, Meyers RM, Ali L, Goodale A, Lee Y, Jiang G, Hsiao J, Gerath WFJ, Howell S, Merkel E, Ghandi M, Garraway LA, Root DE, Golub TR, Boehm JS, Hahn WC. Defining a Cancer Dependency Map. Cell. 2017 Jul 27;170(3):564-576."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import urllib.request\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import pyarrow as pa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_dependency_data(figshare_id, figshare_url, output_file):\n",
    "    \"\"\"\n",
    "    Download the provided figshare resource\n",
    "    \"\"\"\n",
    "    urllib.request.urlretrieve(f\"{figshare_url}/{figshare_id}\", output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set download constants\n",
    "output_dir = pathlib.Path(\"data\")\n",
    "figshare_url = \"https://ndownloader.figshare.com/files/\"\n",
    "\n",
    "download_dict = {\n",
    "    \"46489732\": \"Model.csv\",\n",
    "    \"46489063\": \"CRISPRGeneEffect_Uncorrected.csv\",\n",
    "    \"46489021\": \"CRISPRGeneDependency.csv\",\n",
    "    \"29094531\": \"depmap_gene_meta.tsv\"\n",
    "     # DepMap, Broad (2024). DepMap 24Q2 Public. Figshare+. Dataset. https://doi.org/10.25452/figshare.plus.25880521.v1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make sure directory exists\n",
    "output_dir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data/Model.csv...\n",
      "Downloading data/CRISPRGeneEffect_Uncorrected.csv...\n",
      "Downloading data/CRISPRGeneDependency.csv...\n",
      "Downloading data/depmap_gene_meta.tsv...\n"
     ]
    }
   ],
   "source": [
    "for figshare_id in download_dict:\n",
    "    # Set output file\n",
    "    output_file = pathlib.Path(output_dir, download_dict[figshare_id])\n",
    "\n",
    "    # Download the dependency data\n",
    "    print(f\"Downloading {output_file}...\")\n",
    "\n",
    "    download_dependency_data(\n",
    "        figshare_id=figshare_id, figshare_url=figshare_url, output_file=output_file\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Column name correction for CRISPRGeneEffect\n",
    "df = pd.read_csv(\"../0.data-download/data/CRISPRGeneEffect_Uncorrected.csv\")\n",
    "df = df.rename(columns={df.columns[0]:'ModelID'})\n",
    "filepath = Path('../0.data-download/data/CRISPRGeneEffect.csv')\n",
    "filepath.parent.mkdir(parents=True, exist_ok=True) \n",
    "\n",
    "df.to_csv(filepath, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert to parquet\n",
    "# List of CSV files\n",
    "\n",
    "data_directory = \"../0.data-download/data/\"\n",
    "model_file = pathlib.Path(data_directory, \"Model.csv\").resolve()\n",
    "effect_data_file = pathlib.Path(data_directory, \"CRISPRGeneEffect.csv\").resolve()\n",
    "dependency_file = pathlib.Path(data_directory, \"CRISPRGeneDependency.csv\").resolve()\n",
    "metadata_file = pathlib.Path(data_directory, \"depmap_gene_meta.tsv\").resolve()\n",
    "\n",
    "csv_files = [model_file, effect_data_file, dependency_file]\n",
    "\n",
    "# Convert each CSV to Parquet\n",
    "for csv_file in csv_files:\n",
    "    # Read the CSV file\n",
    "    df = pd.read_csv(csv_file)\n",
    "    \n",
    "    # Define the output Parquet file name\n",
    "    parquet_file = csv_file.with_suffix('.parquet')\n",
    "    \n",
    "    # Save the DataFrame as a Parquet file\n",
    "    df.to_parquet(parquet_file, index=False)\n",
    "\n",
    "meta_df = pd.read_csv(metadata_file, sep=\"\\t\")\n",
    "meta_parquet = metadata_file.with_suffix('.parquet')\n",
    "meta_df.to_parquet(meta_parquet, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}