1.process_data/normalize_jump_pilot.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "M0WvWCZ1Aa"
      },
      "source": [
        "# Preprocess using pycytominer and merge broad samples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "cDsoC3Gcm4"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "DgPm3tAEsr"
      },
      "source": [
        "import time\n",
        "from pathlib import Path\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from pycytominer import normalize\n",
        "from pycytominer.cyto_utils.cells import SingleCells"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "NPDmhE8H6X"
      },
      "source": [
        "## Find the root of the git directory\n",
        "This allows file paths to be referenced in a system agnostic way"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "yZSVxXoYEz"
      },
      "source": [
        "# Get the current working directory\n",
        "cwd = Path.cwd()\n",
        "\n",
        "if (cwd / \".git\").is_dir():\n",
        "    root_dir = cwd\n",
        "\n",
        "else:\n",
        "    root_dir = None\n",
        "    for parent in cwd.parents:\n",
        "        if (parent / \".git\").is_dir():\n",
        "            root_dir = parent\n",
        "            break\n",
        "\n",
        "# Check if a Git root directory was found\n",
        "if root_dir is None:\n",
        "    raise FileNotFoundError(\"No Git root directory found.\")"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "EAFlLWTJ8h"
      },
      "source": [
        "## Define Paths"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "OZedqnuQgs"
      },
      "source": [
        "# Input paths\n",
        "big_drive_path = f\"{root_dir}/big_drive\"\n",
        "sqlite_data_path = f\"{big_drive_path}/sc_data\"\n",
        "ref_path = f\"{root_dir}/reference_plate_data\"\n",
        "barcode_platemap = f\"{ref_path}/barcode_platemap.csv\"\n",
        "\n",
        "# Output paths\n",
        "output_cell_count_path = Path(f\"{big_drive_path}/sc_counts\")\n",
        "normalized_path = Path(f\"{big_drive_path}/normalized_sc_data\")"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "uGOqoaUkaq"
      },
      "source": [
        "## Create directories if non-existent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "RrixMBTKVc"
      },
      "source": [
        "output_cell_count_path.mkdir(parents=True, exist_ok=True)\n",
        "normalized_path.mkdir(parents=True, exist_ok=True)"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "b6AFJS78yk"
      },
      "source": [
        "# Create dataframe from barcode platemap\n",
        "barcode_df = pd.read_csv(barcode_platemap)"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "zNtqXlqkiR"
      },
      "source": [
        "# Process cell data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "5SOBQxOXXE"
      },
      "source": [
        "## Define functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "891gvwyvIr"
      },
      "source": [
        "# Add the 'Metadata' prefix to column names\n",
        "def add_metadata_prefix_to_column_names(df):\n",
        "    \"\"\"\n",
        "    Parameters\n",
        "    ----------\n",
        "    df: pandas Dataframe\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    df: pandas Dataframe\n",
        "        A dataframe with the column names prefixed with the string 'Metadata'\n",
        "    \"\"\"\n",
        "\n",
        "    df.rename(columns=lambda x: f\"Metadata_{x}\", inplace=True)\n",
        "    return df\n",
        "\n",
        "# Fill in broad_sample \"DMSO\" for NaN and prefix the column names\n",
        "def fill_dmso(df):\n",
        "    \"\"\"\n",
        "    Parameters\n",
        "    ----------\n",
        "    df: pandas Dataframe\n",
        "        A dataframe of the platemap data and a corresponding 'broad_sample' column\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    df: pandas Dataframe\n",
        "    A dataframe with without empty broad_samples and renamed columns\n",
        "    \"\"\"\n",
        "\n",
        "    df[\"broad_sample\"] = df[\"broad_sample\"].fillna(\"DMSO\")\n",
        "    df = add_metadata_prefix_to_column_names(df)\n",
        "    return df"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "r5VX7kI86O"
      },
      "source": [
        "## Map reference data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "xlSyhljTz3"
      },
      "source": [
        "# Merge on the broad_sample column\n",
        "merge_col = \"Metadata_broad_sample\"\n",
        "\n",
        "compdf = pd.read_csv(f\"{ref_path}/JUMP-Target-1_compound_metadata_targets.tsv\", sep=\"\\t\")\n",
        "\n",
        "# Set empty broad samples to DMSO if the pert iname is DMSO for the compounds dataframe\n",
        "compdf.loc[compdf[\"pert_iname\"] == \"DMSO\", \"broad_sample\"] = \"DMSO\"\n",
        "\n",
        "# Map platemap names found in the barcode file to metadata dataframes\n",
        "barcode_map = {\n",
        "    \"JUMP-Target-1_orf_platemap\": pd.read_csv(f\"{ref_path}/JUMP-Target-1_orf_metadata.tsv\", sep=\"\\t\"),\n",
        "    \"JUMP-Target-1_crispr_platemap\": pd.read_csv(f\"{ref_path}/JUMP-Target-1_crispr_metadata.tsv\", sep=\"\\t\"),\n",
        "    \"JUMP-Target-1_compound_platemap\": compdf\n",
        "}\n",
        "\n",
        "# Map platemap names found in the barcode file to platemap dataframes\n",
        "platemeta2df = {platemap_name: pd.read_csv(f\"{ref_path}/{platemap_name}.txt\", sep=\"\\t\") for platemap_name, _ in barcode_map.items()}\n",
        "\n",
        "# Map the wells corresponding to the empty broad samples in the plate metadata files\n",
        "platemeta2cols = {name: df.loc[df[\"broad_sample\"].isnull()][\"well_position\"].tolist() for name, df in platemeta2df.items() if name != \"JUMP-Target-1_compound_platemap\"}"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "XGivCKnP82"
      },
      "source": [
        "## Rename columns and fill control values"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "XtCuBmuSwW"
      },
      "source": [
        "# Rename colunns in plate metadata\n",
        "barcode_map = {df_name: add_metadata_prefix_to_column_names(df) for df_name, df in barcode_map.items()}\n",
        "\n",
        "# Fill the broad_sample missing values with DMSO for the plate metadata. This does not affect the CRISPR and ORF plates, since the platemaps have no matching DMSO broad samples\n",
        "platemeta2df = {df_name: fill_dmso(df) for df_name, df in platemeta2df.items()}"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "hNC232BAxW"
      },
      "source": [
        "## Merge, Normalize, and Feature Select plate data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "B4ITtH72EK"
      },
      "source": [
        "# Record the start time\n",
        "start_time = time.time()\n",
        "\n",
        "# Iterate through each plate in the barcode dataframe\n",
        "for idx, row in barcode_df.iterrows():\n",
        "\n",
        "    # Get the plate name\n",
        "    plate_name = row[\"Assay_Plate_Barcode\"]\n",
        "\n",
        "    # Get the platemap name\n",
        "    plate_map = row[\"Plate_Map_Name\"]\n",
        "\n",
        "    # Track progress\n",
        "    print(f\"\\nNormalizing Plate {plate_name}\")\n",
        "\n",
        "    # Get the plate metadata dataframe from the platemap name\n",
        "    broad_mapdf = barcode_map[plate_map]\n",
        "\n",
        "    # Final path of each cell count output file\n",
        "    output_cell_count_file = f\"{output_cell_count_path}/{plate_name}_cellcount.tsv\"\n",
        "\n",
        "    # Path of each normalized output single cell dataset\n",
        "    normalized_output = f\"{normalized_path}/{plate_name}_normalized_sc.parquet\"\n",
        "\n",
        "    # Path of the original sqlite file\n",
        "    sqlite_file = f\"sqlite:///{sqlite_data_path}/{plate_name}.sqlite\"\n",
        "\n",
        "    # Create dataframe from plate metadata\n",
        "    platemeta_df = platemeta2df[plate_map]\n",
        "\n",
        "    # Get the single cell data\n",
        "    sc = SingleCells(sql_file=sqlite_file, default_datatype_float=np.float32)\n",
        "\n",
        "    # Output the cell count data\n",
        "    cell_count_df = sc.count_cells()\n",
        "    cell_count_df.to_csv(output_cell_count_file, sep=\"\\t\", index=False)\n",
        "\n",
        "    # Merge single cells\n",
        "    sc_df = sc.merge_single_cells(platemap=platemeta_df)\n",
        "\n",
        "    # Merge the dataframes based on the broad_sample column, again there will not be matching DMSO broad samples for the CRISPR and ORF plates\n",
        "    sc_df = pd.merge(sc_df, broad_mapdf, how=\"left\", on=merge_col)\n",
        "\n",
        "    # We only change the columns if the plate does not contain empty wells.\n",
        "    # The wells that don't have a broad_sample are assigned \"no_treatment\" for the metadata columns\n",
        "    if plate_map != \"JUMP-Target-1_compound_platemap\":\n",
        "        sc_df.loc[sc_df[\"Metadata_Well\"].isin(platemeta2cols[plate_map]), broad_mapdf.columns] = \"no_treatment\"\n",
        "\n",
        "    # Normalize the data using the negative control as a reference\n",
        "    normalize(\n",
        "        profiles=sc_df,\n",
        "        features=\"infer\",\n",
        "        image_features=False,\n",
        "        meta_features=\"infer\",\n",
        "        samples=\"Metadata_control_type == 'negcon'\",\n",
        "        method=\"standardize\",\n",
        "        output_file=normalized_output,\n",
        "        output_type=\"parquet\",\n",
        "    )\n",
        "\n",
        "# Record the end time\n",
        "end_time = time.time()"
      ],
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "jukit_cell_id": "FwAgRkxcQ6"
      },
      "source": [
        "## Specify the time taken"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "H6A3v3ORWD"
      },
      "source": [
        "t_minutes = (end_time - start_time) // 60\n",
        "t_hours = t_minutes / 60\n",
        "print(f\"Total time taken = {t_minutes} minutes\")\n",
        "print(f\"Total time taken = {t_hours} hours\")"
      ],
      "outputs": [],
      "execution_count": null
    }
  ],
  "metadata": {
    "anaconda-cloud": {},
    "kernelspec": {
      "display_name": "python",
      "language": "python",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}