Skip to content

Commit

Permalink
Consolidation + removing dev GenPipes as GenPipes 5 is a package
Browse files Browse the repository at this point in the history
  • Loading branch information
paulstretenowich committed Oct 24, 2024
1 parent eeceecc commit 8d00c7c
Show file tree
Hide file tree
Showing 14 changed files with 236 additions and 151 deletions.
2 changes: 1 addition & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1 +1 @@
singularity/
container/
72 changes: 72 additions & 0 deletions .github/workflows/docker-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#
name: Create and publish a Docker image

# Configures this workflow to run every time a change is pushed to the branch called 'main' or a PR is opened on 'main.
on:
push:
branches: ['main']
pull_request:
branches: ['main']

# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}

# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
build-and-push-image:
runs-on: ubuntu-latest

# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
attestations: write
id-token: write

steps:
- name: Checkout repository
uses: actions/checkout@v4

# Uses the `docker/login-action` action to log in to the Container registry using the account and password that will publish the packages.
# Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image.
# The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`.
# If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path.
# For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- name: Build and push Docker image
id: push
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
platforms: linux/amd64,linux/arm64
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

# This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built.
# It increases supply chain security for people who consume the image.
# For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)."
- name: Generate artifact attestation
uses: actions/attest-build-provenance@v1
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
subject-digest: ${{ steps.push.outputs.digest }}
push-to-registry: true
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
cvmfschecksum.*
*_mount_cache.sh
singularity/wrapper_genpipes/images
singularity/wrapper_genpipes.tgz
container/wrapper_genpipes/images
container/wrapper_genpipes.tgz
.DS_Store
37 changes: 22 additions & 15 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,20 +1,26 @@
FROM centos:7.9.2009
MAINTAINER P-O Quirion [email protected]
FROM ubuntu:24.04
LABEL authors="P-O Quirion [email protected], Paul Stretenowich [email protected]"

WORKDIR /tmp

# All yum cmd

ENV CVMFS_VERSION latest
ENV CC_STACK latest
ENV MODULE_VERSION 4.1.2
RUN yum install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-${CVMFS_VERSION}.noarch.rpm \
https://package.computecanada.ca/yum/cc-cvmfs-public/prod/RPM/computecanada-release-${CC_STACK}.noarch.rpm
RUN yum update -y \
&& yum install -y ImageMagick pigz which wget zip unzip.x86_64 make.x86_64 gcc expectk file \
dejagnu less tcl-devel.x86_64 cvmfs-config-computecanada \
cvmfs-fuse3 cvmfs-config-default \
&& yum clean all
ENV CVMFS_VERSION=latest
ENV CC_STACK=latest
ENV MODULE_VERSION=4.1.2

# All apt-get cmd
RUN apt-get update -y && \
apt-get install -y wget lsb-release
RUN wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-${CVMFS_VERSION}_all.deb && \
dpkg -i cvmfs-release-${CVMFS_VERSION}_all.deb && \
rm -f cvmfs-release-${CVMFS_VERSION}_all.deb && \
apt-get update -y && \
apt-get install -y cvmfs
RUN wget https://package.computecanada.ca/yum/cc-cvmfs-public/prod/other/cvmfs-config-computecanada-${CC_STACK}.all.deb && \
apt-get install -y ./cvmfs-config-computecanada-${CC_STACK}.all.deb && \
rm -f cvmfs-config-computecanada-${CC_STACK}.all.deb
RUN apt-get install -y libpng-dev libjpeg-dev libtiff-dev && \
apt-get install -y imagemagick pigz which zip unzip make gcc expect file dejagnu less tcl-dev cvmfs-config-computecanada cvmfs-fuse3 cvmfs-config-default && \
apt-get clean all

RUN mkdir /cvmfs-cache && chmod 777 /cvmfs-cache /cvmfs
RUN mkdir /cvmfs/ref.mugqic /cvmfs/soft.mugqic /cvmfs/cvmfs-config.computecanada.ca
Expand All @@ -25,6 +31,7 @@ RUN tar xzf modules-${MODULE_VERSION}.tar.gz && \
rm modules-${MODULE_VERSION}.tar.gz \
&& cd modules-${MODULE_VERSION} && ./configure && make -j 7 && make install \
&& cd .. && rm -rf modules-${MODULE_VERSION} && rm -rf /usr/local/Modules/modulefiles/*

# CVMFS
ADD default.local /etc/cvmfs/default.local
ADD soft.mugqic.local /etc/cvmfs/config.d/soft.mugqic.local
Expand All @@ -35,7 +42,7 @@ RUN echo "source /etc/profile.d/z00_module.sh" >> /etc/bashrc
ADD devmodule/genpipes "/usr/local/Modules/modulefiles/."


ADD genpipesrc /usr/local/etc/genpiperc
ADD genpipesrc /usr/local/etc/genpiperc
RUN ["ln", "-s", "/usr/local/etc/genpiperc", "/etc/profile.d/z90_genpipes.sh"]

ADD init_genpipes /usr/local/bin/init_genpipes
Expand Down
72 changes: 44 additions & 28 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,49 +1,49 @@
# GenPipes in a container

You can use GenPipes in a Container (GiaC) to run GenPipes on a single machine, on a a Torque/PBS cluster or on a SLURM cluster.
You can use GenPipes in a Container (GiaC) to run GenPipes on a single machine, on a Torque/PBS cluster or on a SLURM cluster.

If [Singularity](https://github.com/hpcng/singularity) is installed on your LINUX machine you are all set, a simple user with no special privilege is enough (no sudo needed).

While you can use (GiaC) to debug and develop GenPipes on your laptop, [GenPipes](https://bitbucket.org/mugqic/genpipes/src/master/README.md) is design to run analysis on Super Computers.
If Singularity/Docker is installed on your LINUX machine you are all set, a simple user with no special privilege is enough (no sudo needed).

While you can use (GiaC) to debug GenPipes on your laptop, [GenPipes](https://bitbucket.org/mugqic/genpipes/src/master/README.md) is design to run analysis on Super Computers.

## Install a compatible container technology on your machine
Follow installation procedure from the [Singularity install page](ttps://github.com/hpcng/singularity/blob/master/INSTALL.md)

Follow installation procedure from the [Singularity install page](https://github.com/hpcng/singularity/blob/master/INSTALL.md) or the [Docker install page](https://docs.docker.com/get-docker/).

You can also [Read the GenPipes documentation](https://genpipes.readthedocs.io/)
You can also [Read the GenPipes documentation](https://genpipes.readthedocs.io/).

## What exactly is avalable in that container?

The full tested and integrated C3G/MUGQIC software stack, a complete set of genomics references.
[For more details: http://www.computationalgenomics.ca/cvmfs-modules/](http://www.computationalgenomics.ca/cvmfs-modules/)

The full tested and integrated C3G/MUGQIC software stack: a complete set of genomics references and bioinformatics softwares. [For more details: http://www.computationalgenomics.ca/cvmfs-modules/](http://www.computationalgenomics.ca/cvmfs-modules/)

## Setup a GiaC environment

You can use this container to develop and test new version of GenPipes.
You can use this container to test new version of GenPipes.

Fist, clone GenPipes somewhere under your $HOME folder three. Then get the container wrapper:
First, clone GenPipes and then get the container wrapper:

```
git clone https://bitbucket.org/mugqic/genpipes $WORKDIR/genpipes
git clone https://bitbucket.org/mugqic/genpipes
$WORKDIR/genpipes/resources/container/get_wrapper.sh
genpipes/resources/container/get_wrapper.sh
```

You can now configuere the `$WORKDIR/genpipes/resources/container/etc/wrapper.conf` file:
You can now configure the `genpipes/resources/container/etc/wrapper.conf` file:

```
# GEN_SHARED_CVMFS should have a sufficient amount of space to load full reference files
export GEN_SHARED_CVMFS=$HOME/cvmfs
BIND_LIST=
GEN_CONTAINERTYPE=singularity
```

`GEN_SHARED_CVMFS` will hold a cache for GiaC [CVMFS](https://cernvm.cern.ch/portal/filesystem) system, it will hold the genomes and software being used by GenPipes. This folder will grow with GenPipes usage. You can delete it in between usage, but keep in mind that once deleted it will need to be rebuild by downloading data form the internet.

`BIND_LIST` is a list of file system, separated by comma, you need GenPipes to have access to, by default, only your $HOME is mounted. For example if you are on an HPC system with a `/scratch` and `/data` space, you would have `BIND_LIST=/scratch,/data`. The string will be fed to Singularity `--bind` option, see `singularity --help` for more details.

`GEN_CONTAINERTYPE` is the container to use, either `singularity` (default) or `docker`.

You do not need any other setup on your machine.

## PIPELINE USAGE
Expand All @@ -53,25 +53,41 @@ https://genpipes.readthedocs.io/

# On SLURM or PBS/torque HPC

Create an ini file that fits your system and for the pipeline(s) you want to use.

Add the `-j {pbs,slurm}` option to fit your scheduler then the `--wrap` options so GenPipes with wrap all its command with the container instrumentation.
[Read the GenPipes documentation](https://genpipes.readthedocs.io/), follow guidelines there to launch a GenPipes pipeline and add the `--wrap` option so GenPipes with wrap all its command with the container instrumentation.

# On a single machine.
## With the wrapper
Run the pipeline with the `--wrap`, `-j batch` and `--no-json` options!
[Read the GenPipes documentation](https://genpipes.readthedocs.io/), follow guidelines there to launch a GenPipes pipeline and add the `--wrap`, `-j batch` and `--no-json` options.

You can also run the `./genpipes/resources/container/bin/container_wrapper.sh` command to get inside the container with the right configuration. You will then have access to all the GenPipes tools be able to run them directly inside the container, on a single host without the `--wrap` option.
You can also run the `genpipes/resources/container/bin/container_wrapper.sh` command to get inside the container with the right configuration. You will then have access to all the GenPipes tools be able to run them directly inside the container, on a single host without the `--wrap` option.

## Whitout the wrapper
With `GEN_SHARED_CVMFS` being the cache directory on the host, `BIND_LIST` the file system to be acces by genpipes, {IMAGE_PATH}/genpipes.sif the [latest sif file released](https://github.com/c3g/genpipes_in_a_container/releases/latest).
### Using Singularity
With `GEN_SHARED_CVMFS` being the cache directory on the host, `BIND_LIST` the file system to be accessed by genpipes, {IMAGE_PATH}/genpipes.sif the [latest sif file released](https://github.com/c3g/genpipes_in_a_container/releases/latest).
```
singularity run --cleanenv -S /var/run/cvmfs -B ${GEN_SHARED_CVMFS}:/cvmfs-cache \
-B "$BIND_LIST" \
--fusemount \
"container:cvmfs2 cvmfs-config.computecanada.ca /cvmfs/cvmfs-config.computecanada.ca" \
--fusemount "container:cvmfs2 soft.mugqic /cvmfs/soft.mugqic" \
--fusemount "container:cvmfs2 ref.mugqic /cvmfs/ref.mugqic" \
${IMAGE_PATH}/genpipes.sif "$@"
singularity run \
--cleanenv \
-S /var/run/cvmfs \
-B ${GEN_SHARED_CVMFS}:/cvmfs-cache \
-B "$BIND_LIST" \
--fusemount "container:cvmfs2 cvmfs-config.computecanada.ca /cvmfs/cvmfs-config.computecanada.ca" \
--fusemount "container:cvmfs2 soft.mugqic /cvmfs/soft.mugqic" \
--fusemount "container:cvmfs2 ref.mugqic /cvmfs/ref.mugqic" \
${IMAGE_PATH}/genpipes.sif
```
### Using Docker
With `GEN_SHARED_CVMFS` being the cache directory on the host and `BIND_LIST` the file system to be accessed by genpipes.
```
docker run \
-it \
--env-file $HOME/.genpipes_env \
--rm \
--device /dev/fuse \
--cap-add SYS_ADMIN \
--tmpfs /var/run/cvmfs:rw \
-w $PWD \
-v $PWD:$PWD \
--mount type=bind,source=${BIND_LIST},target=${BIND_LIST} \
--mount type=bind,source=${GEN_SHARED_CVMFS},target=/cvmfs-cache \
c3genomics/genpipes:latest
```

File renamed without changes.
70 changes: 70 additions & 0 deletions container/wrapper_genpipes/bin/container_wrapper.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/bash

# Do not modify this file modify ${SCRIPTPATH}/etc/wrapper.conf instead!
SCRIPTPATH="$(cd "$(dirname "$0")" ; pwd -P)"
SCRIPTPATH=${SCRIPTPATH%%bin}
SCRIPTPATH=${SCRIPTPATH%%/}
GEN_SHARED_CVMFS=/tmp/cvmfs-cache
GEN_CONTAINERTYPE=singularity

source "${SCRIPTPATH}"/etc/wrapper.conf

mkdir -p ${GEN_SHARED_CVMFS}

touch "$HOME/.genpipes_env" # needs to exist for the run cmd not to crash

# If GEN_CONTAINERTYPE equals singularity
if [ "$GEN_CONTAINERTYPE" = "singularity" ]; then
if [ -z ${BIND_LIST+x} ]; then
singularity run \
--env-file $HOME/.genpipes_env \
--cleanenv \
-S /var/run/cvmfs \
-B ${GEN_SHARED_CVMFS}:/cvmfs-cache \
--fusemount "container:cvmfs2 cvmfs-config.computecanada.ca /cvmfs/cvmfs-config.computecanada.ca" \
--fusemount "container:cvmfs2 soft.mugqic /cvmfs/soft.mugqic" \
--fusemount "container:cvmfs2 ref.mugqic /cvmfs/ref.mugqic" \
${SCRIPTPATH}/images/genpipes.sif "$@"
else
singularity run \
--env-file $HOME/.genpipes_env \
--cleanenv \
-S /var/run/cvmfs \
-B ${GEN_SHARED_CVMFS}:/cvmfs-cache \
-B "$BIND_LIST" \
--fusemount "container:cvmfs2 cvmfs-config.computecanada.ca /cvmfs/cvmfs-config.computecanada.ca" \
--fusemount "container:cvmfs2 soft.mugqic /cvmfs/soft.mugqic" \
--fusemount "container:cvmfs2 ref.mugqic /cvmfs/ref.mugqic" \
${SCRIPTPATH}/images/genpipes.sif "$@"
fi
elif [ "$GEN_CONTAINERTYPE" = "docker" ]; then
if [ -z ${BIND_LIST+x} ]; then
docker run \
-it \
--env-file $HOME/.genpipes_env \
--rm \
--device /dev/fuse \
--cap-add SYS_ADMIN \
--tmpfs /var/run/cvmfs:rw \
-w $PWD \
-v $PWD:$PWD \
--mount type=bind,source=${BIND_LIST},target=${BIND_LIST} \
--mount type=bind,source=${GEN_SHARED_CVMFS},target=/cvmfs-cache \
c3genomics/genpipes:latest "$@"
else
docker run \
-it \
--env-file $HOME/.genpipes_env \
--rm \
--device /dev/fuse \
--cap-add SYS_ADMIN \
--tmpfs /var/run/cvmfs:rw \
-w $PWD \
-v $PWD:$PWD \
--mount type=bind,source=${GEN_SHARED_CVMFS},target=/cvmfs-cache \
c3genomics/genpipes:latest "$@"
fi
else
echo "Unknown GEN_CONTAINERTYPE $GEN_CONTAINERTYPE. Choose between 'singularity' or 'docker'. Exiting."
exit 1
fi
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# GEN_SHARED_CVMFS should have a sufficient amount of space to load full reference files
export GEN_SHARED_CVMFS=$HOME/cvmfs
BIND_LIST=
GEN_CONTAINERTYPE=singularity
27 changes: 0 additions & 27 deletions devmodule/genpipes/dev_genpipes

This file was deleted.

Loading

0 comments on commit 8d00c7c

Please sign in to comment.