Merge pull request #391 from nservant/devel

Devel

.travis.yml

@@ -0,0 +1,35 @@
+sudo: required
+language: python
+python: '3.7'
+cache: pip
+matrix:
+  fast_finish: true
+
+before_install:
+  # install conda
+  - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+  - chmod +x miniconda.sh
+  - ./miniconda.sh -b
+  - export PATH=/home/travis/miniconda2/bin:$PATH
+  - conda update --yes conda
+
+
+install:
+  # use the conda env to install all dependencies
+  - mkdir -p /tmp/conda
+  - conda env create -p /tmp/conda/hicpro -f environment.yml
+  - export PATH=/tmp/conda/hicpro/bin:$PATH
+  # Install HiC-Pro
+  - mkdir /tmp/apps
+  - make configure prefix=/tmp/apps
+  - make install
+
+script:
+  # Get test data
+  - mkdir -p $HOME/test-op && cd $HOME/test-op
+  - wget https://zerkalo.curie.fr/partage/HiC-Pro/hicpro_testop_travis.tar.gz -P $HOME/test-op/ && tar -zxvf hicpro_testop_travis.tar.gz
+  - /bin/rm -f hicpro_testop_travis.tar.gz
+  # Run HiC-Pro
+  - INSTALLED_HICPRO_VERSION=$(find /tmp/apps -name HiC-Pro | xargs dirname)
+  - time $INSTALLED_HICPRO_VERSION/HiC-Pro -i test_data -o $HOME/test-op/results -c config_testop_yeast.txt
+

CHANGELOG

@@ -1,3 +1,30 @@
+***********************************
+CHANGES IN VERSION 3.0.0
+
+NEW FEATURES
+
+  o vcf file can be zipped in vcf.gz
+
+  o Add a Dockerfile for automatic Docker build on DockerHub
+
+  o Set up Travis continuous testing
+
+  o iced has been removed from HiC-Pro and is now an independant module
+
+  o Add a environment.yml file for installation with conda
+
+  o HiC-Pro is now based on python3. /!\ Python2 is no longer maintained /!\
+
+SIGNIFICANT USER-VISIBLE CHANGES
+
+  o SORT_RAM is not divided by N_CPU for samtools (#369)
+
+  o Fix bug in make_viewpoints.py if multiple viewpoints are specified in the bed file
+
+  o Add '--float' option for hicpro2higlass to manage normalized Hi-C data
+
+  o Update of the Singularity image
+
 ***********************************
 CHANGES IN VERSION 2.11.4
 

Dockerfile

@@ -0,0 +1,39 @@
+FROM ubuntu:18.04
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+LABEL authors="Nicolas Servant" \
+      description="Docker image containing all requirements for the HiC-Pro pipeline"
+
+## Install system tools
+RUN apt-get update \
+  && apt-get install -y build-essential \
+  wget \
+  unzip \
+  bzip2 \
+  gcc \
+  g++ && apt-get clean
+
+
+## Install miniconda.
+RUN wget https://repo.continuum.io/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh -O ~/anaconda.sh
+RUN bash ~/anaconda.sh -b -p /usr/local/anaconda
+RUN rm ~/anaconda.sh
+ENV PATH /usr/local/anaconda/bin:$PATH
+
+
+## Install all dependencies using conda
+COPY environment.yml /
+RUN conda env create -f /environment.yml && conda clean -a
+ENV PATH /usr/local/anaconda/envs/HiC-Pro_v3.0.0/bin:$PATH
+
+## Install HiCPro
+RUN cd /tmp && \
+    echo "devel.zip" | wget --base=http://github.com/nservant/HiC-Pro/archive/ -i - -O hicpro_latest.zip && \
+    unzip hicpro_latest.zip && \
+    cd HiC-Pro-devel  && \ 
+    make configure prefix=/ && \
+    make install && \
+    cd .. && \
+    rm -fr HiC-Pro*
+
+RUN /HiC-Pro_3.0.0/bin/HiC-Pro -h

INSTALL

@@ -14,22 +14,67 @@ The installation process will generate a config-system.txt file which defines al
 ## 
 
 Dependencies are checked during installation.
-A couple of tools such as bowtie2 and samtools (>1.0) can be automatically installed if not detected.
+A couple of tools such as `bowtie2` and `samtools` (>=1.9) can be automatically installed if not detected.
 Others have to be installed before starting the installation :
 
 - R (http://www.r-project.org/) with the following packages
 --- ggplot2 (>2.2.1)
 --- RColorBrewer
 --- grid
-- Python libraries :
---- bx-python (>=0.5.0) - https://pypi.python.org/pypi/bx-python
---- numpy (>=1.8.2) - http://www.scipy.org/scipylib/download.html
---- scipy (>=0.15.1) - http://www.scipy.org/scipylib/download.html
---- pysam (>=0.8.3) - https://github.com/pysam-developers/pysam
---- argparse if python <= 2.6
+- Python (>3.7) libraries :
+--- bx-python (>=0.8.8) - https://pypi.python.org/pypi/bx-python
+--- numpy (>=1.18.1) - http://www.scipy.org/scipylib/download.html
+--- scipy (>=1.4.1) - http://www.scipy.org/scipylib/download.html
+--- pysam (>=0.15.4) - https://github.com/pysam-developers/pysam
+--- argparse 
 
 Note that the iced module is also required (https://github.com/hiclib/iced). 
-Iced is part of the HiC-Pro source code, but can also be installed independantly (>=2.2)
+Iced is no longer part of the HiC-Pro source code, and should be independantly installed
+
+
+##
+## Containers
+##
+
+If you do not want to install all dependencies manually, HiC-Pro version 3.X now provides `conda`, `Docker` and `Singularity` recipes.
+
+### Using HiC-Pro through `conda`
+
+In order to ease the installation of HiC-Pro dependancies, we provide a `yml` file for conda with all required tools.
+In order to build your conda environment, first install [miniconda](https://docs.conda.io/en/latest/miniconda.html) and use :
+
+```
+conda env create -f MY_INSTALL_PATH/HiC-Pro/environment.yml -p WHERE_TO_INSTALL_MY_ENV
+conda activate WHERE_TO_INSTALL_MY_ENV
+```
+
+### Using the HiC-Pro `Docker` image
+
+A docker image is automatically build and available on [Docker Hub](https://hub.docker.com/repository/docker/nservant/hicpro)
+To pull a Docker image, simply use :
+
+```
+docker pull nservant/hicpro:latest
+```
+
+Note that the `tag` may depend on the HiC-Pro version.
+
+You can also build your own image from the root folder using
+
+```
+docker build -t hicpro:3.0.0 .
+```
+
+### Using HiC-Pro through `Singularity`
+
+HiC-Pro provides a Singularity container to ease its installation process.
+A ready-to-use container is available [here](https://zerkalo.curie.fr/partage/HiC-Pro/singularity_images/hicpro_latest_ubuntu.img).
+
+In order to build you own Singularity image, use :
+
+```
+sudo singularity build hicpro_latest_ubuntu.img MY_INSTALL_PATH/HiC-Pro/envs/Singularity
+```
 
 ##
 ## Install HiC-Pro

Makefile

@@ -14,7 +14,8 @@ CONFIGURE_OUT=$(wildcard ./config-system.txt)
 CONFIG_SYS=$(wildcard ./config-install.txt)
 RUNNER=$(shell whoami)
 
-install : config_check mapbuilder readstrimming iced cp
+#install : config_check mapbuilder readstrimming iced cp
+install : config_check mapbuilder readstrimming cp
 
 ######################################
 ## Config file
@@ -33,7 +34,11 @@ endif
 ######################################
 configure:
 ifneq ("$(CONFIG_SYS)","")
+ifneq ("$(prefix)","")
+	make -f ./scripts/install/Makefile CONFIG_SYS=$(CONFIG_SYS) prefix=$(prefix)
+else
 	make -f ./scripts/install/Makefile CONFIG_SYS=$(CONFIG_SYS)
+endif
 else
 	$(error config-install.txt file not found !)
 endif
@@ -51,14 +56,14 @@ readstrimming: $(INST_SOURCES)/cutsite_trimming.cpp
 	(g++ -Wall -O2 -std=c++0x -o cutsite_trimming ${INST_SOURCES}/cutsite_trimming.cpp; mv cutsite_trimming ${INST_SCRIPTS})
 
 ## Build Python lib
-iced: $(INST_SOURCES)/ice_mod
-ifeq ("$(RUNNER)","root")
-	@echo "Installing the iced package as root"     
-	(cd $(INST_SOURCES)/ice_mod/; ${PYTHON_PATH}/python setup.py install;)
-else
-	@echo "Installing the iced package in --user repository [runner=$(RUNNER)]"
-	(cd $(INST_SOURCES)/ice_mod/; ${PYTHON_PATH}/python setup.py install --user;)
-endif
+#iced: $(INST_SOURCES)/ice_mod
+#ifeq ("$(RUNNER)","root")
+#	@echo "Installing the iced package as root"     
+#	(cd $(INST_SOURCES)/ice_mod/; ${PYTHON_PATH}/python setup.py install;)
+#else
+#	@echo "Installing the iced package in --user repository [runner=$(RUNNER)]"
+#	(cd $(INST_SOURCES)/ice_mod/; ${PYTHON_PATH}/python setup.py install --user;)
+#endif
 
 test: config_check
 	@echo ${PYTHON_PATH}

README.md

@@ -3,8 +3,13 @@
 
 ### An optimized and flexible pipeline for Hi-C data processing
 
-![MultiQC](https://img.shields.io/badge/MultiQC-1.6-blue.svg)
-![Singularity](https://img.shields.io/badge/Singularity-build-brightgreen.svg)
+[![Build Status](https://travis-ci.com/nservant/HiC-Pro.svg?branch=devel_py3)](https://travis-ci.com/nservant/HiC-Pro)
+
+![Conda](https://img.shields.io/badge/Conda-build-brightgreen.svg)
+![Singularity](https://img.shields.io/badge/Singularity-build-brightgreen.svg) 
+[![Docker](https://img.shields.io/badge/Docker-manual-yellow.svg)](https://hub.docker.com/repository/docker/nservant/hicpro)
+
+![MultiQC](https://img.shields.io/badge/MultiQC-1.8-blue.svg)
 [![Forum](https://img.shields.io/badge/Groups-%20join%20chat%20%E2%86%92-4fb99a.svg?style=flat-square)](https://groups.google.com/forum/#!forum/hic-pro)
 [![DOI](https://img.shields.io/badge/DOI-10.1186%2Fs13059--015--0831--x-lightgrey.svg?style=flat-square)](https://doi.org/10.1186/s13059-015-0831-x)
 
@@ -25,7 +30,36 @@ If you use HiC-Pro, please cite :
 
 *Servant N., Varoquaux N., Lajoie BR., Viara E., Chen CJ., Vert JP., Dekker J., Heard E., Barillot E.* HiC-Pro: An optimized and flexible pipeline for Hi-C processing. Genome Biology 2015, 16:259 [doi:10.1186/s13059-015-0831-x](https://doi.org/10.1186/s13059-015-0831-x)
 
-## Using HiC-Pro through Singularity
+## Containers
+
+### Using HiC-Pro through `conda`
+
+In order to ease the installation of HiC-Pro dependancies, we provide a `yml` file for conda with all required tools.
+In order to build your conda environment, first install [miniconda](https://docs.conda.io/en/latest/miniconda.html) and use :
+
+```
+conda env create -f MY_INSTALL_PATH/HiC-Pro/environment.yml -p WHERE_TO_INSTALL_MY_ENV
+conda activate WHERE_TO_INSTALL_MY_ENV
+```
+
+### Using the HiC-Pro `Docker` image
+
+A docker image is automatically build and available on [Docker Hub](https://hub.docker.com/repository/docker/nservant/hicpro)
+To pull a Docker image, simply use :
+
+```
+docker pull nservant/hicpro:latest
+```
+
+Note that the `tag` may depend on the HiC-Pro version.
+
+You can also build your own image from the root folder using
+
+```
+docker build -t hicpro:3.0.0 .
+```
+
+### Using HiC-Pro through `Singularity`
 
 HiC-Pro provides a Singularity container to ease its installation process.
 A ready-to-use container is available [here](https://zerkalo.curie.fr/partage/HiC-Pro/singularity_images/hicpro_latest_ubuntu.img).
@@ -41,7 +75,7 @@ In order to build you own Singularity image;
 2- Build the singularity HiC-Pro image using the 'Singularity' file available in the HiC-Pro root directory.
 
 ```
-sudo singularity build hicpro_latest_ubuntu.img MY_INSTALL_PATH/HiC-Pro/Singularity
+sudo singularity build hicpro_latest_ubuntu.img MY_INSTALL_PATH/HiC-Pro/envs/Singularity
 ```
 
 3- Run HiC-pro
@@ -64,16 +98,16 @@ HiC-Pro -h
 The HiC-Pro pipeline requires the following dependencies :
 
 - The [bowtie2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml) mapper
-- Python (>2.7) with *pysam (>=0.8.3)*, *bx-python(>=0.5.0)*, *numpy(>=1.8.2)*, and *scipy(>=0.15.1)* libraries.  
-**Note that the current version does not support python 3**
+- Python (>3.7) with *pysam (>=0.15.4)*, *bx-python(>=0.8.8)*, *numpy(>=1.18.1)*, and *scipy(>=1.4.1)* libraries.  
+**Note that the current version no longer supports python 2**
 - R with the *RColorBrewer* and *ggplot2 (>2.2.1)* packages
 - g++ compiler
-- samtools (>1.1)
+- samtools (>1.9)
 - Unix sort (**which support -V option**) is required ! For Mac OS user, please install the GNU core utilities !
 
 Note that Bowtie >2.2.2 is strongly recommanded for allele specific analysis.  
 
-To install HiC-Pro (>=2.7.8), be sure to have the appropriate rights and run :
+To install HiC-Pro, be sure to have the appropriate rights and run :
 
 ```
 tar -zxvf HiC-Pro-master.tar.gz
@@ -83,15 +117,6 @@ make configure
 make install
 ```
 
-For older version (<2.7.8), the following process can be used
-
-```
-tar -zxvf HiC-Pro-master.tar.gz
-cd HiC-Pro-master
-## Edit config-install.txt file if necessary
-make CONFIG_SYS=config-install.txt install
-```
-
 Note that if some of these dependencies are not installed (i.e. not detected in the $PATH), HiC-Pro will try to install them.  
 You can also edit the *config-install.txt* file and manually defined the paths to dependencies.
 
@@ -100,9 +125,9 @@ You can also edit the *config-install.txt* file and manually defined the paths t
 |---------------|-------------------------------------------------------------------------------|
 | PREFIX        | Path to installation folder                                                   |
 | BOWTIE2_PATH  | Full path the bowtie2 installation directory                                  |
-| SAMTOOLS_PATH | Full path to the samtools installation directory (>1.1   )                    |
+| SAMTOOLS_PATH | Full path to the samtools installation directory                              |
 | R_PATH        | Full path to the R installation directory                                     |
-| PYTHON_PATH   | Full path to the python installation directory (>2.7 - python3 not supported) |
+| PYTHON_PATH   | Full path to the python installation directory                                |
 | CLUSTER_SYS   | Scheduler to use for cluster submission. Must be TORQUE, SGE, SLURM or LSF    |
 
 
@@ -155,7 +180,7 @@ First have a look at the help message !
   usage : HiC-Pro -i INPUT -o OUTPUT -c CONFIG [-s ANALYSIS_STEP] [-p] [-h] [-v]
   Use option -h|--help for more information
 
-  HiC-Pro 2.11.3
+  HiC-Pro 3.0.0
   ---------------
   OPTIONS
 
@@ -238,7 +263,7 @@ Small fastq files (2M reads) extracted from the Dixon et al. 2012 paper are avai
  ## Run HiC-Pro
  time HICPRO_INSTALL_DIR/bin/HiC-Pro -c config_test_latest.txt -i test_data -o hicpro_latest_test
 
-Run HiC-Pro 2.11.3
+Run HiC-Pro 3.0.0
 --------------------------------------------
 Thu Mar 19, 12:18:10 (UTC+0100)
 Bowtie2 alignment step1 ...

Singularity

@@ -40,15 +40,16 @@ From: ubuntu:latest
 
     # external tools
     echo "Installing dependancies ... "
-    conda install -y bowtie2
-    conda install -y samtools
+    conda install -y bowtie2=2.3.5
+    conda install -y samtools=1.9
 
-    # Python (>2.7) with *pysam (>=0.8.3)*, *bx(>=0.5.0)*, *numpy(>=1.8.2)*, and *scipy(>=0.15.1)* libraries
-    conda install -y -c conda-forge python=2.7.15
-    conda install -y -c anaconda scipy=1.2.1 
-    conda install -y -c anaconda numpy=1.16.3
-    conda install -y -c bcbio bx-python=0.8.2
-    conda install -y -c bioconda pysam=0.15.2 
+    # Python (>3.7.0) with *pysam (>=0.8.3)*, *bx(>=0.5.0)*, *numpy(>=1.8.2)*, and *scipy(>=0.15.1)* libraries
+    conda install -y -c conda-forge python=3.7.6
+    conda install -y -c conda-forge scipy 
+    conda install -y -c conda-forge numpy 
+    conda install -y -c bioconda bx-python 
+    conda install -y -c bioconda pysam 
+    conda install -y -c bioconda iced
 
     # Install R
     conda update readline
@@ -59,13 +60,13 @@ From: ubuntu:latest
     conda install -c r r-gridbase=0.4_7	
 
     # Install MultiQC
-    conda install -c bioconda multiqc=1.7 
+    conda install -c bioconda multiqc=1.8
 
     # Install HiC-pro
     echo "Installing latest HiC-Pro release ..."
     #VERSION=$(curl -s https://github.com/nservant/HiC-Pro/releases/latest | egrep -o '2.[0-9]*.[0-9]*')
     #echo "v"$VERSION".zip" | wget --base=http://github.com/nservant/HiC-Pro/archive/ -i - -O hicpro_latest.zip && unzip hicpro_latest.zip
-    VERSION="devel"
+    VERSION="devel_py3"
     echo $VERSION".zip" | wget --base=http://github.com/nservant/HiC-Pro/archive/ -i - -O hicpro_latest.zip && unzip hicpro_latest.zip
 
     cd $(echo HiC-Pro-$VERSION)