Merge pull request #15 from scilus/update_2.3.0

[ENH] Update 2.4.0 + new features nextflow

Author: arnaudbore

source/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = u'TractoFlow-documentation'
-copyright = u'2021, SCIL'
+copyright = u'2022, SCIL'
 author = u'SCIL'
 
 # The short X.Y version

source/data/install.sh

@@ -1,10 +1,8 @@
 #!/bin/bash
 
 echo "Downloading TractoFlow..."
-wget -q https://github.com/scilus/tractoflow/releases/download/2.2.1/tractoflow-2.2.1.zip
-unzip -q tractoflow-2.2.1.zip
-rm -rf tractoflow-2.2.1.zip
+nextflow pull scilus/tractoflow
 echo "Done: TractoFlow downloaded."
 echo "Downloading TractoFlow Singularity..."
-wget -q --show-progress http://scil.dinf.usherbrooke.ca/containers_list/tractoflow_2.2.1_b9a527_2021-04-13.sif
+wget -q --show-progress http://scil.usherbrooke.ca/containers_list/scilus_1.4.0.sif
 echo "Done: TractoFlow Singularity downloaded."

source/index.rst

@@ -2,8 +2,7 @@ Welcome to the TractoFlow user documentation!
 =============================================
 
 .. note::
-    New release available: 2.2.1.
-    TractoFlow now support BIDS as input data.
+    New release available: 2.4.0.
 
 TractoFlow pipeline is developed by the Sherbrooke Connectivity Imaging Lab (`SCIL`_)
 in order to process diffusion MRI dataset from the raw data to the tractography.
@@ -27,8 +26,7 @@ For MacOS users, please see this section :ref:`docker-tractoflow` for setup.
 For any issues or difficulties with TractoFlow, please use our Neurostar tag: https://neurostars.org/tag/tractoflow
 
 .. tip::
-    If you want to analyse datasets with white-matter lesions, we highly recommends
-    to use our devrived version of TractoFlow: TractoFlow Atlas based Segmentation (ABS) https://github.com/scilus/TractoFlow-ABS
+    If you want to analyse datasets with white-matter lesions use profile ABS.
 
 .. toctree::
    :maxdepth: 1
@@ -43,6 +41,7 @@ For any issues or difficulties with TractoFlow, please use our Neurostar tag: ht
 
    pipeline/steps
    pipeline/input
+
    pipeline/options
    pipeline/profiles
    pipeline/launch

source/installation/install.rst

@@ -1,25 +1,29 @@
-Install
-=======
+Fast Installation
+=================
 
 Easy install method
 -------------------
 
-Enter this command in your terminal (it downloads the container and TractoFlow code in the current directory):
+Enter this command in your terminal (it downloads the container and TractoFlow code in the current directory - Make sure nextflow is already installed before running this command):
 ::
 
-    curl -s https://tractoflow-documentation.readthedocs.io/en/2.2.1/install.sh | bash
+    curl -s https://tractoflow-documentation.readthedocs.io/en/2.4.0/install.sh | bash
+
+
+Detailed Installation
+=====================
 
 TractoFlow pipeline
 -------------------
 
 Release
 #######
 
-Download the last release of TractoFlow pipeline:
+Download TractoFlow pipeline:
 
 ::
 
-    $> wget https://github.com/scilus/tractoflow/releases/download/2.2.1/tractoflow-2.2.1.zip && unzip tractoflow-2.2.1.zip
+    $> nextflow pull scilus/tractoflow
 
 For developers
 ##############
@@ -34,6 +38,11 @@ Clone TractoFlow pipeline repository:
     # Clone with SSH
     $> git clone [email protected]:scilus/tractoflow.git
 
+As a developer you will have to run tractoflow using this command:
+
+::
+
+  nextflow run tractoflow/main.nf --help
 
 .. _singularity-tractoflow:
 
@@ -47,7 +56,13 @@ Download the last release of the Singularity container for TractoFlow:
 
 ::
 
-    $> wget http://scil.dinf.usherbrooke.ca/containers_list/tractoflow_2.2.1_b9a527_2021-04-13.sif
+    $> wget http://scil.usherbrooke.ca/containers_list/scilus_1.4.0.sif
+
+Or if you have sudo privileges
+
+::
+
+    $> sudo singularity build scilus_1.4.0.sif docker://scilus/scilus:1.4.0
 
 For developers
 ##############
@@ -57,16 +72,16 @@ Clone the singularity repository for TractoFlow pipeline:
 ::
 
     # Clone with HTTPS
-    $> git clone https://github.com/scilus/containers-tractoflow.git
+    $> git clone https://github.com/scilus/containers-scilus.git
 
     # Clone with SSH
-    $> git clone [email protected]:scilus/containers-tractoflow.git
+    $> git clone [email protected]:scilus/containers-scilus.git
 
 Then, you can build the singularity image:
 
 ::
 
-    $> singularity build singularity_name.img singularity_tractoflow.def
+    $> singularity build scilus_1.4.0.sif singularity_scilus.def
 
 .. _docker-tractoflow:
 
@@ -80,6 +95,6 @@ Download the last release of the Docker container for TractoFlow:
 
 ::
 
-    $> docker pull scilus/tractoflow:2.2.1
+    $> docker pull scilus/scilus:1.4.0
 
 Please see :ref:`profiles` section to use `macos` profile.

source/installation/requirements.rst

@@ -21,7 +21,7 @@ Local Computer
 
 ::
 
-    $> wget https://github.com/nextflow-io/nextflow/releases/download/v19.04.0/nextflow && chmod +x nextflow && \
+    $> wget https://github.com/nextflow-io/nextflow/releases/download/v21.10.6/nextflow && chmod +x nextflow && \
     echo 'export PATH=$PATH:'$(pwd) >> ~/.bash_profile && source ~/.bash_profile
 
 High Performance computer (HPC)
@@ -32,8 +32,8 @@ High Performance computer (HPC)
 
 ::
 
-    $> wget https://github.com/nextflow-io/nextflow/releases/download/v19.04.0/nextflow-19.04.0-all && \
-    mv nextflow-19.04.0-all nextflow && \
+    $> wget https://github.com/nextflow-io/nextflow/releases/download/v21.10.6/nextflow-21.10.6-all && \
+    mv nextflow-21.10.6-all nextflow && \
     chmod +x nextflow && echo 'export PATH=$PATH:'$(pwd) >> ~/.bash_profile && source ~/.bash_profile
 
 Note that a given HPC system might offer (a) readily available ``nextflow``
@@ -74,7 +74,7 @@ so that you can use the appropriate URL for the `wget` command.
 High Performance computer (HPC)
 ###############################
 
-Please try ``module load singularity/3.5`` or check with an administrator or on the HPC website.
+Please try ``module load singularity/3.7`` or check with an administrator or on the HPC website.
 
 Docker
 ------

source/pipeline/input.rst

@@ -14,13 +14,15 @@ before launching the processing to valide the BIDS format.
 In the case that some tags or informations are missing, TractoFlow will create a json file in ``results/Read_BIDS``.
 Please complete missing informations and relaunch the pipeline replacing ``--bids YOUR_BIDS_DATASET`` with ``--bids_config results/Read_BIDS/tractoflow_bids_struct.json``.
 
+If you have a BIDS structure and want to use `-profile ABS` you need to use the `--fs` option to point to your freesurfer folder output.
+
 If you have any problems, contact us on NeuroStar (https://neurostars.org/tag/tractoflow).
 
 Root parameter
 --------------
 
 It is possible not to follow the BIDS format. In that case, the input root parameter
-is called using ``--root`` and requires the following file structure:
+is called using ``--input`` and requires the following file structure:
 
 ::
 
@@ -29,21 +31,28 @@ is called using ``--root`` and requires the following file structure:
     │   ├── dwi.nii.gz
     │   ├── bval
     │   ├── bvec
+    │   ├── t1.nii.gz
     │   ├── rev_b0.nii.gz  (optional)
-    │   └── t1.nii.gz
+    │   ├── aparc+aseg.nii.gz (optional)
+    │   └── wmparc.nii.gz  (optional)
     └── S2
         ├── dwi.nii.gz
         ├── bval
         ├── bvec
+        ├── t1.nii.gz
         ├── rev_b0.nii.gz  (optional)
-        └── t1.nii.gz
+        ├── aparc+aseg.nii.gz (optional)
+        └── wmparc.nii.gz  (optional)
 
 The `root` folder must contains subjects folders (e.g. `S1`, `S2`,...). Each subject folder contains the required images:
 
     * ``dwi.nii.gz`` are the diffusion weighted images.
     * ``bval`` is the b-value file in the FSL format.
     * ``bvec`` is the b-vector file in the FSL format.
-    * ``rev_b0.nii.gz`` (optional) is the reversed phase encoded b0 image also called blip-up/blip-down. Used to correct distortion due to diffusion acquisition (`Documentation`_).
     * ``t1.nii.gz`` is the T1 weighted image.
+    * ``rev_b0.nii.gz`` (optional) is the reversed phase encoded b0 image also called blip-up/blip-down. Used to correct distortion due to diffusion acquisition (`Documentation`_).
+    * ``aparc+aseg.nii.gz`` (optional) is the freesurfer gm segmented image.
+    * ``wmparc.nii.gz`` (optional) is the freesurfer wm segmented image.
+
 
 .. _Documentation: https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/topup#topup_-_A_tool_for_estimating_and_correcting_susceptibility_induced_distortions

source/pipeline/launch.rst

@@ -9,18 +9,14 @@ To run the pipeline, use the following command:
 ::
 
     # With Singularity
-    $> nextflow run tractoflow/main.nf --bids input_bids --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-singularity singularity_name.img -resume
+    $> nextflow run tractoflow -r 2.4.0 --bids input_bids -with-singularity scilus_1.4.0.sif -resume
     # Or
-    $> nextflow run tractoflow/main.nf --root input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-singularity singularity_name.img -resume
+    $> nextflow run tractoflow -r 2.4.0 --input input_folder -with-singularity scilus_1.4.0.sif -resume
 
     # With Docker
-    $> nextflow run tractoflow/main.nf --bids input_bids --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-docker scilus/docker-tractoflow:2.1.1 -resume
+    $> nextflow run tractoflow -r 2.4.0 --bids input_bids -with-docker scilus/scilus:1.4.0 -resume
     # Or
-    $> nextflow run tractoflow/main.nf --root input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-docker scilus/docker-tractoflow:2.1.1 -resume
-
-Where ``DTI_SHELLS`` are the shells used to compute the DTI metrics
-(typically b-value < 1200 e.g. "0 1000") and ``FODF_SHELLS`` are the shells used
-to compute the fODF metrics (typically b > 700 e.g. "0 1000 2000").
+    $> nextflow run tractoflow -r 2.4.0 --input input_folder -with-docker scilus/scilus:1.4.0 -resume
 
 If you want to skip steps already processed by an anterior run, you can add `-resume` option in the command line.
 
@@ -42,7 +38,7 @@ to be executed with ``sbatch``.
     #SBATCH --mem=0
     #SBATCH --time=48:00:00
 
-    nextflow -c singularity.conf run tractoflow/main.nf --root input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-singularity singularity_name.img -resume
+    nextflow -c singularity.conf run tractoflow -r 2.4.0 --input input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-singularity singularity_name.sif -resume
 
 To launch on multiple nodes, you must to use the MPI option that use Ignite executor.
 The following example use 2 nodes with 32 threads on each nodes. The follwing lines
@@ -59,7 +55,7 @@ must be saved in ``.sh`` file (e.g. ``cmd.sh``) to be executed with ``sbatch``.
 
     export NXF_CLUSTER_SEED=$(shuf -i 0-16777216 -n 1)
 
-    srun nextflow -c singularity.conf run tractoflow/main.nf --root input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-singularity singularity_name.img -with-mpi -resume
+    srun nextflow -c singularity.conf run tractoflow -r 2.4.0 --input input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -with-singularity singularity_name.sif -with-mpi -resume
 
 To launch the pipeline on the HPC:
 

source/pipeline/options.rst

@@ -2,7 +2,7 @@ Options
 =======
 
 To display the options of Tractoflow, please use
-``nextflow run tractoflow/main.nf --help``.
+``nextflow run tractoflow -r 2.4.0 --help``.
 
 Optional BIDS arguments
 ------------
@@ -18,8 +18,13 @@ Optional BIDS arguments
   If set, it will remove all the participants that are missing any information.
 
 
+``--fs "freesurfer_output_folder"`` (default: none)
+  If you want to run Tractoflow-ABS (Atlas Based Segmentation) combined with a BIDS structure input
+  you need to have this argument.
+
 Options list
 ------------
+
 ``--b0_thr_extract_b0 MAX_VALUE``  (default: 10)
     All b-values below a maximum value are considered b=0 images.
 
@@ -84,6 +89,24 @@ Options list
 ``--dwi_interpolation METHOD`` (default: lin)
     Interpolation method [nn, lin, quad, cubic].
 
+``--max_dti_shell_value`` (default: 1200)
+    Maximum shell threshold to be consider as a DTI shell (b <= 1200).
+    This is the default behaviour to select DTI shells.
+
+``--dti_shells``
+    Shells selected to compute the DTI metrics (generally b <= 1200).
+    Please write them between quotes e.g. (--dti_shells "0 300 1000").
+    If selected, it will overwrite max_dti_shell_value.
+
+``--min_fodf_shell_value`` (default: 700)
+    Minimum shell threshold to be consider as a fODF shell (b >= 700).
+    This is the default behaviour to select fODF shells.
+
+``--fodf_shells``
+    Shells selected to compute the fODF metrics (generally b >= 700).
+    Please write them between quotes e.g. (--fodf_shells "0 1000 2000").
+    If selected, it will overwrite min_fodf_shell_value.
+
 ``--run_t1_denoising BOOL`` (default: true)
     Run T1 denoising using NLmean algorithm.
 
@@ -106,6 +129,9 @@ Options list
 ``--min_fa MIN_THRESHOLD`` (default: 0.5)
     Minimum FA threshold to compute the FRF.
 
+``--min_nvox MIN_NVOX_THRESHOLD`` (default: 300)
+    Minimum number of voxels to compute the FRF.
+
 ``--roi_radius RADIUS`` (default: 20)
     Region of interest radius to compute the FRF.
     This ROI starts from the center of the 3D volume (sizeX/2, sizeY/2, sizeZ/2).
@@ -238,8 +264,6 @@ Optional Local Tracking arguments
     [LOCAL] List of random seed numbers for the random number generator.
     Please write them as list separated using commat WITHOUT SPACE e.g. (--local_random_seed 0,1,2)
 
-
-
 ``--template_t1 PATH`` (default: /human-data/mni_152_sym_09c/t1)
     Path to the template T1 directory for antsBrainExtraction.
     The folder must contain t1_template.nii.gz and t1_brain_probability_map.nii.gz.

source/pipeline/profiles.rst

@@ -7,7 +7,7 @@ To select one or multiple profiles, please use the ``-profile`` option. For exam
 
 ::
 
-    $> nextflow run tractoflow/main.nf --root input_folder --dti_shells "DTI_SHELLS" --fodf_shells "FODF_SHELLS" -profile macos,fully_reproducible -with-singularity singularity_name.img -resume
+    $> nextflow run tractoflow -r 2.4.0 --input input_folder -profile macos,fully_reproducible -with-singularity singularity_name.sif -resume
 
 Profiles available
 ------------------
@@ -25,3 +25,14 @@ Profiles available
 
 ``cbrain``
     When this profile is used, Nextflow will copy all the output files in publishDir and not use symlinks.
+
+``ABS``
+    When this profile is used, TractoFlow-ABS (Atlas Based Segmentation) is used.
+    This profile must be used for pathological data.
+    The aparc+aseg.nii.gz and wmparc.nii.gz must be in the same space than t1.nii.gz
+
+``bundling``
+    When this profile is used, it will activate custom tracking parameters to improve recobundle results.
+
+``connectomics``
+    When this profile is used, it will activate custom tracking parameters to improve connectomics analysis.

source/reference/changelog.rst

@@ -1,6 +1,29 @@
 Changelog
 =========
 
+2.4.0
+########
+Date: November 2022
+
+New features
+  - Automatic extraction of shells when computing DTI and fODF
+  - Skip step bet_prelim_dwi when not needed
+  - Add remove_invalid step in Tracking processes
+  - Add possibility for complex BIDS structure with multiband acquisition and full reverse encoding acquisitions. (only available with cuda profile)
+  - New profile "bundling". It will activate custom tracking parameters to improve recobundle results. Local tracking will be enable with fa seeding mask and tracking mask.
+  - New profile "connectomics". It will activate custom tracking parameters to improve connectomics analysis.
+
+2.3.0
+########
+Date: 05 April 2022
+
+New features
+  - New profile Atlas Based Segmentation (-profile ABS)
+  - New profile "skip preprocessing" for HCP dataset (-profile skip_preprocessing)
+  - Add option to compute dwi sh (-sh_fitting true)
+  - Gibbs correction (-run_gibbs_correction true)
+
+
 2.2.1
 #########
 Date: 09 April 2021