Skip to content

Commit 1a2e786

Browse files
SPPearceSPPearce
authored
Merge branch 'dev' into fix_integer_lane
2 parents d82f7a1 + f39c892 commit 1a2e786

25 files changed

+2401
-1058
lines changed

‎.github/actions/nf-test/action.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,14 @@ runs:
5454
conda-solver: libmamba
5555
conda-remove-defaults: true
5656

57+
# Set up secrets
58+
- name: Set up Nextflow secrets
59+
if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != ''
60+
shell: bash
61+
run: |
62+
python -m pip install cryptography
63+
nextflow secrets set SENTIEON_AUTH_DATA $(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
64+
5765
# TODO Skip failing conda tests and document their failures
5866
# https://github.com/nf-core/modules/issues/7017
5967
- name: Run nf-test
@@ -62,6 +70,8 @@ runs:
6270
NFT_DIFF: ${{ env.NFT_DIFF }}
6371
NFT_DIFF_ARGS: ${{ env.NFT_DIFF_ARGS }}
6472
NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
73+
SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
74+
SENTIEON_AUTH_MECH: "GitHub Actions - token"
6575
run: |
6676
nf-test test \
6777
--profile=+${{ inputs.profile }} \

‎.github/workflows/nf-test-gpu.yml

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,5 @@
11
name: Run nf-test on GPU
22
on:
3-
push:
4-
paths-ignore:
5-
- "docs/**"
6-
- "**/meta.yml"
7-
- "**/*.md"
8-
- "**/*.png"
9-
- "**/*.svg"
103
pull_request:
114
paths-ignore:
125
- "docs/**"
@@ -99,6 +92,10 @@ jobs:
9992
uses: ./.github/actions/nf-test
10093
env:
10194
NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
95+
SENTIEON_AUTH_MECH: "GitHub Actions - token"
96+
SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
97+
SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
98+
SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
10299
with:
103100
profile: ${{ matrix.profile }}
104101
shard: ${{ matrix.shard }}

‎.github/workflows/nf-test.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,5 @@
11
name: Run nf-test
22
on:
3-
push:
4-
paths-ignore:
5-
- "docs/**"
6-
- "**/meta.yml"
7-
- "**/*.md"
8-
- "**/*.png"
9-
- "**/*.svg"
103
pull_request:
114
paths-ignore:
125
- "docs/**"

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
105105
- [1879](https://github.com/nf-core/sarek/pull/1879) - Template update for nf-core/tools v3.21
106106
- [1892](https://github.com/nf-core/sarek/pull/1892) - Make jobs automatically resubmit for exit code 175
107107
- [1917](https://github.com/nf-core/sarek/pull/1917) - stub tests have stub tag
108+
- [1927](https://github.com/nf-core/sarek/pull/1927) - Migrate pipeline pytest sentieon tests to nf-test
108109

109110
#### Fixed
110111

@@ -118,6 +119,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
118119
- [1859](https://github.com/nf-core/sarek/pull/1859) - Fix: change dbsnp channel from queue to value in muse subworkflow, wrong implemented in [1744](https://github.com/nf-core/sarek/pull/1744)
119120
- [1899](https://github.com/nf-core/sarek/pull/1899) - Ensure nf-test runs for all profiles on release
120121
- [1917](https://github.com/nf-core/sarek/pull/1917) - Ensure all versions and reports are reported to MultiQC
122+
- [1927](https://github.com/nf-core/sarek/pull/1927) - Fixed Sentieon variant calling broken by [1871](https://github.com/nf-core/sarek/pull/1871)
121123
- [1924](https://github.com/nf-core/sarek/pull/1924) - Fix bug in samplesheet_to_channel workflow due to bad integer handling with lane
122124

123125
#### Removed

‎conf/modules/sentieon_haplotyper_joint_germline.config

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ process {
5050
}
5151

5252
withName: 'SENTIEON_APPLYVARCAL_INDEL' {
53-
ext.args = { '--sensitivity 99.9 --var_type INDEL' }
54-
ext.prefix = { 'joint_germline_recalibrated' }
55-
publishDir = [
53+
ext.args2 = { '--sensitivity 99.9 --var_type INDEL' }
54+
ext.prefix = { 'joint_germline_recalibrated_indel' }
55+
publishDir = [
5656
mode: params.publish_dir_mode,
5757
path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/"},
5858
pattern: "*{vcf.gz,vcf.gz.tbi}"
@@ -68,8 +68,12 @@ process {
6868
}
6969

7070
withName: 'SENTIEON_APPLYVARCAL_SNP' {
71-
ext.args = { '--sensitivity 99.9 --var_type SNP' }
72-
ext.prefix = { "${meta.id}_SNP" }
71+
ext.args2 = { '--sensitivity 99.9 --var_type SNP' }
72+
ext.prefix = { "${meta.id}_SNP" }
73+
// don't publish, as the output is put through SENTIEON_APPLYVARCAL_INDEL afterwards for a second round
74+
publishDir = [
75+
enabled: false
76+
]
7377
}
7478

7579
}

‎subworkflows/local/bam_joint_calling_germline_sentieon/main.nf

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,12 @@ workflow BAM_JOINT_CALLING_GERMLINE_SENTIEON {
3636
.map{ meta, gvcf, tbi, intervals -> [ [ id:'joint_variant_calling', intervals_name:intervals.baseName, num_intervals:meta.num_intervals ], gvcf, tbi, intervals ] }
3737
.groupTuple(by:[0, 3])
3838

39-
SENTIEON_GVCFTYPER(sentieon_input, fasta, fai, dbsnp, dbsnp_tbi)
39+
SENTIEON_GVCFTYPER(
40+
sentieon_input,
41+
fasta,
42+
fai,
43+
dbsnp.map{ file -> [[id:'dbsnp'], file] },
44+
dbsnp_tbi.map{ file -> [[id:'dbsnp'], file] })
4045

4146
BCFTOOLS_SORT(SENTIEON_GVCFTYPER.out.vcf_gz)
4247

‎subworkflows/local/bam_variant_calling_sentieon_haplotyper/main.nf

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,17 +42,16 @@ workflow BAM_VARIANT_CALLING_SENTIEON_HAPLOTYPER {
4242
]
4343
}
4444

45-
4645
emit_mode_items = sentieon_haplotyper_emit_mode.split(',').each{ it -> it.toLowerCase().trim() }
4746
lst = emit_mode_items - 'gvcf'
4847
emit_vcf = lst.size() > 0 ? lst[0] : ''
4948

5049
SENTIEON_HAPLOTYPER(
51-
cram_intervals_for_sentieon.map{ meta, cram, crai, intervals, num_intervals -> [ meta, cram, crai, intervals, [] ]},
50+
cram_intervals_for_sentieon.map{ meta, cram, crai, intervals -> [ meta, cram, crai, intervals, [] ]},
5251
fasta,
5352
fasta_fai,
54-
dbsnp,
55-
dbsnp_tbi,
53+
dbsnp.map{file -> [[id:'dbsnp'], file]},
54+
dbsnp_tbi.map{file -> [[id:'dbsnp'], file]},
5655
emit_vcf,
5756
emit_mode_items.any{ it.equals('gvcf') })
5857

‎tests/nextflow.config

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,20 @@ aws.client.anonymous = true
1313
// Should take care of all basepath
1414
params.modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
1515
params.igenomes_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data'
16+
17+
process {
18+
withName: 'SENTIEON_VARCAL_SNP' {
19+
// complains that training data is constant for MQRankSum, ReadPosRankSum, SOR
20+
ext.args = { '--annotation QD --annotation FS --annotation DP --var_type SNP' }
21+
}
22+
}
23+
24+
env {
25+
// NOTE This is how pipeline users will use Sentieon in real world use
26+
SENTIEON_LICENSE = "$SENTIEON_LICSRVR_IP"
27+
// NOTE This should only happen in GitHub actions or nf-core MegaTests
28+
SENTIEON_AUTH_MECH = "$SENTIEON_AUTH_MECH"
29+
SENTIEON_AUTH_DATA = secrets.SENTIEON_AUTH_DATA
30+
// NOTE This is how pipeline users will test out Sentieon with a license file
31+
// nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
32+
}

‎tests/sentieon_aligner_bwamem.nf.test

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
nextflow_pipeline {
2+
3+
name "Test pipeline"
4+
script "../main.nf"
5+
tag "pipeline"
6+
tag "pipeline_sarek"
7+
tag "cpu"
8+
tag "sentieon"
9+
10+
test("-profile test --aligner sentieon-bwamem --save_reference skip QC/recal/md") {
11+
12+
when {
13+
params {
14+
aligner = 'sentieon-bwamem'
15+
outdir = "$outputDir"
16+
save_reference = true
17+
skip_tools = 'baserecalibrator,fastqc,markduplicates,mosdepth,multiqc,samtools'
18+
tools = ''
19+
}
20+
}
21+
22+
then {
23+
// stable_name: All files + folders in ${params.outdir}/ with a stable name
24+
def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
25+
// stable_path: All files in ${params.outdir}/ with stable content
26+
def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
27+
// bam_files: All bam files
28+
def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam'])
29+
// cram_files: All cram files
30+
def cram_files = getAllFilesFromDir(params.outdir, include: ['**/*.cram'])
31+
// Fasta file for cram verification with nft-bam
32+
def fasta_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
33+
def fasta = fasta_base + 'genomics/homo_sapiens/genome/genome.fasta'
34+
// vcf_files: All vcf files
35+
def vcf_files = getAllFilesFromDir(params.outdir, include: ['**/*.vcf{,.gz}'])
36+
assert workflow.success
37+
assertAll(
38+
{ assert snapshot(
39+
// Number of successful tasks
40+
workflow.trace.succeeded().size(),
41+
// pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key)
42+
removeFromYamlMap("${outputDir}/pipeline_info/nf_core_sarek_software_mqc_versions.yml", "Workflow"),
43+
// All stable path name, with a relative path
44+
stable_name,
45+
// All files with stable contents
46+
stable_path.isEmpty() ? 'No stable content' : stable_path,
47+
// All cram files
48+
bam_files.isEmpty() ? 'No BAM files' : bam_files.collect { file -> file.getName() + ":md5," + bam(file.toString()).readsMD5 },
49+
// All cram files
50+
cram_files.isEmpty() ? 'No CRAM files' : cram_files.collect { file -> file.getName() + ":md5," + cram(file.toString(), fasta).readsMD5 },
51+
// All vcf files
52+
vcf_files.isEmpty() ? 'No VCF files' : vcf_files.collect { file -> file.getName() + ":md5," + path(file.toString()).vcf.variantsMD5 }
53+
).match() }
54+
)
55+
}
56+
}
57+
58+
test("-profile test --aligner sentieon-bwamem --save_reference --build_only_index") {
59+
60+
when {
61+
params {
62+
aligner = 'sentieon-bwamem'
63+
build_only_index = true
64+
input = false
65+
outdir = "$outputDir"
66+
save_reference = true
67+
skip_tools = 'multiqc'
68+
tools = ''
69+
}
70+
}
71+
72+
then {
73+
// stable_name: All files + folders in ${params.outdir}/ with a stable name
74+
def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
75+
// stable_path: All files in ${params.outdir}/ with stable content
76+
def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
77+
// bam_files: All bam files
78+
def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam'])
79+
// cram_files: All cram files
80+
def cram_files = getAllFilesFromDir(params.outdir, include: ['**/*.cram'])
81+
// Fasta file for cram verification with nft-bam
82+
def fasta_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
83+
def fasta = fasta_base + 'genomics/homo_sapiens/genome/genome.fasta'
84+
// vcf_files: All vcf files
85+
def vcf_files = getAllFilesFromDir(params.outdir, include: ['**/*.vcf{,.gz}'])
86+
assert workflow.success
87+
assertAll(
88+
{ assert snapshot(
89+
// Number of successful tasks
90+
workflow.trace.succeeded().size(),
91+
// pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key)
92+
removeFromYamlMap("${outputDir}/pipeline_info/nf_core_sarek_software_mqc_versions.yml", "Workflow"),
93+
// All stable path name, with a relative path
94+
stable_name,
95+
// All files with stable contents
96+
stable_path.isEmpty() ? 'No stable content' : stable_path,
97+
// All cram files
98+
bam_files.isEmpty() ? 'No BAM files' : bam_files.collect { file -> file.getName() + ":md5," + bam(file.toString()).readsMD5 },
99+
// All cram files
100+
cram_files.isEmpty() ? 'No CRAM files' : cram_files.collect { file -> file.getName() + ":md5," + cram(file.toString(), fasta).readsMD5 },
101+
// All vcf files
102+
vcf_files.isEmpty() ? 'No VCF files' : vcf_files.collect { file -> file.getName() + ":md5," + path(file.toString()).vcf.variantsMD5 }
103+
).match() }
104+
)
105+
}
106+
}
107+
108+
}

0 commit comments

Comments
 (0)