skip arm package failures

.github/scripts/add_arm_failure_tag.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Add arm_failure tag to nf-test files for modules that fail on ARM"""
+
+import sys
+import re
+from pathlib import Path
+
+def add_arm_failure_tag(test_file_path):
+    """Add arm_failure tag to the test file if it doesn't already exist"""
+
+    if not test_file_path.exists():
+        print(f"Test file not found: {test_file_path}")
+        return False
+
+    content = test_file_path.read_text()
+
+    # Check if arm_failure tag already exists
+    if 'tag "arm_failure"' in content:
+        print(f"ARM failure tag already exists in {test_file_path}")
+        return False
+
+    # Find the last tag line in the file - looking for pattern: tag "something"
+    tag_pattern = r'(\s*tag\s+"[^"]+"\s*\n)'
+    tag_matches = list(re.finditer(tag_pattern, content))
+
+    if tag_matches:
+        # Get the last tag match
+        last_tag = tag_matches[-1]
+        # Insert the new tag after the last existing tag with proper formatting
+        before = content[:last_tag.end()]
+        after = content[last_tag.end():]
+        new_content = before + '    tag "arm_failure"\n\n' + after
+    else:
+        # No existing tags, add after the process line
+        process_pattern = r'(process\s+"[^"]+"\s*\n)'
+        process_match = re.search(process_pattern, content)
+        if process_match:
+            before = content[:process_match.end()]
+            after = content[process_match.end():]
+            new_content = before + '\n    tag "arm_failure"\n' + after
+        else:
+            print(f"Could not find process line in {test_file_path}")
+            return False
+
+    # Write the updated content
+    test_file_path.write_text(new_content)
+    print(f"Added ARM failure tag to {test_file_path}")
+    return True
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: add_arm_failure_tag.py <module_path> [module_path2] ...")
+        print("Example: add_arm_failure_tag.py modules/nf-core/fastqc")
+        sys.exit(1)
+
+    success_count = 0
+    for module_path in sys.argv[1:]:
+        module_dir = Path(module_path)
+        test_file = module_dir / "tests" / "main.nf.test"
+
+        if add_arm_failure_tag(test_file):
+            success_count += 1
+
+    print(f"Successfully added ARM failure tags to {success_count} module(s)")
+
+if __name__ == "__main__":
+    main()

.github/workflows/nf-test.yml

@@ -62,8 +62,9 @@ jobs:
           head: ${{ github.sha }}
           base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
           n_parents: 0
-          exclude_tags: "gpu"
+          exclude_tags: "gpu,arm_failure"
 
+        # FIXME Can't we just do this with nf-test --filter?
       - name: Separate modules and subworkflows
         id: components
         run: |
@@ -103,8 +104,7 @@ jobs:
         profile: [conda, docker, singularity]
         arch:
           - x64
-          # FIXME
-          # - arm64
+          - arm64
     env:
       NXF_ANSI_LOG: false
       TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
@@ -158,7 +158,6 @@ jobs:
 
       - name: Run nf-test Action
         if: ${{steps.filter.outputs.filtered_paths != '[]'}}
-        continue-on-error: ${{ runner.arch == 'ARM64' }}
         uses: ./.github/actions/nf-test-action
         env:
           SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
@@ -188,7 +187,7 @@ jobs:
         run: exit 1
 
       - name: If no tests run, but filtered_paths was not empty and no tests succeeded, fail
-        if: ${{ needs.nf-test.outputs.filtered_paths != '' && !contains(needs.*.result, 'success') }}
+        if: ${{ needs.nf-test.outputs.arm_filtered_paths != '' && !contains(needs.*.result, 'success') }}
         run: exit 1
 
       - name: All tests ok
@@ -198,10 +197,10 @@ jobs:
       - name: debug-print
         if: always()
         run: |
-          echo "::group::DEBUG: `needs` Contents"
+          echo "::group::DEBUG: \`needs\` Contents"
           echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}"
           echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
-          echo "DEBUG: needs.nf-test.outputs.filtered_paths = ${{ needs.nf-test.outputs.filtered_paths }}"
+          echo "DEBUG: needs.nf-test.outputs.arm_filtered_paths = ${{ needs.nf-test.outputs.arm_filtered_paths }}"
           echo "::endgroup::"
 
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner

.github/workflows/wave.yml

@@ -50,18 +50,16 @@ jobs:
   conda-wave:
     # NOTE This should get skipped because generate-matrix won't run
     # if: github.repository == 'nf-core/modules'
-    if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }}
+    if: ${{ needs.generate-matrix.outputs.conda-matrix != '' && needs.generate-matrix.outputs.conda-matrix != '{"include":[]}' }}
     needs: generate-matrix
     name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       max-parallel: 4
-      matrix:
-        files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}"
-        profile: [docker, singularity]
-        platform: ["linux/amd64", "linux/arm64"]
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}
+
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
@@ -85,23 +83,35 @@ jobs:
               --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
               --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
 
+      - name: Mark ARM build failure
+        if: ${{ failure() && matrix.platform == 'linux/arm64' }}
+        run: |
+          MODULE_DIR=$(dirname "${{ matrix.files }}")
+
+          # Add arm_failure tag to nf-test file
+          python .github/scripts/add_arm_failure_tag.py "$MODULE_DIR"
+
+      - name: Commit ARM failure tag
+        if: ${{ failure() && matrix.platform == 'linux/arm64' }}
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "nf-core-bot"
+          git add .
+          git commit -m "Add ARM failure tag for $(dirname "${{ matrix.files }}")" || exit 0
+          git push
+
   dockerfile-wave:
     # NOTE This should get skipped because generate-matrix won't run
     # if: github.repository == 'nf-core/modules'
-    if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '[]' }}
+    if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '' && needs.generate-matrix.outputs.dockerfile-matrix != '{"include":[]}' }}
     needs: generate-matrix
     name: Build Dockerfile-based ${{ matrix.platform }} Container
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       max-parallel: 4
-      matrix:
-        files: "${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }}"
-        # NOTE singularity build requires a Singularity definition file in place of Dockerfile
-        # https://nextflow.slack.com/archives/C0477AS31T5/p1731755350230149?thread_ts=1731697852.661849&cid=C0477AS31T5
-        # profile: [docker]
-        platform: ["linux/amd64", "linux/arm64"]
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
@@ -132,6 +142,23 @@ jobs:
               --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
               --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
 
+      - name: Mark ARM build failure
+        if: ${{ failure() && matrix.platform == 'linux/arm64' }}
+        run: |
+          MODULE_DIR=$(dirname "${{ matrix.files }}")
+
+          # Add arm_failure tag to nf-test file
+          python .github/scripts/add_arm_failure_tag.py "$MODULE_DIR"
+
+      - name: Commit ARM failure tag
+        if: ${{ failure() && matrix.platform == 'linux/arm64' }}
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "nf-core-bot"
+          git add .
+          git commit -m "Add ARM failure tag for $(dirname "${{ matrix.files }}")" || exit 0
+          git push
+
   # bump-versions:
   #   needs: generate-matrix
   #   name: bump-versions

docs/ARM_BUILD_FAILURES.md

@@ -0,0 +1,171 @@
+# ARM Test Skipping
+
+This document describes the simple system for skipping ARM tests in nf-core modules.
+
+## Overview
+
+Some modules may fail to run tests on ARM64 architecture due to various reasons such as:
+- Dependencies not available for ARM64
+- Architecture-specific compilation issues
+- Upstream package limitations
+
+To handle these cases gracefully, we have implemented a simple system that allows temporarily disabling ARM tests for specific modules until the issues are resolved.
+
+## How It Works
+
+### Skip Marker Files
+
+When a module consistently fails ARM tests, you can create a `.skip-arm` file in the module directory. This file:
+
+1. **Disables ARM tests** for that module in CI/CD pipelines
+2. **Can contain a link** to the wave build failure (optional)
+3. **Can be easily removed** when the issue is resolved
+
+### Workflow Integration
+
+The GitHub Actions workflow automatically:
+- Checks for `.skip-arm` files before running tests on ARM runners
+- Skips ARM tests for modules with skip markers
+- Continues with AMD64 tests as normal
+- Logs which tests are being skipped
+
+## Managing ARM Test Skipping
+
+### Simple File Management
+
+**To skip ARM tests for a module:**
+```bash
+touch modules/nf-core/problematic-module/tests/.skip-arm
+```
+
+**To link to a wave build failure (optional):**
+```bash
+echo "https://wave.seqera.io/build/12345" > modules/nf-core/problematic-module/tests/.skip-arm
+```
+
+**To re-enable ARM tests:**
+```bash
+rm modules/nf-core/problematic-module/tests/.skip-arm
+```
+
+**To list modules with ARM tests disabled:**
+```bash
+find modules/nf-core -name ".skip-arm"
+```
+
+## File Format
+
+The `.skip-arm` file can be:
+- **Empty**: Just presence of the file disables ARM tests
+- **Contain a URL**: Link to wave build failure or GitHub issue
+- **Contain notes**: Brief description of the issue
+
+Examples:
+```bash
+# Empty file
+touch modules/nf-core/fastqc/tests/.skip-arm
+
+# With wave build failure link
+echo "https://wave.seqera.io/build/12345" > modules/nf-core/fastqc/tests/.skip-arm
+
+# With GitHub issue link
+echo "https://github.com/nf-core/modules/issues/12345" > modules/nf-core/fastqc/tests/.skip-arm
+
+# With brief note
+echo "bioconda package unavailable for ARM64" > modules/nf-core/fastqc/tests/.skip-arm
+```
+
+## Workflow Impact
+
+### Test Filtering
+
+- Modules with `.skip-arm` will have ARM tests skipped
+- AMD64 tests continue normally on all profiles (conda, docker, singularity)
+- ARM builds still happen when `environment.yml` files are changed (builds are separate from tests)
+
+### CI/CD Logs
+
+When ARM tests are skipped, you'll see logs like:
+
+```
+⚠️  Skipping ARM tests for modules/nf-core/fastqc
+```
+
+## Best Practices
+
+### When to Add Skip Markers
+
+- ARM tests consistently fail for legitimate architecture reasons
+- The failure is due to missing ARM64 packages or architecture-specific issues
+- The issue has been investigated and documented
+- An upstream issue has been reported (when applicable)
+
+### Documentation Requirements
+
+When adding a skip marker, consider including:
+- Link to wave build failure or GitHub issue in the file
+- Brief note about the reason (if not obvious from linked issue)
+
+### Regular Review
+
+Periodically review ARM test skips:
+- Check if upstream issues have been resolved
+- Test if packages have become available for ARM64
+- Remove markers when issues are fixed
+
+### Re-enabling ARM Tests
+
+To re-enable ARM tests for a module:
+1. Remove the `.skip-arm` file: `rm modules/nf-core/module/.skip-arm`
+2. Test the ARM build manually if possible
+3. Monitor the CI for successful ARM tests
+
+## Examples
+
+### Example 1: Package Not Available
+
+```bash
+# Skip ARM tests due to missing bioconda package
+echo "https://github.com/bioconda/bioconda-recipes/issues/12345" > modules/nf-core/tool/tests/.skip-arm
+```
+
+### Example 2: Wave Build Failure
+
+```bash
+# Skip ARM tests due to wave container build failure
+echo "https://wave.seqera.io/build/abc123def456" > modules/nf-core/tool/tests/.skip-arm
+```
+
+### Example 3: Simple Skip
+
+```bash
+# Just skip without detailed tracking
+touch modules/nf-core/tool/tests/.skip-arm
+```
+
+## Troubleshooting
+
+### Workflow Still Running ARM Tests
+
+If ARM tests still run despite having a skip marker:
+1. Check the file name is exactly `.skip-arm`
+2. Verify the file is in the correct module tests directory: `modules/nf-core/modulename/tests/.skip-arm`
+3. Check the workflow logs for any errors in the filtering step
+
+### Re-enabling Tests
+
+To test if ARM tests can be re-enabled:
+1. Remove the skip marker file: `rm modules/nf-core/module/tests/.skip-arm`
+2. Create a small PR that modifies the module
+3. Monitor the CI workflow for successful ARM tests
+4. If still failing, re-add the marker with updated information
+
+### Finding All Skipped Modules
+
+```bash
+# List all modules with ARM tests disabled
+find modules/nf-core -name ".skip-arm" -exec dirname {} \;
+
+# See what's in the skip files
+find modules/nf-core -name ".skip-arm" -exec echo "=== {} ===" \; -exec cat {} \;
+```