Skip to content

Commit 973b7ab

Browse files
marcromeynmarcromeyn
authored
Merge branch 'main' into feature/wsd_scheduler
2 parents ec5a4d2 + c9a0453 commit 973b7ab

File tree

222 files changed

+11180
-3579
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

222 files changed

+11180
-3579
lines changed

‎.github/workflows/_test_template.yml

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ on:
2626
description: Failure will cancel all other tests if set to true
2727
required: false
2828
default: false
29-
SAVE_COVERAGE_REPORT:
29+
IS_UNIT_TEST:
3030
type: boolean
31-
description: Upload .coverage file
31+
description: Upload coverage as unit test
3232
required: false
3333
default: false
3434
outputs:
@@ -93,19 +93,11 @@ jobs:
9393
env:
9494
DIR: ${{ github.run_id }}
9595
SCRIPT: ${{ inputs.SCRIPT }}
96+
IS_UNIT_TEST: ${{ inputs.IS_UNIT_TEST }}
9697
id: create
9798
run: |
98-
SCRIPT=$(echo "$SCRIPT" | grep -v '^#')
99-
SCRIPT=$(perl -pe 'chomp if eof' <<< "$SCRIPT")
100-
[[ "${SCRIPT: -1}" == ";" ]] && SCRIPT="${SCRIPT::-1}"
10199
102-
if [[ "$SCRIPT" == *pytest* ]]; then
103-
COVERAGE_PREFIX="unit-test"
104-
else
105-
COVERAGE_PREFIX="e2e"
106-
SCRIPT=$(echo "$SCRIPT" | sed 's|python|coverage run --branch -a --data-file=/workspace/.coverage --source=/workspace/nemo|g')
107-
108-
fi
100+
COVERAGE_PREFIX=$([[ "$IS_UNIT_TEST" == "true" ]] && echo "unit-test" || echo "e2e")
109101
echo "coverage-prefix=$COVERAGE_PREFIX" | tee -a "$GITHUB_OUTPUT"
110102
111103
mkdir -p $DIR
@@ -140,7 +132,6 @@ jobs:
140132
- name: Check result
141133
id: check
142134
env:
143-
SAVE_COVERAGE_REPORT: ${{ inputs.SAVE_COVERAGE_REPORT }}
144135
IS_OPTIONAL: ${{ inputs.IS_OPTIONAL }}
145136
run: |
146137
cat $DIR/err.log

‎.github/workflows/cicd-main.yml

Lines changed: 585 additions & 1645 deletions
Large diffs are not rendered by default.

‎.github/workflows/code-formatting.yml

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ name: Isort and Black Formatting
1212
on:
1313
pull_request_target:
1414
paths:
15-
- '**.py'
16-
types: [ opened, synchronize, reopened, labeled, unlabeled ]
15+
- "**.py"
16+
types: [opened, synchronize, reopened, labeled, unlabeled]
1717

1818
defaults:
1919
run:
@@ -36,38 +36,38 @@ jobs:
3636
# custom token is required to trigger actions after reformatting + pushing
3737
token: ${{ secrets.NEMO_REFORMAT_TOKEN }}
3838

39-
# https://github.com/tj-actions/changed-files
40-
- name: Get changed files
41-
id: changed-files
42-
uses: tj-actions/changed-files@v44
43-
with:
44-
files: |
45-
**.py
39+
# # https://github.com/tj-actions/changed-files
40+
# - name: Get changed files
41+
# id: changed-files
42+
# uses: tj-actions/changed-files@v44
43+
# with:
44+
# files: |
45+
# **.py
4646

47-
- name: Setup Python env
48-
uses: actions/setup-python@v5
49-
with:
50-
python-version: "3.10"
47+
# - name: Setup Python env
48+
# uses: actions/setup-python@v5
49+
# with:
50+
# python-version: "3.10"
5151

52-
- name: black
53-
uses: psf/black@stable
54-
if: ${{ steps.changed-files.outputs.any_changed == 'true' }}
55-
with:
56-
options: "--verbose"
57-
# apply only to changed files (pass explicitly the files)
58-
src: "${{ steps.changed-files.outputs.all_changed_files }}"
59-
version: "~= 24.3"
52+
# - name: black
53+
# uses: psf/black@stable
54+
# if: ${{ steps.changed-files.outputs.any_changed == 'true' }}
55+
# with:
56+
# options: "--verbose"
57+
# # apply only to changed files (pass explicitly the files)
58+
# src: "${{ steps.changed-files.outputs.all_changed_files }}"
59+
# version: "~= 24.3"
6060

61-
- name: isort
62-
uses: isort/isort-action@v1
63-
if: ${{ steps.changed-files.outputs.any_changed == 'true' }}
64-
with:
65-
isort-version: "5.13.2"
66-
# reformat all files with isort – safe since the whole repo is already reformatted
67-
configuration: ""
61+
# - name: isort
62+
# uses: isort/isort-action@v1
63+
# if: ${{ steps.changed-files.outputs.any_changed == 'true' }}
64+
# with:
65+
# isort-version: "5.13.2"
66+
# # reformat all files with isort – safe since the whole repo is already reformatted
67+
# configuration: ""
6868

69-
- uses: EndBug/add-and-commit@v9
70-
# Commit changes. Nothing is committed if no changes.
71-
with:
72-
message: Apply isort and black reformatting
73-
commit: --signoff
69+
# - uses: EndBug/add-and-commit@v9
70+
# # Commit changes. Nothing is committed if no changes.
71+
# with:
72+
# message: Apply isort and black reformatting
73+
# commit: --signoff

‎.github/workflows/code-linting.yml

Lines changed: 116 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -19,121 +19,122 @@ jobs:
1919
- name: Checkout
2020
uses: actions/checkout@v4
2121

22-
- name: Select filter
23-
id: filter
24-
run: |
25-
if [[ "$DOMAIN" == "speech" ]]; then
26-
FILTER=$(jq -crn '[
27-
"nemo/collections/asr/**/*.py",
28-
"nemo/collections/tts/**/*.py",
29-
"nemo/collections/audio/**/*.py",
30-
"nemo/collections/multimodal/speech_llm/**/*.py",
31-
"nemo/collections/speechlm/**/*.py"
32-
] | join(",")')
33-
34-
else
35-
FILTER=$(jq -crn '[
36-
"nemo/**/*.py",
37-
"!nemo/collections/asr/**/*.py",
38-
"!nemo/collections/tts/**/*.py",
39-
"!nemo/collections/audio/**/*.py",
40-
"!nemo/collections/multimodal/speech_llm/**/*.py",
41-
"!nemo/collections/speechlm/**/*.py"
42-
] | join(",")')
43-
fi
44-
45-
echo "main=$FILTER" | tee -a "$GITHUB_OUTPUT"
46-
47-
- name: Get changed files
48-
id: changed-files
49-
uses: tj-actions/changed-files@v44
50-
with:
51-
files: ${{ steps.filter.outputs.main }}
52-
files_separator: ","
53-
separator: " "
54-
55-
- name: Run PyLint
56-
id: pylint
57-
env:
58-
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
59-
SKIP_DOCS: ${{ contains(github.event.pull_request.labels.*.name, 'skip-docs') }}
60-
SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }}
61-
run: |
62-
if [[ -z "$CHANGED_FILES" ]]; then
63-
echo Nothing to lint.
64-
echo "exit-code=0" | tee -a "$GITHUB_OUTPUT"
65-
exit 0
66-
fi
67-
68-
if [[ $SKIP_DOCS == true ]]; then
69-
ADDITIONAL_PYLINT_ARGS="--disable=C0115,C0116"
70-
else
71-
ADDITIONAL_PYLINT_ARGS=""
72-
fi
73-
74-
if [[ $SKIP_LINTING == true ]]; then
75-
ADDITIONAL_PYLINT_ARGS="--exit-zero"
76-
fi
77-
78-
pip install pylint
79-
set +e
80-
pylint $ADDITIONAL_PYLINT_ARGS --output "pylintrc.$DOMAIN.txt" --rcfile ".pylintrc.$DOMAIN" ${CHANGED_FILES[@]}
81-
echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT"
82-
83-
- name: Run flake8
84-
id: flake8
85-
env:
86-
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
87-
SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }}
88-
run: |
89-
if [[ -z "$CHANGED_FILES" ]]; then
90-
echo Nothing to lint.
91-
echo "exit-code=0" | tee -a "$GITHUB_OUTPUT"
92-
exit 0
93-
fi
94-
95-
if [[ $SKIP_LINTING == true ]]; then
96-
ADDITIONAL_FLAKE8_ARGS="--exit-zero"
97-
else
98-
ADDITIONAL_FLAKE8_ARGS=""
99-
fi
100-
101-
pip install flake8
102-
set +e
103-
flake8 $ADDITIONAL_FLAKE8_ARGS --output "flake8.$DOMAIN.txt" --config ".flake8.$DOMAIN" ${CHANGED_FILES[@]}
104-
echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT"
105-
106-
- name: Summary
107-
env:
108-
PYLINT: ${{ steps.pylint.outputs.exit-code == 0 }}
109-
FLAKE8: ${{ steps.flake8.outputs.exit-code == 0 }}
110-
run: |
111-
112-
if [[ "$PYLINT" != "true" ]]; then
113-
echo "Pylint output:" | tee -a $GITHUB_STEP_SUMMARY
114-
115-
echo '```' | tee -a $GITHUB_STEP_SUMMARY
116-
cat pylintrc.$DOMAIN.txt | tee -a $GITHUB_STEP_SUMMARY
117-
echo '```' | tee -a $GITHUB_STEP_SUMMARY
118-
fi
119-
120-
if [[ "$FLAKE8" != "true" ]]; then
121-
echo "Flake8 output:" | tee -a $GITHUB_STEP_SUMMARY
122-
123-
echo '```' | tee -a $GITHUB_STEP_SUMMARY
124-
cat flake8.$DOMAIN.txt | tee -a $GITHUB_STEP_SUMMARY
125-
echo '```' | tee -a $GITHUB_STEP_SUMMARY
126-
fi
127-
128-
if [[ "$PYLINT" != "true" || "$FLAKE8" != "true" ]]; then
129-
echo "The following directories got scanned:" | tee -a $GITHUB_STEP_SUMMARY
130-
131-
echo '```' | tee -a $GITHUB_STEP_SUMMARY
132-
echo ${{ steps.filter.outputs.main }} | tee -a $GITHUB_STEP_SUMMARY
133-
echo '```' | tee -a $GITHUB_STEP_SUMMARY
134-
135-
exit 1
136-
fi
22+
# Until tj-actions/changed-files is re-enabled.
23+
# - name: Select filter
24+
# id: filter
25+
# run: |
26+
# if [[ "$DOMAIN" == "speech" ]]; then
27+
# FILTER=$(jq -crn '[
28+
# "nemo/collections/asr/**/*.py",
29+
# "nemo/collections/tts/**/*.py",
30+
# "nemo/collections/audio/**/*.py",
31+
# "nemo/collections/multimodal/speech_llm/**/*.py",
32+
# "nemo/collections/speechlm/**/*.py"
33+
# ] | join(",")')
34+
35+
# else
36+
# FILTER=$(jq -crn '[
37+
# "nemo/**/*.py",
38+
# "!nemo/collections/asr/**/*.py",
39+
# "!nemo/collections/tts/**/*.py",
40+
# "!nemo/collections/audio/**/*.py",
41+
# "!nemo/collections/multimodal/speech_llm/**/*.py",
42+
# "!nemo/collections/speechlm/**/*.py"
43+
# ] | join(",")')
44+
# fi
45+
46+
# echo "main=$FILTER" | tee -a "$GITHUB_OUTPUT"
47+
48+
# - name: Get changed files
49+
# id: changed-files
50+
# uses: tj-actions/changed-files@v44
51+
# with:
52+
# files: ${{ steps.filter.outputs.main }}
53+
# files_separator: ","
54+
# separator: " "
55+
56+
# - name: Run PyLint
57+
# id: pylint
58+
# env:
59+
# CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
60+
# SKIP_DOCS: ${{ contains(github.event.pull_request.labels.*.name, 'skip-docs') }}
61+
# SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }}
62+
# run: |
63+
# if [[ -z "$CHANGED_FILES" ]]; then
64+
# echo Nothing to lint.
65+
# echo "exit-code=0" | tee -a "$GITHUB_OUTPUT"
66+
# exit 0
67+
# fi
68+
69+
# if [[ $SKIP_DOCS == true ]]; then
70+
# ADDITIONAL_PYLINT_ARGS="--disable=C0115,C0116"
71+
# else
72+
# ADDITIONAL_PYLINT_ARGS=""
73+
# fi
74+
75+
# if [[ $SKIP_LINTING == true ]]; then
76+
# ADDITIONAL_PYLINT_ARGS="--exit-zero"
77+
# fi
78+
79+
# pip install pylint
80+
# set +e
81+
# pylint $ADDITIONAL_PYLINT_ARGS --output "pylintrc.$DOMAIN.txt" --rcfile ".pylintrc.$DOMAIN" ${CHANGED_FILES[@]}
82+
# echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT"
83+
84+
# - name: Run flake8
85+
# id: flake8
86+
# env:
87+
# CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
88+
# SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }}
89+
# run: |
90+
# if [[ -z "$CHANGED_FILES" ]]; then
91+
# echo Nothing to lint.
92+
# echo "exit-code=0" | tee -a "$GITHUB_OUTPUT"
93+
# exit 0
94+
# fi
95+
96+
# if [[ $SKIP_LINTING == true ]]; then
97+
# ADDITIONAL_FLAKE8_ARGS="--exit-zero"
98+
# else
99+
# ADDITIONAL_FLAKE8_ARGS=""
100+
# fi
101+
102+
# pip install flake8
103+
# set +e
104+
# flake8 $ADDITIONAL_FLAKE8_ARGS --output "flake8.$DOMAIN.txt" --config ".flake8.$DOMAIN" ${CHANGED_FILES[@]}
105+
# echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT"
106+
107+
# - name: Summary
108+
# env:
109+
# PYLINT: ${{ steps.pylint.outputs.exit-code == 0 }}
110+
# FLAKE8: ${{ steps.flake8.outputs.exit-code == 0 }}
111+
# run: |
112+
113+
# if [[ "$PYLINT" != "true" ]]; then
114+
# echo "Pylint output:" | tee -a $GITHUB_STEP_SUMMARY
115+
116+
# echo '```' | tee -a $GITHUB_STEP_SUMMARY
117+
# cat pylintrc.$DOMAIN.txt | tee -a $GITHUB_STEP_SUMMARY
118+
# echo '```' | tee -a $GITHUB_STEP_SUMMARY
119+
# fi
120+
121+
# if [[ "$FLAKE8" != "true" ]]; then
122+
# echo "Flake8 output:" | tee -a $GITHUB_STEP_SUMMARY
123+
124+
# echo '```' | tee -a $GITHUB_STEP_SUMMARY
125+
# cat flake8.$DOMAIN.txt | tee -a $GITHUB_STEP_SUMMARY
126+
# echo '```' | tee -a $GITHUB_STEP_SUMMARY
127+
# fi
128+
129+
# if [[ "$PYLINT" != "true" || "$FLAKE8" != "true" ]]; then
130+
# echo "The following directories got scanned:" | tee -a $GITHUB_STEP_SUMMARY
131+
132+
# echo '```' | tee -a $GITHUB_STEP_SUMMARY
133+
# echo ${{ steps.filter.outputs.main }} | tee -a $GITHUB_STEP_SUMMARY
134+
# echo '```' | tee -a $GITHUB_STEP_SUMMARY
135+
136+
# exit 1
137+
# fi
137138

138139
Nemo_Linting_Test:
139140
needs: linting

0 commit comments

Comments
 (0)