Skip to content

Commit 61489f0

Browse files
seemetherekit1980
seemethere
authored and
kit1980
committed
ci: Add generic windows_job (#887)
1 parent 2272244 commit 61489f0

File tree

4 files changed

+272
-0
lines changed

4 files changed

+272
-0
lines changed

‎.github/actions/setup-windows/action.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Setup Windows
2+
3+
description: Set up for windows jobs
4+
5+
inputs:
6+
cuda-version:
7+
description: which cuda version to install, 'cpu' for none
8+
required: true
9+
10+
runs:
11+
using: composite
12+
steps:
13+
- name: Display EC2 information
14+
shell: bash
15+
run: |
16+
set -euo pipefail
17+
function get_ec2_metadata() {
18+
# Pulled from instance metadata endpoint for EC2
19+
# see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
20+
category=$1
21+
curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
22+
}
23+
echo "ami-id: $(get_ec2_metadata ami-id)"
24+
echo "instance-id: $(get_ec2_metadata instance-id)"
25+
echo "instance-type: $(get_ec2_metadata instance-type)"
26+
echo "system info $(uname -a)"
27+
28+
# Needed for binary builds, see: https://github.com/pytorch/pytorch/issues/73339#issuecomment-1058981560
29+
- name: Enable long paths on Windows
30+
shell: powershell
31+
run: |
32+
Set-ItemProperty -Path "HKLM:\\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1
33+
34+
# Since it's just a defensive command, the workflow should continue even the command fails
35+
- name: Disables Windows Defender scheduled and real-time scanning for files in pytorch directory.
36+
shell: powershell
37+
run: |
38+
Add-MpPreference -ExclusionPath $(Get-Location).tostring() -ErrorAction Ignore
39+
40+
- name: Setup useful environment variables
41+
shell: bash
42+
working-directory: ${{ inputs.repository }}
43+
run: |
44+
RUNNER_ARTIFACT_DIR="$(cygpath ${RUNNER_TEMP})/artifacts"
45+
mkdir -p "${RUNNER_ARTIFACT_DIR}"
46+
echo "RUNNER_ARTIFACT_DIR=$(cygpath ${RUNNER_TEMP})/artifacts" >> "${GITHUB_ENV}"

‎.github/actions/teardown-windows/action.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Teardown Windows
2+
3+
description: Set up Docker workspace on linux
4+
5+
inputs:
6+
extra-delete-dir:
7+
description: If set, cleaning up the workspace will delete this too
8+
required: false
9+
default: ""
10+
11+
runs:
12+
using: composite
13+
steps:
14+
- name: Wait until all sessions have drained
15+
shell: powershell
16+
if: always()
17+
run: |
18+
function Get-SSH-Users {
19+
# Gets ssh sessions for all users not named SYSTEM
20+
Get-CimInstance -ClassName Win32_Process -Filter "Name = 'sshd.exe'" |
21+
Get-CimAssociatedInstance -Association Win32_SessionProcess |
22+
Get-CimAssociatedInstance -Association Win32_LoggedOnUser |
23+
Where-Object {$_.Name -ne 'SYSTEM'} |
24+
Measure-Object
25+
}
26+
27+
$usersLoggedOn = Get-SSH-Users
28+
29+
Write-Output "Holding runner until all ssh sessions have logged out"
30+
while ($usersLoggedOn.Count -gt 0) {
31+
$usersLoggedOn = Get-SSH-Users
32+
Write-Output "."
33+
Start-Sleep -s 5
34+
}
35+
36+
- name: Kill active ssh sessions if still around (Useful if workflow was cancelled)
37+
shell: powershell
38+
if: always()
39+
run: |
40+
function Get-SSH-Sessions {
41+
Get-Process sshd -IncludeUserName |
42+
Where-Object UserName -notLike "*SYSTEM*" |
43+
Select-Object Id
44+
}
45+
46+
$runningSessions = Get-SSH-Sessions
47+
48+
foreach ($session in $runningSessions) {
49+
Stop-Process -id $session.Id
50+
}

‎.github/workflows/test_windows_job.yml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Test build/test windows workflow
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- .github/workflows/windows_job.yml
7+
- .github/workflows/test_windows_job.yml
8+
workflow_dispatch:
9+
10+
jobs:
11+
test-cpu:
12+
uses: ./.github/workflows/windows_job.yml
13+
with:
14+
runner: windows.4xlarge
15+
test-infra-repository: ${{ github.repository }}
16+
test-infra-ref: ${{ github.ref }}
17+
script: |
18+
conda create -y -n test python=3.8
19+
conda activate test
20+
python -m pip install --extra-index-url https://download.pytorch.org/whl/nightly/cpu --pre torch
21+
# Can import pytorch
22+
python -c 'import torch'
23+
test-gpu:
24+
uses: ./.github/workflows/windows_job.yml
25+
with:
26+
runner: windows.8xlarge.nvidia.gpu
27+
test-infra-repository: ${{ github.repository }}
28+
test-infra-ref: ${{ github.ref }}
29+
timeout: 60
30+
script: |
31+
conda create -y -n test python=3.8
32+
conda activate test
33+
python -m pip install --extra-index-url https://download.pytorch.org/whl/nightly/cu116 --pre torch
34+
# Can import pytorch, cuda is available
35+
python -c 'import torch;assert(torch.cuda.is_available())'
36+
test-upload-artifact:
37+
uses: ./.github/workflows/windows_job.yml
38+
with:
39+
runner: windows.4xlarge
40+
test-infra-repository: ${{ github.repository }}
41+
test-infra-ref: ${{ github.ref }}
42+
upload-artifact: my-cool-artifact
43+
script: |
44+
echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
45+
test-download-artifact:
46+
needs: test-upload-artifact
47+
uses: ./.github/workflows/windows_job.yml
48+
with:
49+
runner: windows.4xlarge
50+
test-infra-repository: ${{ github.repository }}
51+
test-infra-ref: ${{ github.ref }}
52+
download-artifact: my-cool-artifact
53+
script: |
54+
grep "hello" "${RUNNER_ARTIFACT_DIR}/cool_beans"

‎.github/workflows/windows_job.yml

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
name: Run a Windows job
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
script:
7+
description: 'Script to utilize'
8+
default: "python setup.py bdist_wheel"
9+
type: string
10+
timeout:
11+
description: 'Timeout for the job (in minutes)'
12+
default: 30
13+
type: number
14+
runner:
15+
description: 'Runner type to utilize'
16+
default: "windows.4xlarge"
17+
type: string
18+
upload-artifact:
19+
description: 'Name to give artifacts uploaded from ${RUNNER_ARTIFACT_DIR}'
20+
default: ''
21+
type: string
22+
download-artifact:
23+
description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}'
24+
default: ''
25+
type: string
26+
repository:
27+
description: 'Repository to checkout, defaults to ""'
28+
default: ""
29+
type: string
30+
ref:
31+
description: 'Reference to checkout, defaults to "nightly"'
32+
default: ""
33+
type: string
34+
test-infra-repository:
35+
description: "Test infra repository to use"
36+
default: "pytorch/test-infra"
37+
type: string
38+
test-infra-ref:
39+
description: "Test infra reference to use"
40+
default: ""
41+
type: string
42+
43+
jobs:
44+
job:
45+
env:
46+
REPOSITORY: ${{ inputs.repository || github.repository }}
47+
SCRIPT: ${{ inputs.script }}
48+
runs-on: ${{ inputs.runner }}
49+
timeout-minutes: ${{ inputs.timeout }}
50+
steps:
51+
- name: Checkout repository (${{ inputs.test-infra-repository }}@${{ inputs.test-infra-ref }})
52+
uses: actions/checkout@v3
53+
with:
54+
# Support the use case where we need to checkout someone's fork
55+
repository: ${{ inputs.test-infra-repository }}
56+
ref: ${{ inputs.test-infra-ref }}
57+
path: test-infra
58+
59+
- name: Setup Windows
60+
uses: ./test-infra/.github/actions/setup-windows
61+
62+
- name: Setup SSH
63+
uses: ./test-infra/.github/actions/setup-ssh
64+
65+
- name: Checkout repository (${{ inputs.repository || github.repository }}@${{ inputs.ref }})
66+
uses: actions/checkout@v3
67+
with:
68+
# Support the use case where we need to checkout someone's fork
69+
repository: ${{ inputs.repository || github.repository }}
70+
ref: ${{ inputs.ref || github.ref }}
71+
path: ${{ inputs.repository || github.repository }}
72+
73+
- name: Download artifacts (if any)
74+
uses: actions/download-artifact@v3
75+
if: ${{ inputs.download-artifact != '' }}
76+
with:
77+
name: ${{ inputs.download-artifact }}
78+
path: ${{ runner.temp }}/artifacts/
79+
80+
- name: Run script
81+
shell: bash -l {0}
82+
working-directory: ${{ inputs.repository }}
83+
run: |
84+
{
85+
echo "#!/usr/bin/env bash";
86+
echo "set -eou pipefail";
87+
# Without this specific version of pywin32 conda the default conda installation does not work
88+
# See https://github.com/conda/conda/issues/11503
89+
echo "/c/Jenkins/Miniconda3/python.exe -m pip install --upgrade pywin32==304"
90+
# Source conda so it's available to the script environment
91+
echo "source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh";
92+
echo "${SCRIPT}";
93+
} > "${RUNNER_TEMP}/exec_script"
94+
bash "${RUNNER_TEMP}/exec_script"
95+
96+
- name: Check if there are potential artifacts and move them to the correct artifact location
97+
shell: bash -l {0}
98+
working-directory: ${{ inputs.repository }}
99+
id: check-artifacts
100+
if: ${{ inputs.upload-artifact != '' }}
101+
env:
102+
UPLOAD_ARTIFACT_NAME: ${{ inputs.upload-artifact }}
103+
run: |
104+
# If the default execution path is followed then we should get a wheel in the dist/ folder
105+
# attempt to just grab whatever is in there and scoop it all up
106+
if find "dist/" -name "*.whl" >/dev/null 2>/dev/null; then
107+
mv -v dist/*.whl "${RUNNER_ARTIFACT_DIR}/"
108+
fi
109+
# Set to fail upload step if there are no files for upload and expected files for upload
110+
echo '::set-output name=if-no-files-found::error'
111+
112+
- name: Upload artifacts to GitHub (if any)
113+
uses: actions/upload-artifact@v3
114+
if: ${{ inputs.upload-artifact != '' }}
115+
with:
116+
name: ${{ inputs.upload-artifact }}
117+
path: ${{ runner.temp }}/artifacts/
118+
if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
119+
120+
- name: Teardown Windows
121+
if: ${{ always() }}
122+
uses: ./test-infra/.github/actions/teardown-windows

0 commit comments

Comments
 (0)