GH-113464: Get LLVM from cpython-bin-deps on Windows (GH-133278)

Author: brandtbucher

.github/workflows/jit.yml

@@ -95,10 +95,10 @@ jobs:
         with:
           python-version: '3.11'
 
+      # PCbuild downloads LLVM automatically:
       - name: Windows
         if: runner.os == 'Windows'
         run: |
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst

@@ -0,0 +1,3 @@
+Use the cpython-bin-deps "externals" repository for Windows LLVM dependency
+management. Installing LLVM manually is no longer necessary for Windows JIT
+builds.

PCbuild/build.bat

@@ -111,6 +111,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true
 if "%IncludeCTypes%"=="" set IncludeCTypes=true
 if "%IncludeSSL%"=="" set IncludeSSL=true
 if "%IncludeTkinter%"=="" set IncludeTkinter=true
+if "%UseJIT%" NEQ "true" set IncludeLLVM=false
 
 if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
 

PCbuild/get_externals.bat

@@ -15,6 +15,7 @@ set IncludeSSLSrc=false
 if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
 if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
 if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts
+if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts
 if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts
 if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts
 if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts
@@ -80,6 +81,7 @@ if NOT "%IncludeLibffi%"=="false"  set binaries=%binaries% libffi-3.4.4
 if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-3.0.16.2
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
+if NOT "%IncludeLLVM%"=="false"    set binaries=%binaries% llvm-19.1.7.0
 
 for %%b in (%binaries%) do (
     if exist "%EXTERNALS_DIR%\%%b" (
@@ -98,7 +100,7 @@ goto end
 
 :usage
 echo.Valid options: -c, --clean, --clean-only, --organization, --python,
-echo.--no-tkinter, --no-openssl
+echo.--no-tkinter, --no-openssl, --no-llvm
 echo.
 echo.Pull all sources and binaries necessary for compiling optional extension
 echo.modules that rely on external libraries.

PCbuild/regen.targets

@@ -30,7 +30,11 @@
     <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
     <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
     <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/>
+    <!-- Need to explicitly enumerate these, since globbing doesn't work for missing outputs: -->
     <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
+    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/>
+    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/>
+    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/>
     <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/>
     <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/>
     <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" />
@@ -124,6 +128,9 @@
     <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)'
           WorkingDirectory="$(GeneratedPyConfigDir)"/>
   </Target>
+  <Target Name="_CleanJIT" AfterTargets="Clean">
+    <Delete Files="@(_JITOutputs)"/>
+  </Target>
 
   <Target Name="_RegenNoPGUpdate"
           Condition="$(Configuration) != 'PGUpdate'"

Tools/jit/README.md

@@ -41,7 +41,9 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri
 
 ### Windows
 
-Install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
+LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`.
+
+Otherwise, you can install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
 
 Alternatively, you can use [chocolatey](https://chocolatey.org):
 

Tools/jit/_llvm.py

@@ -8,8 +8,11 @@
 import subprocess
 import typing
 
+import _targets
+
 _LLVM_VERSION = 19
 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+")
+_EXTERNALS_LLVM_TAG = "llvm-19.1.7.0"
 
 _P = typing.ParamSpec("_P")
 _R = typing.TypeVar("_R")
@@ -72,6 +75,11 @@ async def _find_tool(tool: str, *, echo: bool = False) -> str | None:
         return path
     # Versioned executables:
     path = f"{tool}-{_LLVM_VERSION}"
+    if await _check_tool_version(path, echo=echo):
+        return path
+    # PCbuild externals:
+    externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS)
+    path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool)
     if await _check_tool_version(path, echo=echo):
         return path
     # Homebrew-installed executables:

Tools/jit/_targets.py

@@ -23,8 +23,10 @@
 TOOLS_JIT = TOOLS_JIT_BUILD.parent
 TOOLS = TOOLS_JIT.parent
 CPYTHON = TOOLS.parent
+EXTERNALS = CPYTHON / "externals"
 PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
 TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
+
 ASYNCIO_RUNNER = asyncio.Runner()
 
 _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)

Tools/jit/build.py

@@ -8,6 +8,7 @@
 import _targets
 
 if __name__ == "__main__":
+    out = pathlib.Path.cwd().resolve()
     comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}"
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -31,17 +32,22 @@
         target.force = args.force
         target.verbose = args.verbose
         target.build(
-            pathlib.Path.cwd(),
+            out,
             comment=comment,
             stencils_h=f"jit_stencils-{target.triple}.h",
             force=args.force,
         )
-
-    with open("jit_stencils.h", "w") as fp:
-        for idx, target in enumerate(args.target):
-            fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n")
-            fp.write(f'#include "jit_stencils-{target.triple}.h"\n')
-
-        fp.write("#else\n")
-        fp.write('#error "unexpected target"\n')
-        fp.write("#endif\n")
+    jit_stencils_h = out / "jit_stencils.h"
+    lines = [f"// {comment}\n"]
+    guard = "#if"
+    for target in args.target:
+        lines.append(f"{guard} {target.condition}\n")
+        lines.append(f'#include "jit_stencils-{target.triple}.h"\n')
+        guard = "#elif"
+    lines.append("#else\n")
+    lines.append('#error "unexpected target"\n')
+    lines.append("#endif\n")
+    body = "".join(lines)
+    # Don't touch the file if it hasn't changed (so we don't trigger a rebuild):
+    if not jit_stencils_h.is_file() or jit_stencils_h.read_text() != body:
+        jit_stencils_h.write_text(body)