GH-127429: fix sysconfig data generation on cross-builds (#127430)

Author: FFY00

.github/workflows/jit.yml

@@ -104,7 +104,7 @@ jobs:
 
         # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
         # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
-        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes 
+        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
         # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
       - name: Native macOS
         if: runner.os == 'macOS'

Lib/sysconfig/__init__.py

@@ -318,14 +318,22 @@ def get_default_scheme():
 
 def get_makefile_filename():
     """Return the path of the Makefile."""
+
+    # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python.
+    if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'):
+        return os.path.join(cross_base, 'Makefile')
+
     if _PYTHON_BUILD:
         return os.path.join(_PROJECT_BASE, "Makefile")
+
     if hasattr(sys, 'abiflags'):
         config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}'
     else:
         config_dir_name = 'config'
+
     if hasattr(sys.implementation, '_multiarch'):
         config_dir_name += f'-{sys.implementation._multiarch}'
+
     return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile')
 
 
@@ -464,27 +472,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True):
 def _init_config_vars():
     global _CONFIG_VARS
     _CONFIG_VARS = {}
+
+    prefix = _PREFIX
+    exec_prefix = _EXEC_PREFIX
+    base_prefix = _BASE_PREFIX
+    base_exec_prefix = _BASE_EXEC_PREFIX
+
+    try:
+        abiflags = sys.abiflags
+    except AttributeError:
+        abiflags = ''
+
+    if os.name == 'posix':
+        _init_posix(_CONFIG_VARS)
+        # If we are cross-compiling, load the prefixes from the Makefile instead.
+        if '_PYTHON_PROJECT_BASE' in os.environ:
+            prefix = _CONFIG_VARS['prefix']
+            exec_prefix = _CONFIG_VARS['exec_prefix']
+            base_prefix = _CONFIG_VARS['prefix']
+            base_exec_prefix = _CONFIG_VARS['exec_prefix']
+            abiflags = _CONFIG_VARS['ABIFLAGS']
+
     # Normalized versions of prefix and exec_prefix are handy to have;
     # in fact, these are the standard versions used most places in the
     # Distutils.
-    _CONFIG_VARS['prefix'] = _PREFIX
-    _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX
+    _CONFIG_VARS['prefix'] = prefix
+    _CONFIG_VARS['exec_prefix'] = exec_prefix
     _CONFIG_VARS['py_version'] = _PY_VERSION
     _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT
     _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT
-    _CONFIG_VARS['installed_base'] = _BASE_PREFIX
-    _CONFIG_VARS['base'] = _PREFIX
-    _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX
-    _CONFIG_VARS['platbase'] = _EXEC_PREFIX
+    _CONFIG_VARS['installed_base'] = base_prefix
+    _CONFIG_VARS['base'] = prefix
+    _CONFIG_VARS['installed_platbase'] = base_exec_prefix
+    _CONFIG_VARS['platbase'] = exec_prefix
     _CONFIG_VARS['projectbase'] = _PROJECT_BASE
     _CONFIG_VARS['platlibdir'] = sys.platlibdir
     _CONFIG_VARS['implementation'] = _get_implementation()
     _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
-    try:
-        _CONFIG_VARS['abiflags'] = sys.abiflags
-    except AttributeError:
-        # sys.abiflags may not be defined on all platforms.
-        _CONFIG_VARS['abiflags'] = ''
+    _CONFIG_VARS['abiflags'] = abiflags
     try:
         _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '')
     except AttributeError:
@@ -493,8 +518,6 @@ def _init_config_vars():
     if os.name == 'nt':
         _init_non_posix(_CONFIG_VARS)
         _CONFIG_VARS['VPATH'] = sys._vpath
-    if os.name == 'posix':
-        _init_posix(_CONFIG_VARS)
     if _HAS_USER_BASE:
         # Setting 'userbase' is done below the call to the
         # init function to enable using 'get_config_var' in

Lib/sysconfig/__main__.py

@@ -7,6 +7,7 @@
     _PYTHON_BUILD,
     _get_sysconfigdata_name,
     get_config_h_filename,
+    get_config_var,
     get_config_vars,
     get_default_scheme,
     get_makefile_filename,
@@ -161,7 +162,7 @@ def _print_config_dict(d, stream):
 
 def _get_pybuilddir():
     pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}'
-    if hasattr(sys, "gettotalrefcount"):
+    if get_config_var('Py_DEBUG') == '1':
         pybuilddir += '-pydebug'
     return pybuilddir
 
@@ -229,11 +230,15 @@ def _generate_posix_vars():
         f.write('build_time_vars = ')
         _print_config_dict(vars, stream=f)
 
+    print(f'Written {destfile}')
+
     # Write a JSON file with the output of sysconfig.get_config_vars
     jsonfile = os.path.join(pybuilddir, _get_json_data_name())
     with open(jsonfile, 'w') as f:
         json.dump(get_config_vars(), f, indent=2)
 
+    print(f'Written {jsonfile}')
+
     # Create file used for sys.path fixup -- see Modules/getpath.c
     with open('pybuilddir.txt', 'w', encoding='utf8') as f:
         f.write(pybuilddir)

Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst

@@ -0,0 +1,3 @@
+Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being
+generated with the host Python's ``Makefile``. The data is now generated from
+current build's ``Makefile``.

configure

@@ -1609,8 +1609,8 @@ if test "$cross_compiling" = yes; then
   RUNSHARED=
 fi
 
+# HOSTRUNNER - Program to run CPython for the host platform
 AC_MSG_CHECKING([HOSTRUNNER])
-AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform])
 if test -z "$HOSTRUNNER"
 then
   AS_CASE([$ac_sys_system],