Merge branch 'bitcoin' into auxpow

Author: domob1812

.github/ci-test-each-commit-exec.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+import subprocess
+import sys
+import shlex
+
+
+def run(cmd, **kwargs):
+    print("+ " + shlex.join(cmd), flush=True)
+    try:
+        return subprocess.run(cmd, check=True, **kwargs)
+    except Exception as e:
+        sys.exit(e)
+
+
+def main():
+    print("Running test-one-commit on ...")
+    run(["git", "log", "-1"])
+
+    num_procs = int(run(["nproc"], stdout=subprocess.PIPE).stdout)
+
+    # Use clang++, because it is a bit faster and uses less memory than g++
+    run([
+        "cmake",
+        "-B",
+        "build",
+        "-DCMAKE_C_COMPILER=clang",
+        "-DCMAKE_CXX_COMPILER=clang++",
+        "-DWERROR=ON",
+        "-DWITH_ZMQ=ON",
+        "-DBUILD_GUI=ON",
+        "-DBUILD_BENCH=ON",
+        "-DBUILD_FUZZ_BINARY=ON",
+        "-DWITH_USDT=ON",
+        "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function",
+    ])
+    run(["cmake", "--build", "build", "-j", str(num_procs)])
+    run([
+        "ctest",
+        "--output-on-failure",
+        "--stop-on-failure",
+        "--test-dir",
+        "build",
+        "-j",
+        str(num_procs),
+    ])
+    run([
+        sys.executable,
+        "./build/test/functional/test_runner.py",
+        "-j",
+        str(num_procs * 2),
+        "--combinedlogslen=99999999",
+    ])
+
+
+if __name__ == "__main__":
+    main()

.github/ci-test-each-commit-exec.sh

@@ -1,6 +1,6 @@
 # Copyright (c) 2023-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+# file COPYING or https://opensource.org/license/mit.
 
 name: CI
 on:
@@ -81,7 +81,7 @@ jobs:
       - name: Compile and run tests
         run: |
           # Run tests on commits after the last merge commit and before the PR head commit
-          git rebase --exec "git merge --no-commit origin/${GITHUB_BASE_REF} && ./.github/ci-test-each-commit-exec.sh && git reset --hard" ${{ env.TEST_BASE }}
+          git rebase --exec "git merge --no-commit origin/${GITHUB_BASE_REF} && python3 ./.github/ci-test-each-commit-exec.py && git reset --hard" ${{ env.TEST_BASE }}
 
   macos-native-arm64:
     name: ${{ matrix.job-name }}

.github/workflows/ci.yml

@@ -115,6 +115,7 @@ fixes or code moves with actual code changes.
 
 Make sure each individual commit is hygienic: that it builds successfully on its
 own without warnings, errors, regressions, or test failures.
+This means tests must be updated in the same commit that changes the behavior.
 
 Commit messages should be verbose by default consisting of a short subject line
 (50 chars max), a blank line and detailed explanatory text as separate

CONTRIBUTING.md

@@ -56,8 +56,8 @@ in Python.
 These tests can be run (if the [test dependencies](/test) are installed) with: `build/test/functional/test_runner.py`
 (assuming `build` is your build directory).
 
-The CI (Continuous Integration) systems make sure that every pull request is built for Windows, Linux, and macOS,
-and that unit/sanity tests are run automatically.
+The CI (Continuous Integration) systems make sure that every pull request is tested on Windows, Linux, and macOS.
+The CI must pass on all commits before merge to avoid unrelated CI failures on new pull requests.
 
 ### Manual Quality Assurance (QA) Testing
 

README.md

@@ -38,7 +38,7 @@ python3 --version
 
 ${CI_RETRY_EXE} pip3 install \
   codespell==2.4.1 \
-  lief==0.13.2 \
+  lief==0.16.6 \
   mypy==1.4.1 \
   pyzmq==25.1.0 \
   ruff==0.5.5 \

ci/lint/04_install.sh

@@ -69,6 +69,24 @@ fi
 
 mkdir -p "$VERSION_BASE"
 
+################
+# SOURCE_DATE_EPOCH should not unintentionally be set
+################
+
+if [ -n "$SOURCE_DATE_EPOCH" ] && [ -z "$FORCE_SOURCE_DATE_EPOCH" ]; then
+cat << EOF
+ERR: Environment variable SOURCE_DATE_EPOCH is set which may break reproducibility.
+
+     Aborting...
+
+Hint: You may want to:
+      1. Unset this variable: \`unset SOURCE_DATE_EPOCH\` before rebuilding
+      2. Set the 'FORCE_SOURCE_DATE_EPOCH' environment variable if you insist on
+         using your own epoch
+EOF
+exit 1
+fi
+
 ################
 # Build directories should not exist
 ################

contrib/guix/guix-build

@@ -15,8 +15,10 @@
              (gnu packages ninja)
              (gnu packages pkg-config)
              ((gnu packages python) #:select (python-minimal))
-             ((gnu packages python-build) #:select (python-tomli python-poetry-core))
+             ((gnu packages python-build) #:select (python-poetry-core))
              ((gnu packages python-crypto) #:select (python-asn1crypto))
+             ((gnu packages python-science) #:select (python-scikit-build-core))
+             ((gnu packages python-xyz) #:select (python-pydantic-2 python-pydantic-core))
              ((gnu packages tls) #:select (openssl))
              ((gnu packages version-control) #:select (git-minimal))
              (guix build-system cmake)
@@ -158,37 +160,35 @@ chain for " target " development."))
 (define-public python-lief
   (package
     (name "python-lief")
-    (version "0.13.2")
+    (version "0.16.6")
     (source (origin
               (method git-fetch)
               (uri (git-reference
                     (url "https://github.com/lief-project/LIEF")
                     (commit version)))
               (file-name (git-file-name name version))
-              (modules '((guix build utils)))
-              (snippet
-               '(begin
-                  ;; Configure build for Python bindings.
-                  (substitute* "api/python/config-default.toml"
-                    (("(ninja         = )true" all m)
-                     (string-append m "false"))
-                    (("(parallel-jobs = )0" all m)
-                     (string-append m (number->string (parallel-job-count)))))))
               (sha256
                (base32
-                "0y48x358ppig5xp97ahcphfipx7cg9chldj2q5zrmn610fmi4zll"))))
-    (build-system python-build-system)
-    (native-inputs (list cmake-minimal python-tomli))
+                "1pq9nagrnkl1x943bqnpiyxmkd9vk99znfxiwqp6vf012b50bz2a"))
+              (patches (search-our-patches "lief-scikit-0-9.patch"))))
+    (build-system pyproject-build-system)
+    (native-inputs (list cmake-minimal
+                         ninja
+                         python-scikit-build-core
+                         python-pydantic-core
+                         python-pydantic-2))
     (arguments
      (list
       #:tests? #f                  ;needs network
       #:phases #~(modify-phases %standard-phases
-                   (add-before 'build 'change-directory
+                   (add-before 'build 'set-pythonpath
                      (lambda _
-                       (chdir "api/python")))
-                   (replace 'build
+                       (setenv "PYTHONPATH"
+                         (string-append (string-append (getcwd) "/api/python/backend")
+                                      ":" (or (getenv "PYTHONPATH") "")))))
+                  (add-after 'set-pythonpath 'change-directory
                      (lambda _
-                       (invoke "python" "setup.py" "build"))))))
+                       (chdir "api/python"))))))
     (home-page "https://github.com/lief-project/LIEF")
     (synopsis "Library to instrument executable formats")
     (description

contrib/guix/manifest.scm

@@ -0,0 +1,21 @@
+Partially revert f23ced2f4ffc170d0a6f40ff4a1bee575e3447cf
+
+Restore compat with python-scikit-build-core 0.9.x
+Can be dropped when using python-scikit-build-core >= 0.10.x
+
+--- a/api/python/backend/setup.py
++++ b/api/python/backend/setup.py
+@@ -101,12 +101,12 @@ def _get_hooked_config(is_editable: bool) -> Optional[dict[str, Union[str, List[
+     config_settings = {
+         "logging.level": "DEBUG",
+         "build-dir": config.build_dir,
+-        "build.targets": config.build.targets,
+         "install.strip": config.strip,
+         "backport.find-python": "0",
+         "wheel.py-api":  config.build.py_api,
+         "cmake.source-dir": SRC_DIR.as_posix(),
+         "cmake.build-type": config.build.build_type,
++        "cmake.targets": config.build.targets,
+         "cmake.args": [
+             *config.cmake_generator,
+             *config.get_cmake_args(is_editable),

contrib/guix/patches/lief-scikit-0-9.patch

@@ -30,13 +30,13 @@ def check_ELF_RELRO(binary) -> bool:
         # However, the dynamic linker need to write to this area so these are RW.
         # Glibc itself takes care of mprotecting this area R after relocations are finished.
         # See also https://marc.info/?l=binutils&m=1498883354122353
-        if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO:
+        if segment.type == lief.ELF.Segment.TYPE.GNU_RELRO:
             have_gnu_relro = True
 
     have_bindnow = False
     try:
-        flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS)
-        if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW:
+        flags = binary.get(lief.ELF.DynamicEntry.TAG.FLAGS)
+        if flags.has(lief.ELF.DynamicEntryFlags.FLAG.BIND_NOW):
             have_bindnow = True
     except Exception:
         have_bindnow = False
@@ -55,9 +55,9 @@ def check_ELF_SEPARATE_CODE(binary):
     based on their permissions. This checks for missing -Wl,-z,separate-code
     and potentially other problems.
     '''
-    R = lief.ELF.SEGMENT_FLAGS.R
-    W = lief.ELF.SEGMENT_FLAGS.W
-    E = lief.ELF.SEGMENT_FLAGS.X
+    R = lief.ELF.Segment.FLAGS.R
+    W = lief.ELF.Segment.FLAGS.W
+    E = lief.ELF.Segment.FLAGS.X
     EXPECTED_FLAGS = {
         # Read + execute
         '.init': R | E,
@@ -99,7 +99,7 @@ def check_ELF_SEPARATE_CODE(binary):
     # and for each section, remember the flags of the associated program header.
     flags_per_section = {}
     for segment in binary.segments:
-        if segment.type ==  lief.ELF.SEGMENT_TYPES.LOAD:
+        if segment.type == lief.ELF.Segment.TYPE.LOAD:
             for section in segment.sections:
                 flags_per_section[section.name] = segment.flags
     # Spot-check ELF LOAD program header flags per section
@@ -144,13 +144,13 @@ def check_ELF_FORTIFY(binary) -> bool:
 
 def check_PE_DYNAMIC_BASE(binary) -> bool:
     '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)'''
-    return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists
+    return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists
 
 # Must support high-entropy 64-bit address space layout randomization
 # in addition to DYNAMIC_BASE to have secure ASLR.
 def check_PE_HIGH_ENTROPY_VA(binary) -> bool:
     '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR'''
-    return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists
+    return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists
 
 def check_PE_RELOC_SECTION(binary) -> bool:
     '''Check for a reloc section. This is required for functional ASLR.'''
@@ -181,7 +181,7 @@ def check_MACHO_NOUNDEFS(binary) -> bool:
     '''
     Check for no undefined references.
     '''
-    return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS)
+    return binary.header.has(lief.MachO.Header.FLAGS.NOUNDEFS)
 
 def check_MACHO_FIXUP_CHAINS(binary) -> bool:
     '''
@@ -206,7 +206,13 @@ def check_NX(binary) -> bool:
     '''
     Check for no stack execution
     '''
-    return binary.has_nx
+
+    # binary.has_nx checks are only for the stack, but MachO binaries might
+    # have executable heaps.
+    if binary.format == lief.Binary.FORMATS.MACHO:
+        return binary.concrete.has_nx_stack and binary.concrete.has_nx_heap
+    else:
+        return binary.has_nx
 
 def check_MACHO_CONTROL_FLOW(binary) -> bool:
     '''
@@ -229,6 +235,7 @@ def check_MACHO_BRANCH_PROTECTION(binary) -> bool:
     return False
 
 BASE_ELF = [
+    ('FORTIFY', check_ELF_FORTIFY),
     ('PIE', check_PIE),
     ('NX', check_NX),
     ('RELRO', check_ELF_RELRO),
@@ -253,31 +260,31 @@ def check_MACHO_BRANCH_PROTECTION(binary) -> bool:
 ]
 
 CHECKS = {
-    lief.EXE_FORMATS.ELF: {
-        lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW), ('FORTIFY', check_ELF_FORTIFY)],
-        lief.ARCHITECTURES.ARM: BASE_ELF + [('FORTIFY', check_ELF_FORTIFY)],
-        lief.ARCHITECTURES.ARM64: BASE_ELF + [('FORTIFY', check_ELF_FORTIFY)],
-        lief.ARCHITECTURES.PPC: BASE_ELF + [('FORTIFY', check_ELF_FORTIFY)],
-        lief.ARCHITECTURES.RISCV: BASE_ELF, # Skip FORTIFY. See https://github.com/lief-project/LIEF/issues/1082.
+    lief.Binary.FORMATS.ELF: {
+        lief.Header.ARCHITECTURES.X86_64: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)],
+        lief.Header.ARCHITECTURES.ARM: BASE_ELF,
+        lief.Header.ARCHITECTURES.ARM64: BASE_ELF,
+        lief.Header.ARCHITECTURES.PPC64: BASE_ELF,
+        lief.Header.ARCHITECTURES.RISCV: BASE_ELF,
     },
-    lief.EXE_FORMATS.PE: {
-        lief.ARCHITECTURES.X86: BASE_PE,
+    lief.Binary.FORMATS.PE: {
+        lief.Header.ARCHITECTURES.X86_64: BASE_PE,
     },
-    lief.EXE_FORMATS.MACHO: {
-        lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE),
+    lief.Binary.FORMATS.MACHO: {
+        lief.Header.ARCHITECTURES.X86_64: BASE_MACHO + [('PIE', check_PIE),
                                               ('NX', check_NX),
                                               ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)],
-        lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)],
+        lief.Header.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)],
     }
 }
 
 if __name__ == '__main__':
     retval: int = 0
     for filename in sys.argv[1:]:
         binary = lief.parse(filename)
+
         etype = binary.format
         arch = binary.abstract.header.architecture
-        binary.concrete
 
         failed: list[str] = []
         for (name, func) in CHECKS[etype][arch]: