feat(cmds): Expand input of all functions to file paths, bytes, or file-like objects

Author: radimkarnis

docs/en/conf.py

@@ -23,6 +23,7 @@
 copyright = "2016 - {}, Espressif Systems (Shanghai) Co., Ltd".format(
     datetime.datetime.now().year
 )
+autodoc_typehints_format = "short"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/en/esptool/scripting.rst

@@ -95,6 +95,40 @@ The following example demonstrates running a series of flash memory operations i
 - This example doesn't use ``detect_chip()``, but instantiates a ``ESP32ROM`` class directly. This is useful when you know the target chip in advance. In this scenario ``esp.connect()`` is required to establish a connection with the device.
 - Multiple operations can be chained together in a single context manager block.
 
+------------
+
+The Public API implements a custom ``ImageSource`` input type, which expands to ``str | bytes | IO[bytes]`` - a path to the firmware image file, an opened file-like object, or the image data as bytes.
+
+As output, the API returns a ``bytes`` object representing the binary image or writes the image to a file if the ``output`` parameter is provided.
+
+The following example converts an ELF file to a flashable binary, prints the image information, and flashes the image. The example demonstrates three different ways to achieve the same result, showcasing the flexibility of the API:
+
+.. code-block:: python
+
+    ELF = "firmware.elf"
+
+    # var 1 - Loading ELF from a file, not writing binary to a file
+    bin_file = elf2image(ELF, "esp32c3")
+    image_info(bin_file)
+    with detect_chip(PORT) as esp:
+        attach_flash(esp)
+        write_flash(esp, [(0, bin_file)])
+
+    # var 2 - Loading ELF from an opened file object, not writing binary to a file
+    with open(ELF, "rb") as elf_file, detect_chip(PORT) as esp:
+        bin_file = elf2image(elf_file, "esp32c3")
+        image_info(bin_file)
+        attach_flash(esp)
+        write_flash(esp, [(0, bin_file)])
+
+    # var 3 - Loading ELF from a file, writing binary to a file
+    elf2image(ELF, "esp32c3", "image.bin")
+    image_info("image.bin")
+    with detect_chip(PORT) as esp:
+        attach_flash(esp)
+        write_flash(esp, [(0, "image.bin")])
+
+
 ------------
 
 **The following section provides a detailed reference for the public API functions.**

esptool/__init__.py

@@ -627,11 +627,11 @@ def run_cli(ctx):
 @click.pass_context
 def image_info_cli(ctx, filename):
     """Dump headers from a binary file (bootloader or application)"""
-    image_info(filename, ctx.obj["chip"])
+    image_info(filename, chip=None if ctx.obj["chip"] == "auto" else ctx.obj["chip"])
 
 
 @cli.command("elf2image")
-@click.argument("input", type=click.Path(exists=True))
+@click.argument("filename", type=click.Path(exists=True))
 @click.option(
     "--output",
     "-o",
@@ -721,14 +721,16 @@ def image_info_cli(ctx, filename):
 )
 @add_spi_flash_options(allow_keep=False, auto_detect=False)
 @click.pass_context
-def elf2image_cli(ctx, **kwargs):
+def elf2image_cli(ctx, filename, **kwargs):
     """Create an application image from ELF file"""
+    if ctx.obj["chip"] == "auto":
+        raise FatalError(
+            f"Specify the --chip argument (choose from {', '.join(CHIP_LIST)})"
+        )
     append_digest = not kwargs.pop("dont_append_digest", False)
-    # Default to ESP8266 for backwards compatibility
-    chip = "esp8266" if ctx.obj["chip"] == "auto" else ctx.obj["chip"]
     output = kwargs.pop("output", None)
     output = "auto" if output is None else output
-    elf2image(chip=chip, output=output, append_digest=append_digest, **kwargs)
+    elf2image(filename, ctx.obj["chip"], output, append_digest=append_digest, **kwargs)
 
 
 @cli.command("read_mac")
@@ -917,6 +919,10 @@ def read_flash_sfdp_cli(ctx, address, bytes, **kwargs):
 @click.pass_context
 def merge_bin_cli(ctx, addr_filename, **kwargs):
     """Merge multiple raw binary files into a single file for later flashing"""
+    if ctx.obj["chip"] == "auto":
+        raise FatalError(
+            f"Specify the --chip argument (choose from {', '.join(CHIP_LIST)})"
+        )
     merge_bin(addr_filename, chip=ctx.obj["chip"], **kwargs)
 
 

esptool/bin_image.py

@@ -32,7 +32,7 @@
     ESP32S3ROM,
     ESP8266ROM,
 )
-from .util import FatalError, byte, pad_to
+from .util import FatalError, byte, ImageSource, get_bytes, pad_to
 
 
 def align_file_position(f, size):
@@ -62,48 +62,43 @@ def intel_hex_to_bin(file: IO[bytes], start_addr: int | None = None) -> IO[bytes
         return file
 
 
-def LoadFirmwareImage(chip, image_file):
+def LoadFirmwareImage(chip: str, image_data: ImageSource):
     """
     Load a firmware image. Can be for any supported SoC.
 
     ESP8266 images will be examined to determine if they are original ROM firmware
     images (ESP8266ROMFirmwareImage) or "v2" OTA bootloader images.
 
-    Returns a BaseFirmwareImage subclass, either ESP8266ROMFirmwareImage (v1)
-    or ESP8266V2FirmwareImage (v2).
+    Returns a BaseFirmwareImage subclass.
     """
-
-    def select_image_class(f, chip):
-        chip = re.sub(r"[-()]", "", chip.lower())
-        if chip != "esp8266":
-            return {
-                "esp32": ESP32FirmwareImage,
-                "esp32s2": ESP32S2FirmwareImage,
-                "esp32s3": ESP32S3FirmwareImage,
-                "esp32c3": ESP32C3FirmwareImage,
-                "esp32c2": ESP32C2FirmwareImage,
-                "esp32c6": ESP32C6FirmwareImage,
-                "esp32c61": ESP32C61FirmwareImage,
-                "esp32c5": ESP32C5FirmwareImage,
-                "esp32h2": ESP32H2FirmwareImage,
-                "esp32h21": ESP32H21FirmwareImage,
-                "esp32p4": ESP32P4FirmwareImage,
-                "esp32h4": ESP32H4FirmwareImage,
-            }[chip](f)
-        else:  # Otherwise, ESP8266 so look at magic to determine the image type
-            magic = ord(f.read(1))
-            f.seek(0)
-            if magic == ESPLoader.ESP_IMAGE_MAGIC:
-                return ESP8266ROMFirmwareImage(f)
-            elif magic == ESP8266V2FirmwareImage.IMAGE_V2_MAGIC:
-                return ESP8266V2FirmwareImage(f)
-            else:
-                raise FatalError("Invalid image magic number: %d" % magic)
-
-    if isinstance(image_file, str):
-        with open(image_file, "rb") as f:
-            return select_image_class(f, chip)
-    return select_image_class(image_file, chip)
+    data, _ = get_bytes(image_data)
+    f = io.BytesIO(data)
+    chip = re.sub(r"[-()]", "", chip.lower())
+    if chip == "esp8266":
+        # Look at the magic number to determine the ESP8266 image type
+        magic = ord(f.read(1))
+        f.seek(0)
+        if magic == ESPLoader.ESP_IMAGE_MAGIC:
+            return ESP8266ROMFirmwareImage(f)
+        elif magic == ESP8266V2FirmwareImage.IMAGE_V2_MAGIC:
+            return ESP8266V2FirmwareImage(f)
+        else:
+            raise FatalError(f"Invalid image magic number: {magic}")
+    else:
+        return {
+            "esp32": ESP32FirmwareImage,
+            "esp32s2": ESP32S2FirmwareImage,
+            "esp32s3": ESP32S3FirmwareImage,
+            "esp32c3": ESP32C3FirmwareImage,
+            "esp32c2": ESP32C2FirmwareImage,
+            "esp32c6": ESP32C6FirmwareImage,
+            "esp32c61": ESP32C61FirmwareImage,
+            "esp32c5": ESP32C5FirmwareImage,
+            "esp32h2": ESP32H2FirmwareImage,
+            "esp32h21": ESP32H21FirmwareImage,
+            "esp32p4": ESP32P4FirmwareImage,
+            "esp32h4": ESP32H4FirmwareImage,
+        }[chip](f)
 
 
 class ImageSegment(object):
@@ -1225,11 +1220,10 @@ class ELFFile(object):
     SEG_TYPE_LOAD = 0x01
     LEN_SEG_HEADER = 0x20
 
-    def __init__(self, name):
-        # Load sections from the ELF file
-        self.name = name
-        with open(self.name, "rb") as f:
-            self._read_elf_file(f)
+    def __init__(self, data):
+        self.data, self.name = get_bytes(data)
+        f = io.BytesIO(self.data)
+        self._read_elf_file(f)
 
     def get_section(self, section_name):
         for s in self.sections:
@@ -1240,6 +1234,7 @@ def get_section(self, section_name):
     def _read_elf_file(self, f):
         # read the ELF file header
         LEN_FILE_HEADER = 0x34
+        source = "Image" if self.name is None else f"'{self.name}'"
         try:
             (
                 ident,
@@ -1257,25 +1252,23 @@ def _read_elf_file(self, f):
                 shnum,
                 shstrndx,
             ) = struct.unpack("<16sHHLLLLLHHHHHH", f.read(LEN_FILE_HEADER))
-        except struct.error as e:
-            raise FatalError(
-                "Failed to read a valid ELF header from %s: %s" % (self.name, e)
-            )
 
+        except struct.error as e:
+            raise FatalError(f"{source} does not have a valid ELF header: {e}")
         if byte(ident, 0) != 0x7F or ident[1:4] != b"ELF":
-            raise FatalError("%s has invalid ELF magic header" % self.name)
+            raise FatalError(f"{source} has invalid ELF magic header")
         if machine not in [0x5E, 0xF3]:
             raise FatalError(
-                "%s does not appear to be an Xtensa or an RISCV ELF file. "
-                "e_machine=%04x" % (self.name, machine)
+                f"{source} does not appear to be an Xtensa or an RISCV ELF image. "
+                f"(e_machine = {machine:#06x})"
             )
         if shentsize != self.LEN_SEC_HEADER:
             raise FatalError(
-                "%s has unexpected section header entry size 0x%x (not 0x%x)"
-                % (self.name, shentsize, self.LEN_SEC_HEADER)
+                f"{source} has unexpected section header entry size {shentsize:#x} "
+                f"(not {self.LEN_SEC_HEADER:#x})"
             )
         if shnum == 0:
-            raise FatalError("%s has 0 section headers" % (self.name))
+            raise FatalError(f"{source} has 0 section headers")
         self._read_sections(f, shoff, shnum, shstrndx)
         self._read_segments(f, _phoff, _phnum, shstrndx)
 
@@ -1285,13 +1278,13 @@ def _read_sections(self, f, section_header_offs, section_header_count, shstrndx)
         section_header = f.read(len_bytes)
         if len(section_header) == 0:
             raise FatalError(
-                "No section header found at offset %04x in ELF file."
-                % section_header_offs
+                f"No section header found at offset {section_header_offs:#06x} "
+                "in ELF image."
             )
         if len(section_header) != (len_bytes):
             raise FatalError(
-                "Only read 0x%x bytes from section header (expected 0x%x.) "
-                "Truncated ELF file?" % (len(section_header), len_bytes)
+                f"Only read {len(section_header):#x} bytes from section header "
+                f"(expected {len_bytes:#x}). Truncated ELF image?"
             )
 
         # walk through the section header and extract all sections
@@ -1347,13 +1340,13 @@ def _read_segments(self, f, segment_header_offs, segment_header_count, shstrndx)
         segment_header = f.read(len_bytes)
         if len(segment_header) == 0:
             raise FatalError(
-                "No segment header found at offset %04x in ELF file."
-                % segment_header_offs
+                f"No segment header found at offset {segment_header_offs:#06x} "
+                "in ELF image."
             )
         if len(segment_header) != (len_bytes):
             raise FatalError(
-                "Only read 0x%x bytes from segment header (expected 0x%x.) "
-                "Truncated ELF file?" % (len(segment_header), len_bytes)
+                f"Only read {len(segment_header):#x} bytes from segment header "
+                f"(expected {len_bytes:#x}). Truncated ELF image?"
             )
 
         # walk through the segment header and extract all segments
@@ -1389,6 +1382,6 @@ def read_data(offs, size):
     def sha256(self):
         # return SHA256 hash of the input ELF file
         sha256 = hashlib.sha256()
-        with open(self.name, "rb") as f:
-            sha256.update(f.read())
+        f = io.BytesIO(self.data)
+        sha256.update(f.read())
         return sha256.digest()

esptool/cli_util.py

@@ -170,7 +170,7 @@ def convert(
             if sector_start < end:
                 raise click.BadParameter(
                     f"Detected overlap at address: "
-                    f"0x{address:x} for file: {argfile.name}",
+                    f"{address:#x} for file: {argfile.name}",
                 )
             end = sector_end
         return pairs