check_autopkg_recipes.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""This hook checks AutoPkg recipes to ensure they meet various
requirements."""

import argparse
import os
import sys
from contextlib import contextmanager
from distutils.version import LooseVersion

from pre_commit_hooks.util import (
    load_autopkg_recipe,
    validate_pkginfo_key_types,
    validate_required_keys,
    validate_restart_action_key,
)


# Import AutoPkg libraries, but ignore any warnings generated by the import.
@contextmanager
def suppress_stdout():
    with open(os.devnull, "w") as devnull:
        old_stdout = sys.stdout
        sys.stdout = devnull
        try:
            yield
        finally:
            sys.stdout = old_stdout


sys.path.append("/Library/AutoPkg")
try:
    with suppress_stdout():
        from autopkglib import get_processor, processor_names

    HAS_AUTOPKGLIB = True
except ImportError:
    # Silently skip checks that require autopkglib.
    HAS_AUTOPKGLIB = False


def build_argument_parser():
    """Build and return the argument parser."""

    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        "--override-prefix",
        nargs="+",
        default=["local."],
        help='Expected prefix(es) for recipe override identifiers (defaults to ["local."]). '
        "Case sensitive. Multiple acceptable identifier prefixes can be provided.",
    )
    parser.add_argument(
        "--recipe-prefix",
        nargs="+",
        default=["com.github."],
        help='Expected prefix(es) for recipe identifiers (defaults to ["com.github."]). '
        "Case sensitive. Multiple acceptable identifier prefixes can be provided.",
    )
    parser.add_argument(
        "--ignore-min-vers-before",
        default="1.0",
        help="Ignore MinimumVersion/processor mismatches below this version of AutoPkg "
        '(defaults to "1.0").\nSet to 0.1.0 to warn about all '
        "MinimumVersion/processor mismatches.\nDefaults to 0.1.0 if --strict is used.",
    )
    parser.add_argument(
        "--strict",
        action="store_true",
        default=False,
        help="Apply strictest set of rules when evaluating AutoPkg recipes, including "
        "adherence to recipe type conventions, flagging all MinimumVersion/processor "
        "mismatches, and forbidding <!-- --> comments. Very opinionated.",
    )
    parser.add_argument("filenames", nargs="*", help="Filenames to check.")
    return parser


def validate_recipe_prefix(recipe, filename, prefix):
    """Verify that the recipe identifier starts with the expected prefix."""

    passed = True
    if not any([recipe["Identifier"].startswith(x) for x in prefix]):
        print(
            "{}: identifier does not start with {}".format(
                filename,
                'one of: "%s"' % '", "'.join(prefix) if len(prefix) > 1 else prefix[0],
            )
        )
        passed = False

    return passed


def validate_comments(filename, strict):
    """Warn about comments in <!-- --> format that would break when running
    plutil -convert xml1."""

    passed = True
    with open(filename, "r") as openfile:
        recipe_text = openfile.read()
        if "<!--" in recipe_text and "-->" in recipe_text:
            if strict:
                print(
                    "{}: Convert from <!-- --> style comments "
                    "to a Comment key.".format(filename)
                )
                passed = False
            else:
                print(
                    "{}: WARNING: Recommend converting from <!-- --> style comments "
                    "to a Comment key.".format(filename)
                )

    return passed


def validate_processor_keys(process, filename):
    """Ensure all items in Process array have a "Processor" specified."""

    passed = True
    missing_processor_keys = [x for x in process if "Processor" not in x]
    if missing_processor_keys:
        for missing_proc in missing_processor_keys:
            print(
                '{}: Item in processor array is missing "Processor" key:\n{}'.format(
                    filename, missing_proc
                )
            )
        passed = False

    return passed


def validate_endofcheckphase(process, filename):
    """Ensure EndOfCheckPhase comes after a downloader."""

    passed = True
    downloader_idx = next(
        (
            idx
            for (idx, x) in enumerate(process)
            if x.get("Processor") in ("URLDownloader", "CURLDownloader")
        ),
        None,
    )
    if downloader_idx is None:
        return passed
    endofcheck_idx = next(
        (
            idx
            for (idx, x) in enumerate(process)
            if x.get("Processor") == "EndOfCheckPhase"
        ),
        None,
    )
    if endofcheck_idx is None:
        print(
            "{}: Contains a download processor, but no EndOfCheckPhase "
            "processor.".format(filename)
        )
        passed = False
    elif endofcheck_idx < downloader_idx:
        print(
            "{}: EndOfCheckPhase typically goes after a download processor, "
            "not before.".format(filename)
        )
        passed = False

    return passed


def validate_minimumversion(process, min_vers, ignore_min_vers_before, filename):
    """Ensure MinimumVersion is set appropriately for the processors used."""

    # Warn if using a MinimumVersion greater than or equal to 2
    # warn_on_vers = "2"
    # suggest_vers = "1.4.1"
    # if LooseVersion(min_vers) >= LooseVersion(warn_on_vers):
    #     print(
    #         "{}: WARNING: Choosing MinimumVersion {} limits the potential "
    #         "audience for your AutoPkg recipe. Consider using MinimumVersion "
    #         "{} if your processors support it.".format(filename, min_vers, suggest_vers)
    #     )

    # Processors for which a minimum version of AutoPkg is required.
    # Note: Because LooseVersion considers version 1.0 to be "less than" 1.0.0,
    # specifying more trailing zeros than needed in the dict below may result
    # in false positive errors for users of the check-autopkg-recipes hook.
    proc_min_versions = {
        "AppPkgCreator": "1.0",
        "BrewCaskInfoProvider": "0.2.5",
        "CodeSignatureVerifier": "0.3.1",
        "CURLDownloader": "0.5.1",
        "CURLTextSearcher": "0.5.1",
        "DeprecationWarning": "1.1",
        "EndOfCheckPhase": "0.1.0",
        "FileFinder": "0.2.3",
        "FileMover": "0.2.9",
        "FlatPkgPacker": "0.2.4",
        "FlatPkgUnpacker": "0.1.0",
        "GitHubReleasesInfoProvider": "0.5.0",
        "Installer": "0.4.0",
        "InstallFromDMG": "0.4.0",
        "MunkiCatalogBuilder": "0.1.0",
        "MunkiImporter": "0.1.0",
        "MunkiInstallsItemsCreator": "0.1.0",
        "MunkiPkginfoMerger": "0.1.0",
        "MunkiSetDefaultCatalog": "0.4.2",
        "PackageRequired": "0.5.1",
        "PathDeleter": "0.1.0",
        "PkgCopier": "0.1.0",
        "PkgExtractor": "0.1.0",
        "PkgPayloadUnpacker": "0.1.0",
        "PlistEditor": "0.1.0",
        "PlistReader": "0.2.5",
        "SparkleUpdateInfoProvider": "0.1.0",
        "StopProcessingIf": "0.1.0",
        "Symlinker": "0.1.0",
        "Unarchiver": "0.1.0",
        "URLTextSearcher": "0.2.9",
        "Versioner": "0.1.0",
    }

    passed = True
    for proc in [
        x
        for x in proc_min_versions
        if LooseVersion(proc_min_versions[x]) >= LooseVersion(ignore_min_vers_before)
    ]:
        if proc in [x.get("Processor") for x in process]:
            if LooseVersion(min_vers) < LooseVersion(proc_min_versions[proc]):
                print(
                    "{}: {} processor requires minimum AutoPkg "
                    "version {}".format(filename, proc, proc_min_versions[proc])
                )
                passed = False

    return passed


def validate_no_deprecated_procs(process, filename):
    """Warn if any deprecated processors are used."""

    # Processors that have been deprecated.
    deprecated_procs = ("CURLDownloader", "BrewCaskInfoProvider")

    passed = True
    for proc in process:
        if proc.get("Processor") in deprecated_procs:
            print(
                "{}: WARNING: Deprecated processor {} "
                "is used.".format(filename, proc.get("Processor"))
            )

    return passed


def validate_no_superclass_procs(process, filename):
    """Warn if any superclass processors (which are used by other processors
    rather than called in recipes) are used."""

    # Processors that are superclasses and shouldn't be referenced directly.
    superclass_procs = ("URLGetter",)

    passed = True
    for proc in process:
        if proc.get("Processor") in superclass_procs:
            print(
                "{}: WARNING: The processor {} is intended to be used "
                "by other processors, not used directly in recipes.".format(
                    filename, proc.get("Processor")
                )
            )

    return passed


def validate_jamf_processor_order(process, filename):
    """Warn if JamfUploader processors are not in their conventional order.
    https://youtu.be/srz4U9RHliQ?list=PLlxHm_Px-Ie1EIRlDHG2lW5H7c2UYvops&t=1010
    """

    # Recommended order of Jamf processors
    rec_order = (
        "com.github.grahampugh.jamf-upload.processors/JamfCategoryUploader",
        "com.github.grahampugh.jamf-upload.processors/JamfExtensionAttributeUploader",
        "com.github.grahampugh.jamf-upload.processors/JamfPackageUploader",
        "com.github.grahampugh.jamf-upload.processors/JamfScriptUploader",
        "com.github.grahampugh.jamf-upload.processors/JamfComputerGroupUploader",
        # TODO: The three below may depend on computer groups, but there's no
        # easy way to ignore relative order if multiple are used. Focusing on
        # JamfPolicyUploader only for now.
        "com.github.grahampugh.jamf-upload.processors/JamfPolicyUploader",
        # "com.github.grahampugh.jamf-upload.processors/JamfComputerProfileUploader",
        # "com.github.grahampugh.jamf-upload.processors/JamfSoftwareRestrictionUploader",
    )

    passed = True
    # All JamfUploader processors in recipe, ignoring duplicates, preserving order.
    actual_order = list(
        dict.fromkeys(
            [x.get("Processor") for x in process if x.get("Processor") in rec_order]
        )
    )
    desired_order = [x for x in rec_order if x in actual_order]
    if desired_order != actual_order:
        print(
            "{}: WARNING: JamfUploader processors are not in "
            "the recommended order: {}.".format(
                filename,
                ", ".join([x.split("/")[-1] for x in desired_order]),
            )
        )

    return passed


# def validate_unused_input_vars(recipe, recipe_text, filename):
#     """Warn if any input variables are not referenced in the recipe."""

#     # List of variables that are commonly allowed to be unreferenced (lowercase).
#     ignored_vars = (
#         "name",
#         "pkginfo",
#     )

#     passed = True
#     for input_var, _ in recipe.get("Input", {}).items():
#         if input_var.lower() in ignored_vars:
#             continue
#         subst = "%" + input_var + "%"
#         if subst not in recipe_text:
#             print(
#                 "{}: WARNING: Input variable {} not referenced in recipe.".format(
#                     filename, input_var
#                 )
#             )

#     return passed


def validate_no_var_in_app_path(process, filename):
    """Ensure %NAME% is not used in app paths that should be hard coded."""

    # Processors for which %NAME%.app should not be present in the arguments.
    no_name_var_in_proc_args = (
        "CodeSignatureVerifier",
        "Versioner",
        "PkgPayloadUnpacker",
        "FlatPkgUnpacker",
        "FileFinder",
        "Copier",
        "AppDmgVersioner",
        "InstallFromDMG",
    )

    passed = True
    for proc in process:
        if proc.get("Processor") in no_name_var_in_proc_args and "Arguments" in proc:
            for _, argvalue in proc["Arguments"].items():
                if isinstance(argvalue, str) and "%NAME%.app" in argvalue:
                    print(
                        "{}: Use actual app name instead of %NAME%.app in {} "
                        "processor argument.".format(filename, proc.get("Processor"))
                    )
                    passed = False

    return passed


def validate_proc_type_conventions(process, filename):
    """Ensure that processors used align with recipe type conventions."""

    # For each processor type, this is the list of processors that
    # we only expect to see in that type.
    proc_type_conventions = {
        "download": [
            "SparkleUpdateInfoProvider",
            "GitHubReleasesInfoProvider",
            "URLDownloader",
            "CURLDownloader",
            "EndOfCheckPhase",
        ],
        "munki": [
            "MunkiInstallsItemsCreator",
            "MunkiPkginfoMerger",
            "MunkiCatalogBuilder",
            "MunkiSetDefaultCatalog",
            "MunkiImporter",
        ],
        "pkg": ["AppPkgCreator", "PkgCreator"],
        "install": ["InstallFromDMG", "Installer"],
        # https://github.com/jssimporter/JSSImporter
        "jss": ["JSSImporter"],
        # https://github.com/grahampugh/jamf-upload
        "jamf": [
            "com.github.grahampugh.jamf-upload.processors/JamfCategoryUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfComputerGroupUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfComputerProfileUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfExtensionAttributeUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfPackageUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfPolicyDeleter",
            "com.github.grahampugh.jamf-upload.processors/JamfPolicyUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfScriptUploader",
            "com.github.grahampugh.jamf-upload.processors/JamfSoftwareRestrictionUploader",
        ],
        # https://github.com/autopkg/filewave
        "filewave": ["FileWaveImporter"],
    }

    passed = True
    processors = [x.get("Processor") for x in process]
    for recipe_type in proc_type_conventions:
        type_hint = ".{}.".format(recipe_type)
        if type_hint not in filename:
            for processor in processors:
                if processor in proc_type_conventions[recipe_type]:
                    print(
                        "{}: Processor {} is not conventional for this "
                        "recipe type.".format(filename, processor)
                    )
                    passed = False

    return passed


def validate_required_proc_for_types(process, filename):
    """Ensure that certain recipe types always have specific processors."""

    # For each recipe type, this is the list of processors that
    # MUST exist in that type. Uses "OR" logic, not "AND."
    required_proc_for_type = {
        # Skipping EndOfCheckPhase because validate_endofcheckphase()
        # already tests this.
        # "download": ["EndOfCheckPhase"],
        "munki": ["MunkiImporter"],
        "pkg": ["AppPkgCreator", "PkgCreator", "PkgCopier"],
        "install": ["InstallFromDMG", "Installer"],
        # https://github.com/jssimporter/JSSImporter
        "jss": ["JSSImporter"],
        # https://github.com/autopkg/filewave
        "filewave": ["com.github.autopkg.filewave.FWTool/FileWaveImporter"],
        # https://derflounder.wordpress.com/2021/07/30/signing-autopkg-built-packages-using-a-sign-recipe/
        "sign": ["com.github.rtrouton.SharedProcessors/PkgSigner"],
        "verify": [
            "com.github.autopkg.gerardkok-recipes.SharedProcessors/GPGSignatureVerifier"
        ],
    }

    passed = True
    processors = [x.get("Processor") for x in process]
    for recipe_type in required_proc_for_type:
        req_procs = required_proc_for_type[recipe_type]
        type_hint = ".{}.".format(recipe_type)
        if type_hint in filename:
            if recipe_type == "pkg" and processors == []:
                # OK for pkg recipes to have an empty process list, as long as
                # their parent is a download recipe that produces a pkg.
                # TODO: Validate parent is a download recipe.
                break
            if not any([x in processors for x in req_procs]):
                if len(req_procs) == 1:
                    print(
                        "{}: Recipe type {} should contain processor "
                        "{}.".format(filename, recipe_type, req_procs[0])
                    )
                else:
                    print(
                        "{}: Recipe type {} should contain one of these "
                        "processors: {}.".format(filename, recipe_type, req_procs)
                    )
                passed = False

    return passed


def validate_proc_args(process, filename):
    """Warn if invalid processor arguments are used."""

    passed = True

    # List of argument names (lowercase) that will not be flagged as invalid.
    ignored_args = ("note", "notes", "comment", "comments")

    # Create dictionary of AutoPkg core processors and their inputs.
    core_procs = {}
    for proc in processor_names():
        if hasattr(get_processor(proc), "input_variables"):
            core_procs[proc] = get_processor(proc).input_variables
        else:
            core_procs[proc] = {}

    for proc in process:
        if proc["Processor"] not in core_procs:
            # Skip input variable validation for non-core processors.
            continue
        for arg in proc.get("Arguments", {}):
            if arg.lower() in ignored_args:
                # Skip args in ignored list above.
                continue

            if not core_procs[proc["Processor"]]:
                print(
                    "{}: Unknown argument {} for processor {}, "
                    "which does not accept any arguments.".format(
                        filename,
                        arg,
                        proc["Processor"],
                    )
                )
                passed = False
            elif arg not in core_procs[proc["Processor"]]:
                print(
                    "{}: Unknown argument {} for processor {}. "
                    "Allowed arguments are: {}".format(
                        filename,
                        arg,
                        proc["Processor"],
                        ", ".join(core_procs[proc["Processor"]]),
                    )
                )
                passed = False

    return passed


def main(argv=None):
    """Main process."""

    # Parse command line arguments.
    argparser = build_argument_parser()
    args = argparser.parse_args(argv)
    if args.strict:
        args.ignore_min_vers_before = "0.1.0"

    # Track identifiers we've seen.
    seen_identifiers = []

    retval = 0
    for filename in args.filenames:

        recipe = load_autopkg_recipe(filename)
        if not recipe:
            retval = 1
            break  # No need to continue checking this file

        # For future implementation of validate_unused_input_vars()
        # with open(filename, "r") as openfile:
        #     recipe_text = openfile.read()

        # Top level keys that all AutoPkg recipes should contain.
        required_keys = ["Identifier"]
        if not validate_required_keys(recipe, filename, required_keys):
            retval = 1
            break  # No need to continue checking this file

        # Ensure the recipe identifier isn't duplicated.
        if recipe["Identifier"] in seen_identifiers:
            print(
                '{}: Identifier "{}" is shared by another recipe in this repo.'.format(
                    filename, recipe["Identifier"]
                )
            )
            retval = 1
        else:
            seen_identifiers.append(recipe["Identifier"])

        # Validate identifiers.
        if args.override_prefix and "Process" not in recipe:
            if not validate_recipe_prefix(recipe, filename, args.override_prefix):
                retval = 1
        if args.recipe_prefix and "Process" in recipe:
            if not validate_recipe_prefix(recipe, filename, args.recipe_prefix):
                retval = 1
        if recipe["Identifier"] == recipe.get("ParentRecipe"):
            print(
                "{}: Identifier and ParentRecipe should not "
                "be the same.".format(filename)
            )
            retval = 1

        # Validate that all input variables are used.
        # (Disabled for now because it's a little too opinionated, and doesn't take into account
        # whether environmental variables are used in custom processors.)
        # if args.strict:
        #     if not validate_unused_input_vars(recipe, recipe_text, filename):
        #         retval = 1

        # If the Input key contains a pkginfo dict, make a best effort to validate its contents.
        input_key = recipe.get("Input", recipe.get("input", recipe.get("INPUT")))
        if input_key and "pkginfo" in input_key:
            if not validate_pkginfo_key_types(input_key["pkginfo"], filename):
                retval = 1
            if not validate_restart_action_key(input_key["pkginfo"], filename):
                retval = 1

            # Check for common mistakes in min/max OS version keys
            os_vers_corrections = {
                "min_os": "minimum_os_version",
                "max_os": "maximum_os_version",
                "min_os_vers": "minimum_os_version",
                "max_os_vers": "maximum_os_version",
                "minimum_os": "minimum_os_version",
                "maximum_os": "maximum_os_version",
                "minimum_os_vers": "minimum_os_version",
                "maximum_os_vers": "maximum_os_version",
            }
            for os_vers_key in os_vers_corrections:
                if os_vers_key in input_key["pkginfo"]:
                    print(
                        "{}: You used {} when you probably meant {}.".format(
                            filename, os_vers_key, os_vers_corrections[os_vers_key]
                        )
                    )
                    retval = 1

            # TODO: Additional pkginfo checks here.

        # Warn about comments that would be lost during `plutil -convert xml1`
        if not validate_comments(filename, args.strict):
            retval = 1

        # Processor checks.
        if "Process" in recipe:
            process = recipe["Process"]

            if not validate_processor_keys(process, filename):
                retval = 1

            if not validate_endofcheckphase(process, filename):
                retval = 1

            if not validate_no_var_in_app_path(process, filename):
                retval = 1

            min_vers = recipe.get("MinimumVersion")
            if min_vers and not validate_minimumversion(
                process, min_vers, args.ignore_min_vers_before, filename
            ):
                retval = 1

            if not validate_no_deprecated_procs(process, filename):
                retval = 1

            if not validate_no_superclass_procs(process, filename):
                retval = 1

            if not validate_jamf_processor_order(process, filename):
                retval = 1

            if HAS_AUTOPKGLIB:
                if not validate_proc_args(process, filename):
                    retval = 1

            if args.strict:
                if not validate_proc_type_conventions(process, filename):
                    retval = 1

                if not validate_required_proc_for_types(process, filename):
                    retval = 1

    return retval


if __name__ == "__main__":
    exit(main())