Merge branch 'main' into pr_1452

Author: terriko

.github/actions/spelling/allow.txt

@@ -1,3 +1,11 @@
+refactoring
+peb
+conventionalcommits
+nisamson
+bhargavh
+jerinjtitus
+Molkree
+Romi
 abhaykatheria
 ableabhinav
 accountsservice
@@ -23,6 +31,7 @@ backported
 bcca
 bdbd
 bdist
+bestpractices
 bigbird
 binutils
 bksahu
@@ -52,6 +61,7 @@ codecov
 conda
 config
 copyleft
+coreinfrastructure
 cpio
 cpp
 CQA

.github/workflows/spelling.yml

@@ -8,3 +8,5 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: check-spelling/[email protected]
+      with:
+        post_comment: '0'

.pre-commit-config.yaml

@@ -1,16 +1,16 @@
 repos:
 -   repo: https://github.com/pycqa/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
     - id: isort
 
 -   repo: https://github.com/python/black
-    rev: 21.9b0
+    rev: 21.11b1
     hooks:
     - id: black
 
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.29.1
     hooks:
     - id: pyupgrade
       args: ["--py36-plus"]

README.md

@@ -7,22 +7,22 @@
 [![On PyPI](https://img.shields.io/pypi/v/cve-bin-tool)](https://pypi.org/project/cve-bin-tool/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black)
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5380/badge)](https://bestpractices.coreinfrastructure.org/projects/5380)
 
+The CVE Binary Tool is a free, open source tool to help you find known vulnerabilities in software, using data from the [National Vulnerability Database](https://nvd.nist.gov/) (NVD) list of [Common Vulnerabilities and Exposures](https://en.wikipedia.org/wiki/Common_Vulnerabilities_and_Exposures#:~:text=Common%20Vulnerabilities%20and%20Exposures%20(CVE)%20is%20a%20dictionary%20of%20common,publicly%20known%20information%20security%20vulnerabilities.) (CVEs).
 
-The CVE Binary Tool scans for a number of common, vulnerable open source
-components such as openssl, libpng, libxml2, and expat to let you know
-if a given directory or binary file includes common libraries with
-known vulnerabilities., known as CVEs ([Common Vulnerabilities and Exposures](https://en.wikipedia.org/wiki/Common_Vulnerabilities_and_Exposures#:~:text=Common%20Vulnerabilities%20and%20Exposures%20(CVE)%20is%20a%20dictionary%20of%20common,publicly%20known%20information%20security%20vulnerabilities.)).
+The tool has two main modes of operation:
+
+1. A binary scanner which helps you determine which packages may have been included as part of a piece of software.  There are around 100 checkers which focus on common, vulnerable open source components such as openssl, libpng, libxml2 and expat.
+2. Tools for scanning known component lists in various formats, including .csv, Python's requirements.txt, several linux distribution package lists, and several Software Bill of Materials (SBOM) formats.
+
+It is intended to be used as part of your continuous integration system to enable regular vulnerability scanning and give you early warning of known issues in your supply chain.
 
 See our [documentation](https://cve-bin-tool.readthedocs.io/en/latest/) and [quickstart guide](https://cve-bin-tool.readthedocs.io/en/latest/README.html)  
+
 Usage:
 `cve-bin-tool <directory/file to scan> `
 
-You can also do `python -m cve_bin_tool.cli`
-which is useful if you're trying the latest code from
-[the cve-bin-tool github](https://github.com/intel/cve-bin-tool).
-
-
     optional arguments:
       -h, --help            show this help message and exit
       -e, --exclude         exclude path while scanning
@@ -90,6 +90,10 @@ which is useful if you're trying the latest code from
        CVE Binary Tool autoextracts all compressed files by default now
 
 
+You can also do `python -m cve_bin_tool.cli`
+which is useful if you're trying the latest code from
+[the cve-bin-tool github](https://github.com/intel/cve-bin-tool).
+
 Note that if the CVSS and Severity flags are both specified, the CVSS flag takes precedence.
 
 `--input-file` extends the functionality of *csv2cve* for other formats like JSON.  It also allows cve-bin-tool to specify triage data so you can group issues which may have been mitigated (through patches, configuration, or other methods not detectable by our version scanning method) or mark false positives.  Triage data can be re-used and applied to multiple scans.  You can provide either CSV or JSON file as input_file with vendor, product and version fields. You can also add optional fields like remarks, comments, cve_number, severity.

cve_bin_tool/cli.py

@@ -13,6 +13,7 @@
 """
 
 import argparse
+import importlib.util
 import logging
 import os
 import platform
@@ -364,6 +365,15 @@ def main(argv=None):
         version_check = args["disable_version_check"]
         db_update = args["update"]
 
+    # Check for PDF support
+    output_format = args["format"]
+    if output_format == "pdf" and importlib.util.find_spec("reportlab") is None:
+        LOGGER.info("PDF output not available. Default to console.")
+        LOGGER.info(
+            "If you want to produce PDF output, please install reportlab using pip install reportlab"
+        )
+        output_format = "console"
+
     merged_reports = None
     if args["merge"]:
         LOGGER.info(
@@ -573,7 +583,7 @@ def main(argv=None):
             )
 
             if not args["quiet"]:
-                output.output_file(args["format"])
+                output.output_file(output_format)
                 if args["backport_fix"] or args["available_fix"]:
                     distro_info = args["backport_fix"] or args["available_fix"]
                     is_backport = True if args["backport_fix"] else False

cve_bin_tool/cve_scanner.py

@@ -75,19 +75,8 @@ def get_cves(self, product_info: ProductInfo, triage_data: TriageData):
         vendor = product_info.vendor.replace("*", "")
 
         # Need to manipulate version to ensure canonical form of version
-        if product_info.product == "openssl":
-            pv = re.search(r"\d[.\d]*[a-z]?", product_info.version)
-            parsed_version_between = parse_version(self.openssl_convert(pv.group(0)))
-        else:
-            # Ensure canonical form of version numbering
-            if ":" in product_info.version:
-                # Handle x:a.b<string> e.g. 2:7.4+23
-                components = product_info.version.split(":")
-                pv = re.search(r"\d[.\d]*", components[1])
-            else:
-                # Handle a.b.c<string> e.g. 1.20.9rel1
-                pv = re.search(r"\d[.\d]*", product_info.version)
-        parsed_version = parse_version(pv.group(0))
+
+        parsed_version, parsed_version_between = self.canonical_convert(product_info)
 
         self.cursor.execute(query, [vendor, product_info.product, str(parsed_version)])
 
@@ -262,6 +251,25 @@ def openssl_convert(self, version: str) -> str:
             version = f"{version[:-1]}.{self.ALPHA_TO_NUM[last_char]}"
         return version
 
+    def canonical_convert(self, product_info: ProductInfo) -> str:
+        version_between = ""
+        if product_info.version == "":
+            return product_info.version, version_between
+        if product_info.product == "openssl":
+            pv = re.search(r"\d[.\d]*[a-z]?", product_info.version)
+            version_between = parse_version(self.openssl_convert(pv.group(0)))
+        else:
+            # Ensure canonical form of version numbering
+            if ":" in product_info.version:
+                # Handle x:a.b<string> e.g. 2:7.4+23
+                components = product_info.version.split(":")
+                pv = re.search(r"\d[.\d]*", components[1])
+            else:
+                # Handle a.b.c<string> e.g. 1.20.9rel1
+                pv = re.search(r"\d[.\d]*", product_info.version)
+        parsed_version = parse_version(pv.group(0))
+        return parsed_version, version_between
+
     def affected(self):
         """Returns list of product name and version tuples identified from
         scan"""

cve_bin_tool/extractor.py

@@ -54,11 +54,16 @@ def __init__(self, logger=None, error_mode=ErrorMode.TruncTrace):
                 ".msi",
                 ".egg",
                 ".whl",
+                ".war",
+                ".ear",
             },
         }
 
     def can_extract(self, filename):
         """Check if the filename is something we know how to extract"""
+        # Do not try to extract symlinks
+        if os.path.islink(filename):
+            return False
         for extension in itertools.chain(*self.file_extractors.values()):
             if filename.endswith(extension):
                 return True
@@ -182,9 +187,9 @@ async def extract_file_zip(filename, extraction_path, process_can_fail=True):
         is_exe = filename.endswith(".exe")
         if await aio_inpath("unzip"):
             stdout, stderr, _ = await aio_run_command(
-                ["unzip", "-n", "-d", extraction_path, filename], process_can_fail
+                ["unzip", "-n", "-q", "-d", extraction_path, filename], process_can_fail
             )
-            if stderr or not stdout:
+            if stderr:
                 if is_exe:
                     return 0  # not all .exe files are zipfiles, no need for error
                 return 1

cve_bin_tool/output_engine/pdfbuilder.py

@@ -3,6 +3,7 @@
 
 from datetime import datetime
 
+from reportlab import rl_config
 from reportlab.graphics.shapes import Drawing, Rect, String
 from reportlab.lib import colors
 from reportlab.lib.styles import ParagraphStyle as PS
@@ -169,6 +170,9 @@ def __init__(self, includeTOC=True):
         self.table_data = []
         self.note_data = []
         self.table_validation = None
+        # Set default configuration parameters
+        rl_config.trustedHosts = ["localhost", "127.0.0.1"]
+        rl_config.trustedSchemes = ["http", "https"]
 
     def _spacer(self):
         self.contents.append(self.spacer)

cve_bin_tool/package_list_parser.py

@@ -21,7 +21,6 @@
 from cve_bin_tool.util import ProductInfo, Remarks
 
 ROOT_PATH = join(dirname(__file__), "..")
-PYPI_CSV = join(ROOT_PATH, "package_list_parser", "pypi_list.csv")
 
 DEB_DISTROS = ["debian", "pop", "ubuntu"]
 PACMAN_DISTROS = ["arch", "manjaro"]

cve_bin_tool/sbom_manager/__init__.py

@@ -1,7 +1,6 @@
 # Copyright (C) 2021 Anthony Harrison
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import sqlite3
 from collections import defaultdict
 from logging import Logger
 from typing import DefaultDict, Dict, List, Optional
@@ -83,21 +82,11 @@ def scan_file(self) -> Dict[ProductInfo, TriageData]:
         return self.sbom_data
 
     def get_vendor(self, product: str) -> Optional[str]:
-        self.cvedb.db_open()
-        if not self.cvedb.connection:
-            raise ConnectionError()
-        self.cursor = self.cvedb.connection.cursor()
-        get_vendor_request = "SELECT DISTINCT VENDOR FROM cve_range where PRODUCT=?"
-        self.cursor.execute(get_vendor_request, [product])
-        try:
-            # If multiple unique vendors then shouldn't proceed....
-            vendor = self.cursor.fetchone()[0]
-            # print(f"{product} is produced by {vendor}")
-        except (sqlite3.Error, TypeError) as e:
-            LOGGER.debug(e, exc_info=True)
-            vendor = None
-        self.cvedb.db_close()
-        return vendor
+        vendor_package_pair = self.cvedb.get_vendor_product_pairs(product)
+        if vendor_package_pair != []:
+            vendor = vendor_package_pair[0]["vendor"]
+            return vendor
+        return None
 
 
 if __name__ == "__main__":

cve_bin_tool/version.py

@@ -9,7 +9,7 @@
 
 from cve_bin_tool.log import LOGGER
 
-VERSION: str = "3.0.dev0"
+VERSION: str = "3.0"
 
 
 def check_latest_version():

dev-requirements.txt

@@ -1,4 +1,4 @@
-black==21.9b0
-isort==5.9.3
-pre-commit==2.15.0
+black==21.11b1
+isort==5.10.1
+pre-commit==2.16.0
 flake8==4.0.1

doc/RELEASE.md

@@ -1,8 +1,57 @@
 # CVE Binary Tool Release Notes
 
+## CVE Binary Tool 3.0
+
+The CVE Binary Tool 3.0 release includes improved tools for checking known lists of packages including Linux distributions, improved methods of communication with NVD to get vulnerability data, additional checkers, and significant refactoring to streamline the output.
+
+### New feature highlights:
+* **SBOM Scanning**: CVE Binary Tool can now take Software Bill of Materials (SBOM) files to help users improve their supply chain security data for all known dependencies. The initial feature can handle some versions of SPDX, CycloneDX and SWID formats. More information on SBOM scanning can be found here: https://github.com/intel/cve-bin-tool/blob/main/doc/how_to_guides/sbom.md
+* **Known vulnerability information**: Users scanning some linux distro packages can now get additional information about fixes available for those platforms.
+* **Vulnerability Data**: The default method for getting NVD vulnerability lists has been changed.  Previously we downloaded full yearly JSON files if anything in the year had changed, the new API allows us to get only the latest changes. Users may see a speedup during the update phase as a result.
+* **(Breaking change) Return codes:** The return codes used by CVE Binary Tool have changed.  
+   * A 0 will be returned if no CVEs are found, a 1 will be returned if any CVEs were found (no matter how many), and codes 2+ indicate operational errors.  A full list of error codes is available here: https://github.com/intel/cve-bin-tool/blob/main/cve_bin_tool/error_handler.py 
+   * Previously we returned the number of CVEs found, but this could exceed the expected range for return codes and cause unexpected behaviour.
+
+Thanks especially to our 2021 GSoC students, @BreadGenie, @imsahil007 and @peb-peb whose final GSoC contributions are part of this release.
+
+A full list of changes is available in GitHub. https://github.com/intel/cve-bin-tool/releases/tag/v3.0 
+
+Commit messages use the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) format.
+
+
+## CVE Binary Tool 2.2.1
+
+Release date: 04 Aug 2021
+
+The 2.2.1 release relaxes the behaviour when file extraction fails, which was causing problems for some users scanning files with .exe and .apk file extensions using the previous release. In 2.2 all extraction fails caused the tool to halt and throw an exception, in 2.2.1 the tool will log a warning and continue.
+
+## CVE Binary Tool 2.2 
+
+Release date: 08 Jul 2021
+
+The 2.2 release contains a number of bugfixes and improvements thanks to the many students who contributed as part of our Google Summer of Code selection process.  Congratulations to @BreadGenie, @imsahil007 and @peb-peb who will be continuing to work with us for the next few months!  
+
+New feature highlights:
+- CVE Binary Tool can now be used to get lists of vulnerabilities affecting a python requirements.txt file, as well as lists of packages installed on .deb or .rpm based systems (Thanks to @BreadGenie)
+- Scan reports can now be merged (Thanks to @imsahil007)
+- Reports can now be generated in PDF format (Thanks to @anthonyharrison)
+- A new helper script is available to help new contributors find appropriate patterns for new checkers (Thanks to @peb-peb)
+- Reports can now be generated even if no CVEs are found (Thanks to @BreadGenie) 
+- We've added rate limiting for our NVD requests (Thanks to @nisamson, @param211,  @bhargavh)
+
+There are also a number of new checkers and bug fixes.
+
+Thanks also to @jerinjtitus, @Molkree, @alt-glitch, @CabTheProgrammer, @Romi-776, @chaitanyamogal, @Rahul2044, @utkarsh147-del , @SinghHrmn, @SaurabhK122, @pdxjohnny and @terriko for their contributions to this release.
+
+## CVE Binary Tool 2.1.post1
+
+Release date: 27 Apr 2021
+
+Rate limiting temporary fix in response to NVD API update
+
 ## CVE Binary Tool 2.1
 
-Release Date: 07 Dec 2020
+Release date: 07 Dec 2020
 
 This release fixes an issue with jinja2 autoescape breaking the HTML reports
 and includes some updates to tests.

test/test_cvescanner.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2021 Anthony Harrison
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import pytest
+
+from cve_bin_tool.cve_scanner import CVEScanner
+from cve_bin_tool.util import ProductInfo
+
+
+class TestCVEScanner:
+    @pytest.mark.parametrize(
+        "version, expected_result",
+        (
+            ("1.1.0f", "1.1.0.5"),
+            ("1.1.0", "1.1.0"),
+        ),
+    )
+    def test_openssl_convert(self, version: str, expected_result: str):
+        scanner = CVEScanner()
+        assert scanner.openssl_convert(version) == expected_result
+
+    @pytest.mark.parametrize(
+        "product, expected_result, between_result",
+        (
+            (ProductInfo(vendor="", product="glibc", version="2.11.1"), "2.11.1", ""),
+            (
+                ProductInfo(vendor="", product="glibc", version="2.11.1_pre1"),
+                "2.11.1",
+                "",
+            ),
+            (
+                ProductInfo(vendor="", product="openssl", version="1.1.0h"),
+                "1.1.0h",
+                "1.1.0.7",
+            ),
+            (
+                ProductInfo(vendor="", product="openssl", version="1.1.0h_kali2"),
+                "1.1.0h",
+                "1.1.0.7",
+            ),
+            (ProductInfo(vendor="", product="openssl", version=""), "", ""),
+            (ProductInfo(vendor="", product="php", version="2:7.4"), "7.4", ""),
+            (ProductInfo(vendor="", product="php", version="2:7.4_deb0"), "7.4", ""),
+        ),
+    )
+    def test_canonical_convert(
+        self, product: ProductInfo, expected_result: str, between_result: str
+    ):
+        scanner = CVEScanner()
+        res1, res2 = scanner.canonical_convert(product)
+        assert str(res1) == expected_result
+        assert str(res2) == between_result