Merge branch 'develop' into improve-pypi-package-detection

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -34,6 +34,14 @@ v33.0.0 (next next, roadmap)
   of these in other summary plugins.
   See https://github.com/nexB/scancode-toolkit/issues/1745
 
+- Add support for parsing resolved packages and dependency relationships
+  from nuget lockfile `packages.lock.json`.
+  See https://github.com/nexB/scancode-toolkit/pull/3825
+
+- Add support for parsing resolved packages and dependency relationships
+  from cocoapods lockfile `Podfile.lock`.
+  See https://github.com/nexB/scancode-toolkit/pull/3827
+
 v32.2.0 - 2024-06-19
 ----------------------
 

README.rst

@@ -161,6 +161,8 @@ There are a few common ways to `install ScanCode <https://scancode-toolkit.readt
 - `Run in a Docker container with a git clone and "docker run"
   <https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#installation-via-docker>`_
 
+- In Fedora 40+ you can `dnf install scancode-toolkit`
+
 
 Quick Start
 ===========
@@ -258,4 +260,4 @@ the third-party code used in ScanCode for more details.
 
 .. |release-github-actions| image:: https://github.com/nexB/scancode-toolkit/actions/workflows/scancode-release.yml/badge.svg?event=push
     :target: https://github.com/nexB/scancode-toolkit/actions/workflows/scancode-release.yml
-    :alt: Release tests
+    :alt: Release tests

docs/source/getting-started/install.rst

@@ -34,6 +34,11 @@ For advanced usage and experienced users, you can also use any of these mode:
     ``pip``. This is recommended for developers or users familiar with Python
     that want to embed ScanCode as a library.
 
+- :ref:`fedora_install`
+
+    ScanCode is part of main Fedora Linux repository. It will automatically install
+    all dependencies. This is recommended for production deployments.
+
 ----
 
 Before Installing
@@ -412,6 +417,22 @@ To uninstall, run::
     pip uninstall scancode-toolkit
 
 
+----
+
+.. _fedora_install:
+
+Install from Fedora's repository
+--------------------------------
+
+The package is available in Fedora 40 and newer. Run::
+
+    dnf install scancode-toolkit
+
+To uninstall, run::
+
+    dnf remove scancode-toolkit
+
+
 ----
 
 .. _commands_variation:

etc/scripts/gen_copyright_tests.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/skeleton for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import time
+
+from datetime import datetime
+
+import click
+import requests
+
+
+def timestamp():
+    return datetime.utcnow().isoformat().split("T")[0]
+
+
+EMPTY_COPY_TEST = """what:
+  - copyrights
+  - holders
+copyrights:
+holders:
+"""
+
+
+@click.command()
+@click.option(
+    "-u",
+    "--urls",
+    "urls_file",
+    type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
+    metavar="URLS-FILE",
+    multiple=False,
+    required=True,
+    help="Path to URLs file, one per line.",
+)
+@click.help_option("-h", "--help")
+def create_copyright_tests(
+    urls_file,
+):
+    """
+    Download the URLs listed in the URLS-FILE and create a copyight test for each in the current
+    directory.
+
+    If a line number is provided as a URL fragment #L2, uses only 5 lines before and after this
+    line.
+    
+    If the URL is a plain GitHub URL, convert the URL to a raw URL.
+    If the URL does not start with http it is treated as a plain copyright text to test
+    """
+    
+    with open(urls_file) as urls:
+        for i, url in enumerate(urls):
+            url = url.strip()
+            if not url:
+                continue
+
+            name = ""
+            if url.startswith("http"):
+                print(f"Fetching URL: {url}")
+                if url.startswith("https://github.com"):
+                    url = url.replace("https://github.com", "https://raw.githubusercontent.com")
+                    url = url.replace("/blob/", "/")
+    
+                if "github" in url:
+                    segs = url.split("/")
+                    org = segs[3]
+                    repo = segs[4]
+                    name = f"copyright-test-{timestamp()}-{i}-{org}-{repo}.copyright"
+            else:
+                print(f"Processing test: {url}")
+                name = f"copyright-test-{timestamp()}-{i}.copyright"
+
+
+            start_line = 0
+            end_line = 0
+            if "#L" in url:
+                _, _, line = url.rpartition("#L")
+                line = int(line)
+                if line > 5:
+                    start_line = line - 5
+                end_line = line + 5
+
+            if url.startswith("http"):
+                _header, content = get_remote_file_content(url, as_text=True)
+            else:
+                content = url
+
+            if end_line != 0:
+                content = "".join(content.strip().splitlines()[start_line:end_line])
+
+            with open(name, "w") as out:
+                out.write(content)
+
+            yml = EMPTY_COPY_TEST
+            if url.startswith("http"):
+                yml = f"{yml}\nnotes: from {url}\n"
+
+            with open(f"{name}.yml", "w") as out:
+                out.write(yml)
+
+            if url.startswith("http"):
+                time.sleep(1)
+
+
+class RemoteNotFetchedException(Exception):
+    pass
+
+
+def get_remote_file_content(
+    url,
+    as_text=True,
+    headers_only=False,
+    headers=None,
+    _delay=0,
+):
+    """
+    Fetch and return a tuple of (headers, content) at `url`. Return content as a
+    text string if `as_text` is True. Otherwise return the content as bytes.
+
+    If `header_only` is True, return only (headers, None). Headers is a mapping
+    of HTTP headers.
+    Retries multiple times to fetch if there is a HTTP 429 throttling response
+    and this with an increasing delay.
+    """
+    time.sleep(_delay)
+    headers = headers or {}
+    # using a GET with stream=True ensure we get the the final header from
+    # several redirects and that we can ignore content there. A HEAD request may
+    # not get us this last header
+    print(f"    DOWNLOADING: {url}")
+    with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response:
+        status = response.status_code
+        if status != requests.codes.ok:  # NOQA
+            if status == 429 and _delay < 20:
+                # too many requests: start some exponential delay
+                increased_delay = (_delay * 2) or 1
+
+                return get_remote_file_content(
+                    url,
+                    as_text=as_text,
+                    headers_only=headers_only,
+                    _delay=increased_delay,
+                )
+
+            else:
+                raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}")
+
+        if headers_only:
+            return response.headers, None
+
+        return response.headers, response.text if as_text else response.content
+
+
+if __name__ == "__main__":
+    create_copyright_tests()

requirements.txt

@@ -10,7 +10,7 @@ chardet==5.0.0
 charset-normalizer==2.1.0
 click==8.1.7
 colorama==0.4.5
-commoncode==31.0.3
+commoncode==31.2.1
 construct==2.10.68
 container-inspector==31.1.0
 cryptography==42.0.5