fix: Remove any YAML front matter from ScanCode license files

ScanCode 32.0.0 started to prepend its `*.LICENSE` files with
YAML-encoded metadata, see [1]. This is a hot fix to remove this header,
if present, from the license files. A better solution will be
implemented later as part of a larger refactoring of license providers.

Different ScanCode versions also differ in whether license files come
with a final newline or not. Align on not having a final newline to make
tests pass either way.

[1]: aboutcode-org/scancode-toolkit#3100

Signed-off-by: Sebastian Schuberth <[email protected]>

Author: sschuberth

plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json

@@ -11,7 +11,7 @@
   "dataLicense" : "CC0-1.0",
   "comment" : "some document comment",
   "hasExtractedLicensingInfos" : [ {
-    "extractedText" : "ASMUS License\n\nDisclaimer and legal rights\n---------------------------\n\nThis file contains bugs. All representations to the contrary are void.\n\nSource code in this file and the accompanying headers and included \nfiles may be distributed free of charge by anyone, as long as full \ncredit is given and any and all liabilities are assumed by the \nrecipient.\n",
+    "extractedText" : "ASMUS License\n\nDisclaimer and legal rights\n---------------------------\n\nThis file contains bugs. All representations to the contrary are void.\n\nSource code in this file and the accompanying headers and included \nfiles may be distributed free of charge by anyone, as long as full \ncredit is given and any and all liabilities are assumed by the \nrecipient.",
     "licenseId" : "LicenseRef-scancode-asmus"
   }, {
     "extractedText" : "To anyone who acknowledges that the file \"sRGB Color Space Profile.icm\" \nis provided \"AS IS\" WITH NO EXPRESS OR IMPLIED WARRANTY:\npermission to use, copy and distribute this file for any purpose is hereby \ngranted without fee, provided that the file is not changed including the HP \ncopyright notice tag, and that the name of Hewlett-Packard Company not be \nused in advertising or publicity pertaining to distribution of the software \nwithout specific, written prior permission. Hewlett-Packard Company makes \nno representations about the suitability of this software for any purpose.",

plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml

@@ -15,7 +15,7 @@ hasExtractedLicensingInfos:
     \nThis file contains bugs. All representations to the contrary are void.\n\nSource\
     \ code in this file and the accompanying headers and included \nfiles may be distributed\
     \ free of charge by anyone, as long as full \ncredit is given and any and all\
-    \ liabilities are assumed by the \nrecipient.\n"
+    \ liabilities are assumed by the \nrecipient."
   licenseId: "LicenseRef-scancode-asmus"
 - extractedText: "To anyone who acknowledges that the file \"sRGB Color Space Profile.icm\"\
     \ \nis provided \"AS IS\" WITH NO EXPRESS OR IMPLIED WARRANTY:\npermission to\

utils/spdx/src/main/kotlin/Utils.kt

@@ -130,8 +130,18 @@ fun getLicenseTextReader(
 ): (() -> String)? {
     return if (id.startsWith(LICENSE_REF_PREFIX)) {
         getLicenseTextResource(id)?.let { { it.readText() } }
-            ?: addScanCodeLicenseTextsDir(licenseTextDirectories).firstNotNullOfOrNull {
-                getLicenseTextFile(id, it)?.let { file -> { file.readText() } }
+            ?: addScanCodeLicenseTextsDir(licenseTextDirectories).firstNotNullOfOrNull { dir ->
+                getLicenseTextFile(id, dir)?.let { file ->
+                    {
+                        val lines = file.readLines()
+
+                        // Remove any YAML front matter enclosed by "---" from ScanCode license files.
+                        val licenseLines = lines.takeUnless { it.first() == "---" }
+                            ?: lines.drop(1).dropWhile { it != "---" }.drop(1)
+
+                        licenseLines.joinToString("\n").trim()
+                    }
+                }
             }
     } else {
         SpdxLicense.forId(id.removeSuffix("+"))?.let { { it.text } }

utils/spdx/src/test/kotlin/UtilsTest.kt

@@ -184,7 +184,7 @@ class UtilsTest : WordSpec() {
         "getLicenseText provided a custom dir" should {
             "return the custom license text for a license ID not known by ort but in custom dir" {
                 val id = "LicenseRef-ort-abc"
-                val text = "a\nb\nc\n"
+                val text = "a\nb\nc"
 
                 setupTempFile(id, text)