Skip to content

Commit 0d294bf

Browse files
pombredannepombredanne
authored
Merge pull request #3197 from nexB/fix-license-dump
Update license db generation
2 parents 0aa964e + 567e9ff commit 0d294bf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+63
-1543
lines changed

‎CHANGELOG.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ License detection:
175175

176176
- See https://github.com/nexB/scancode-toolkit/issues/3049
177177

178-
- There is a new ``--get-license-data`` scancode command line option to export
178+
- There is a new console script ``scancode-license-data`` to export
179179
license data in JSON, YAML and HTML, with indexes and a static website for use
180180
in the licensedb web site. This becomes the API way to getr scancode license
181181
data.

‎setup-mini.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ packages =
147147
console_scripts =
148148
scancode = scancode.cli:scancode
149149
scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
150+
scancode-license-data = licensedcode.license_db:dump_scancode_license_data
150151

151152
# These are configurations for ScanCode plugins as setuptools entry points.
152153
# Each plugin entry hast this form:

‎setup.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ packages =
147147
console_scripts =
148148
scancode = scancode.cli:scancode
149149
scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
150+
scancode-license-data = licensedcode.license_db:dump_scancode_license_data
150151

151152
# These are configurations for ScanCode plugins as setuptools entry points.
152153
# Each plugin entry hast this form:

‎src/licensedcode/cache.py

+6
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,12 @@ def load_or_build(
182182
# TODO: handle unable to lock in a nicer way
183183
raise
184184

185+
@property
186+
def has_additional_licenses(self):
187+
cache = get_cache()
188+
if cache.additional_license_directory or cache.additional_license_plugins:
189+
return True
190+
185191

186192
def build_index(
187193
licenses_db=None,

‎src/licensedcode/license_db.py

+38-10
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@
3131
from os.path import join
3232
from distutils.dir_util import copy_tree
3333

34+
import click
3435
import saneyaml
36+
37+
from commoncode.cliutils import MISC_GROUP
38+
from commoncode.cliutils import PluggableCommandLineOption
3539
from jinja2 import Environment, FileSystemLoader
3640
from licensedcode.models import load_licenses
3741
from licensedcode.models import licenses_data_dir
@@ -126,13 +130,16 @@ def generate_details(output_path, environment, licenses, test=False):
126130
127131
``test`` is to generate a stable output for testing only
128132
"""
133+
from licensedcode.cache import get_cache
134+
include_builtin = get_cache().has_additional_licenses
135+
129136
if test:
130137
base_context_mapping = base_context_test
131138
else:
132139
base_context_mapping = base_context
133140
license_details_template = environment.get_template("license_details.html")
134141
for lic in licenses.values():
135-
license_data = lic.to_dict(include_text=True)
142+
license_data = lic.to_dict(include_text=False, include_builtin=include_builtin)
136143
html = license_details_template.render(
137144
**base_context_mapping,
138145
license=lic,
@@ -200,19 +207,40 @@ def generate(
200207
return count
201208

202209

203-
def dump_license_data(ctx, param, value):
210+
def scancode_license_data(path):
204211
"""
205-
Dump license data from scancode licenses to the directory ``value`` passed
212+
Dump license data from scancode licenses to the directory ``path`` passed
206213
in from command line.
207214
208215
Dumps data in JSON, YAML and HTML formats and also dumps the .LICENSE file
209216
with the license text and the data as YAML frontmatter.
210217
"""
211-
if not value or ctx.resilient_parsing:
212-
return
213-
214-
import click
215-
click.secho(f'Dumping license data to: {value}', err=True)
216-
count = generate(build_location=value)
218+
click.secho(f'Dumping license data to: {path}', err=True)
219+
count = generate(build_location=path)
217220
click.secho(f'Done dumping #{count} licenses.', err=True)
218-
ctx.exit(0)
221+
222+
223+
@click.command(name='scancode-license-data')
224+
@click.option(
225+
'--path',
226+
type=click.Path(exists=False, writable=True, file_okay=False, resolve_path=True, path_type=str),
227+
metavar='DIR',
228+
help='Dump the license data in this directory in the LicenseDB format and exit. '
229+
'Creates the directory if it does not exist. ',
230+
help_group=MISC_GROUP,
231+
cls=PluggableCommandLineOption,
232+
)
233+
@click.help_option('-h', '--help')
234+
def dump_scancode_license_data(
235+
path,
236+
*args,
237+
**kwargs,
238+
):
239+
"""
240+
Dump scancode license data in various formats, and the licenseDB static website at `path`.
241+
"""
242+
scancode_license_data(path=path)
243+
244+
245+
if __name__ == '__main__':
246+
dump_scancode_license_data()

‎src/licensedcode/plugin_license.py

+3-21
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,9 @@
1212
from functools import partial
1313

1414
import attr
15-
import click
1615
from commoncode.cliutils import PluggableCommandLineOption
1716
from commoncode.cliutils import SCAN_GROUP
1817
from commoncode.cliutils import SCAN_OPTIONS_GROUP
19-
from commoncode.cliutils import MISC_GROUP
2018
from plugincode.scan import ScanPlugin
2119
from plugincode.scan import scan_impl
2220

@@ -33,7 +31,6 @@
3331
from licensedcode.detection import LicenseDetectionFromResult
3432
from licensedcode.detection import LicenseMatchFromResult
3533
from licensedcode.detection import UniqueDetection
36-
from licensedcode.license_db import dump_license_data
3734
from packagedcode.utils import combine_expressions
3835
from scancode.api import SCANCODE_LICENSEDB_URL
3936

@@ -121,19 +118,7 @@ class LicenseScanner(ScanPlugin):
121118
required_options=['license'],
122119
help='[EXPERIMENTAL] Detect unknown licenses. ',
123120
help_group=SCAN_OPTIONS_GROUP,
124-
),
125-
126-
# TODO: consider creating a separate comamnd line option exe instead
127-
PluggableCommandLineOption(
128-
('--dump-license-data',),
129-
type=click.Path(exists=False, readable=True, file_okay=False, resolve_path=True, path_type=str),
130-
metavar='DIR',
131-
callback=dump_license_data,
132-
help='Dump the license data in this directory in the LicenseDB format and exit. '
133-
'Creates the directory if it does not exist. ',
134-
help_group=MISC_GROUP,
135-
is_eager=True,
136-
),
121+
)
137122
]
138123

139124
def is_enabled(self, license, **kwargs): # NOQA
@@ -176,20 +161,17 @@ def process_codebase(self, codebase, **kwargs):
176161
cche = cache.get_cache()
177162

178163
cle = codebase.get_or_create_current_header()
179-
has_additional_licenses = False
180164

181165
if cche.additional_license_directory:
182166
cle.extra_data['additional_license_directory'] = cche.additional_license_directory
183-
has_additional_licenses = True
184167

185168
if cche.additional_license_plugins:
186169
cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins
187-
has_additional_licenses = True
188170

189-
if TRACE and has_additional_licenses:
171+
if TRACE and cche.has_additional_licenses:
190172
logger_debug(
191173
f'add_referenced_filenames_license_matches: additional_licenses',
192-
f'has_additional_licenses: {has_additional_licenses}\n',
174+
f'has_additional_licenses: {cche.has_additional_licenses}\n',
193175
f'additional_license_directory: {cche.additional_license_directory}\n',
194176
f'additional_license_plugins : {cche.additional_license_plugins}'
195177
)

‎src/licensedcode/templates/footer.html

+1
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@
88
</p>
99
<p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
1010
<p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> {{ scancode_version }} on {{ now }}.</p>
11+
<p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
1112
</footer>
1213
</div>

‎tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.html

-20
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,6 @@
106106

107107
</dd>
108108

109-
<dt style="">is_builtin</dt>
110-
<dd>
111-
112-
True
113-
114-
</dd>
115-
116109
<dt style="">is_exception</dt>
117110
<dd>
118111

@@ -136,19 +129,6 @@
136129

137130
</dd>
138131

139-
<dt style="">text</dt>
140-
<dd>
141-
142-
The Free Software Foundation has exempted Bash from the requirement of
143-
Paragraph 2c of the General Public License. This is to say, there is
144-
no requirement for Bash to print a notice when it is started
145-
interactively in the usual way. We made this exception because users
146-
and standards expect shells not to print such messages. This
147-
exception applies to any program that serves as a shell and that is
148-
based primarily on Bash as opposed to other GNU software.
149-
150-
</dd>
151-
152132
</dl>
153133
<div class="text-bold">license_text</div>
154134
<pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>The Free Software Foundation has exempted Bash from the requirement of

‎tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@
55
"category": "Copyleft",
66
"owner": "Free Software Foundation (FSF)",
77
"notes": "this used with GPL 1.0 and 2.0. It was removed from the V3 text https://git.savannah.gnu.org/cgit/bash.git/commit/COPYING?id=3185942a5234e26ab13fa02f9c51d340cec514f8",
8-
"is_builtin": true,
98
"is_exception": true,
109
"spdx_license_key": "LicenseRef-scancode-bash-exception-gpl-2.0",
1110
"text_urls": [
1211
"https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283"
13-
],
14-
"text": "The Free Software Foundation has exempted Bash from the requirement of\nParagraph 2c of the General Public License. This is to say, there is\nno requirement for Bash to print a notice when it is started\ninteractively in the usual way. We made this exception because users\nand standards expect shells not to print such messages. This\nexception applies to any program that serves as a shell and that is\nbased primarily on Bash as opposed to other GNU software."
12+
]
1513
}

‎tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.yml

-9
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,7 @@ name: Bash exception to GPL
44
category: Copyleft
55
owner: Free Software Foundation (FSF)
66
notes: this used with GPL 1.0 and 2.0. It was removed from the V3 text https://git.savannah.gnu.org/cgit/bash.git/commit/COPYING?id=3185942a5234e26ab13fa02f9c51d340cec514f8
7-
is_builtin: yes
87
is_exception: yes
98
spdx_license_key: LicenseRef-scancode-bash-exception-gpl-2.0
109
text_urls:
1110
- https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283
12-
text: |
13-
The Free Software Foundation has exempted Bash from the requirement of
14-
Paragraph 2c of the General Public License. This is to say, there is
15-
no requirement for Bash to print a notice when it is started
16-
interactively in the usual way. We made this exception because users
17-
and standards expect shells not to print such messages. This
18-
exception applies to any program that serves as a shell and that is
19-
based primarily on Bash as opposed to other GNU software.

‎tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.html

-39
Original file line numberDiff line numberDiff line change
@@ -106,52 +106,13 @@
106106

107107
</dd>
108108

109-
<dt style="">is_builtin</dt>
110-
<dd>
111-
112-
True
113-
114-
</dd>
115-
116109
<dt style="">spdx_license_key</dt>
117110
<dd>
118111

119112
LicenseRef-scancode-binary-linux-firmware
120113

121114
</dd>
122115

123-
<dt style="">text</dt>
124-
<dd>
125-
126-
Redistribution. Redistribution and use in binary form, without
127-
modification, are permitted provided that the following conditions are
128-
met:
129-
130-
* Redistributions must reproduce the above copyright notice and the
131-
following disclaimer in the documentation and/or other materials
132-
provided with the distribution.
133-
134-
* Neither the name of the Copyright Holder nor the names of its
135-
suppliers may be used to endorse or promote products derived from this
136-
software without specific prior written permission.
137-
138-
* No reverse engineering, decompilation, or disassembly of this software
139-
is permitted.
140-
141-
DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
142-
CONTRIBUTORS &#34;AS IS&#34; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
143-
BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
144-
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
145-
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
146-
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
147-
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
148-
USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
149-
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
150-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
151-
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE
152-
153-
</dd>
154-
155116
</dl>
156117
<div class="text-bold">license_text</div>
157118
<pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>Redistribution. Redistribution and use in binary form, without

‎tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,5 @@
55
"category": "Proprietary Free",
66
"owner": "Unspecified",
77
"homepage_url": "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/",
8-
"is_builtin": true,
9-
"spdx_license_key": "LicenseRef-scancode-binary-linux-firmware",
10-
"text": "Redistribution. Redistribution and use in binary form, without\nmodification, are permitted provided that the following conditions are\nmet:\n\n* Redistributions must reproduce the above copyright notice and the\nfollowing disclaimer in the documentation and/or other materials\nprovided with the distribution.\n\n* Neither the name of the Copyright Holder nor the names of its\nsuppliers may be used to endorse or promote products derived from this\nsoftware without specific prior written permission.\n\n* No reverse engineering, decompilation, or disassembly of this software\nis permitted.\n\nDISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND\nCONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,\nBUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE\nCOPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,\nINCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\nNOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\nUSE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\nANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\nTHIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE"
8+
"spdx_license_key": "LicenseRef-scancode-binary-linux-firmware"
119
}

‎tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.yml

-28
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,4 @@ name: Binary-Only Linux Firmware License
44
category: Proprietary Free
55
owner: Unspecified
66
homepage_url: https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/
7-
is_builtin: yes
87
spdx_license_key: LicenseRef-scancode-binary-linux-firmware
9-
text: |
10-
Redistribution. Redistribution and use in binary form, without
11-
modification, are permitted provided that the following conditions are
12-
met:
13-
14-
* Redistributions must reproduce the above copyright notice and the
15-
following disclaimer in the documentation and/or other materials
16-
provided with the distribution.
17-
18-
* Neither the name of the Copyright Holder nor the names of its
19-
suppliers may be used to endorse or promote products derived from this
20-
software without specific prior written permission.
21-
22-
* No reverse engineering, decompilation, or disassembly of this software
23-
is permitted.
24-
25-
DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
26-
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
27-
BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
28-
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29-
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30-
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
31-
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
32-
USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
33-
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
35-
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE

‎tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.html

-14
Original file line numberDiff line numberDiff line change
@@ -99,13 +99,6 @@
9999

100100
</dd>
101101

102-
<dt style="">is_builtin</dt>
103-
<dd>
104-
105-
True
106-
107-
</dd>
108-
109102
<dt style="">is_exception</dt>
110103
<dd>
111104

@@ -149,13 +142,6 @@
149142

150143
</dd>
151144

152-
<dt style="">text</dt>
153-
<dd>
154-
155-
As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison.
156-
157-
</dd>
158-
159145
</dl>
160146
<div class="text-bold">license_text</div>
161147
<pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison.</code></pre>

‎tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,8 @@
44
"name": "Bison exception to GPL 2.0 or later",
55
"category": "Copyleft Limited",
66
"owner": "Free Software Foundation (FSF)",
7-
"is_builtin": true,
87
"is_exception": true,
98
"spdx_license_key": "LicenseRef-scancode-bison-exception-2.0",
109
"faq_url": "http://www.gnu.org/software/bison/manual/bison.html#Conditions",
11-
"standard_notice": "This library is free software; you can redistribute it and/or modify it\nunder the terms of the GNU General Public License as published by the Free\nSoftware Foundation; either version 2, or (at your option) any later\nversion.\nThis library is distributed in the hope that it will be useful, but WITHOUT\nANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\nFITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for\nmore details.\nYou should have received a copy of the GNU General Public License along\nwith this library; see the file COPYING. If not, write to the Free Software\nFoundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\nAs a special exception, when this file is copied by Bison into a Bison\noutput file, you may use that output file without restriction. This special\nexception was added by the Free Software Foundation in version 1.24 of\nBison.\n",
12-
"text": "As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison."
10+
"standard_notice": "This library is free software; you can redistribute it and/or modify it\nunder the terms of the GNU General Public License as published by the Free\nSoftware Foundation; either version 2, or (at your option) any later\nversion.\nThis library is distributed in the hope that it will be useful, but WITHOUT\nANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\nFITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for\nmore details.\nYou should have received a copy of the GNU General Public License along\nwith this library; see the file COPYING. If not, write to the Free Software\nFoundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\nAs a special exception, when this file is copied by Bison into a Bison\noutput file, you may use that output file without restriction. This special\nexception was added by the Free Software Foundation in version 1.24 of\nBison.\n"
1311
}

0 commit comments

Comments
 (0)