Skip to content

Commit ea8953b

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
authored
Merge pull request #3662 from nexB/misc-updates
Add Misc updates
2 parents 66d7166 + 6df90d5 commit ea8953b

File tree

15 files changed

+98
-89
lines changed

15 files changed

+98
-89
lines changed

‎src/cluecode/copyrights.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -2517,7 +2517,8 @@ def build_detection_from_node(
25172517
COMPANY: {<COMPANY> <MAINT>} #19603
25182518
25192519
2520-
#######################################
2520+
################################# #COPYRIGHT: {<COPY> <COPY> <MIT>} #1802
2521+
######
25212522
# VARIOUS FORMS OF COPYRIGHT
25222523
#######################################
25232524
@@ -2572,6 +2573,8 @@ def build_detection_from_node(
25722573
25732574
COPYRIGHT: {<COPY> <COPY> <COMP>+} #1690
25742575
2576+
COPYRIGHT: {<COPY> <COPY> <MIT>} #1802
2577+
25752578
COPYRIGHT: {<COPY> <COPY> <NN>+ <COMPANY|NAME|NAME-EMAIL>+} #1710
25762579
25772580
COPYRIGHT: {<COPY> <COPY> <NN> <NNP> <NN> <COMPANY>} #1711
@@ -4125,8 +4128,10 @@ def prepare_text_line(line, dedeb=True, to_ascii=True):
41254128

41264129
# normalize (possibly repeated) quotes to unique single quote '
41274130
# backticks ` and "
4128-
.replace('`', u"'")
4129-
.replace('"', u"'")
4131+
.replace('`', "'")
4132+
.replace('"', "'")
4133+
# see https://github.com/nexB/scancode-toolkit/issues/3667
4134+
.replace('§', " ")
41304135
)
41314136

41324137
if TRACE_TOK:

‎src/packagedcode/gemfile_lock.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -340,8 +340,8 @@ def get_option(s):
340340
'%(NAME_VERSION)s'
341341
'$' % locals()).match
342342

343-
PLATS = re.compile('^ (?P<platform>.*)$').match
344-
BUNDLED_WITH = re.compile('^\s+(?P<version>(?:\d+.)+\d+)\s*$').match
343+
PLATS = re.compile(r'^ (?P<platform>.*)$').match
344+
BUNDLED_WITH = re.compile(r'^\s+(?P<version>(?:\d+.)+\d+)\s*$').match
345345

346346

347347
class GemfileLockParser:

‎src/packagedcode/recognize.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from packagedcode import ALL_DATAFILE_HANDLERS
1717
from packagedcode import models
1818

19-
TRACE = False or os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
19+
TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
2020

2121

2222
def logger_debug(*args):
@@ -62,7 +62,6 @@ def recognize_package_data(
6262
datafile_handlers = APPLICATION_PACKAGE_DATAFILE_HANDLERS
6363
elif system:
6464
datafile_handlers = SYSTEM_PACKAGE_DATAFILE_HANDLERS
65-
6665
return list(_parse(location, datafile_handlers=datafile_handlers))
6766

6867

@@ -78,6 +77,9 @@ def _parse(
7877
"""
7978

8079
for handler in datafile_handlers:
80+
if TRACE:
81+
logger_debug(f'_parse:.is_datafile: {handler}')
82+
8183
if not handler.is_datafile(location):
8284
continue
8385

‎src/packagedcode/rpm.py

+3
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,9 @@ class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
233233
# TODO: add dependencies!!!
234234
class RpmInstalledSqliteDatabaseHandler(BaseRpmInstalledDatabaseHandler):
235235
# used by newer RHEL/CentOS/Fedora/CoreOS
236+
# Filetype: SQLite 3.x database, ...
237+
# Mimetype: application/vnd.sqlite3
238+
236239
datasource_id = 'rpm_installed_database_sqlite'
237240
path_patterns = ('*rpm/rpmdb.sqlite',)
238241
default_package_type = 'rpm'

‎src/summarycode/generated.py

+12
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,17 @@ def generated_scanner(location, **kwargs):
176176
# protoc
177177
'generated by the protocol buffer compiler',
178178
'generated by the protocol buffer compiler. do not edit!',
179+
'generated by the protocol buffer compiler. do not edit!',
180+
'code generated by protoc-gen-go. do not edit.',
181+
'code generated by generate-types. do not edit.',
182+
'code generated by generate-protos. do not edit.',
183+
'do not edit -- your changes will be discarded when the file is', # next line: * regenerated.
184+
'do not edit -- generated code',
185+
'generated code -- do not edit!',
186+
'autogenerated by method_dump.sh. do not edit by hand.',
187+
'this file is generated from the .proto files for the well-known', # next line: types. Do not edit!
188+
'this file was generated by upbc (the upb compiler) from the input',
189+
'do not edit -- your changes will be discarded when the file is', # next line: regenerated
179190

180191
# autotools
181192
'makefile.in generated by automake',
@@ -191,6 +202,7 @@ def generated_scanner(location, **kwargs):
191202
'generated by:javacc: do not edit this line',
192203
'generated by:jjtree: do not edit this line',
193204
'generated code (do not edit this line)',
205+
194206
))
195207

196208

‎src/textcode/strings.py

+8-8
Original file line numberDiff line numberDiff line change
@@ -180,33 +180,33 @@ def is_shared_object(s):
180180
return so(s)
181181

182182

183+
# TODO: implement me
184+
_posix = re.compile('^/[\\w_\\-].*$', re.IGNORECASE).match
183185
def is_posix_path(s):
184186
"""
185187
Return True if s looks like a posix path.
186188
Example: /usr/lib/librt.so.1 or /usr/lib
187189
"""
188-
# TODO: implement me
189-
posix = re.compile('^/[\\w_\\-].*$', re.IGNORECASE).match
190-
posix(s)
191-
return False
190+
return _posix(s)
192191

193192

193+
# TODO: implement me
194+
_relative = re.compile('^(?:([^/]|\\.\\.)[\\w_\\-]+/.*$)', re.IGNORECASE).match
194195
def is_relative_path(s):
195196
"""
196197
Return True if s looks like a relative posix path.
197198
Example: usr/lib/librt.so.1 or ../usr/lib
198199
"""
199-
relative = re.compile('^(?:([^/]|\\.\\.)[\\w_\\-]+/.*$)', re.IGNORECASE).match
200-
return relative(s)
200+
return bool(_relative(s))
201201

202202

203+
_winpath = re.compile('^[\\w_\\-]+\\.so\\.[0-9]+\\.*.[0-9]*$', re.IGNORECASE).match
203204
def is_win_path(s):
204205
"""
205206
Return True if s looks like a win path.
206207
Example: c:\\usr\\lib\\librt.so.1.
207208
"""
208-
winpath = re.compile('^[\\w_\\-]+\\.so\\.[0-9]+\\.*.[0-9]*$', re.IGNORECASE).match
209-
return winpath(s)
209+
return _winpath(s)
210210

211211

212212
def is_c_source(s):

‎src/textcode/strings2.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import re
2424

2525
ASCII_BYTE = (
26-
" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
27-
"\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"
26+
r" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
27+
r"\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"
2828
)
2929

3030

‎tests/cluecode/data/copyrights/misco2/pom.xml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<!--
3+
Copyright (C) 2016§ The Android Open Source Project
4+
-->

‎tests/cluecode/data/copyrights/misco2/pom.xml.yml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
what:
2+
- copyrights
3+
copyrights:
4+
- Copyright (c) 2016 The Android Open Source Project

‎tests/packagedcode/test_gemfile_lock.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ def test_Gem_as_nv_tree(self):
274274
}
275275
}
276276
}
277-
self.assertEqual(expected, a.as_nv_tree())
277+
assert a.as_nv_tree() == expected
278278

279279
def test_Gem_flatten(self):
280280
Gem = gemfile_lock.Gem
@@ -320,7 +320,6 @@ def test_Gem_as_nv_tree_with_no_deps(self):
320320
results = a.as_nv_tree()
321321
assert results == expected
322322

323-
324323
def test_Gem_to_dict(self):
325324
Gem = gemfile_lock.Gem
326325
a = Gem('a', '1')

‎tests/packagedcode/test_maven.py

+2-3
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,6 @@ def test_maven_unknown_reference_to_license_in_manifest(self):
250250
run_scan_click(['--package', '--license', '--license-diagnostics', '--processes', '-1', test_dir, '--json', result_file])
251251
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
252252

253-
254253
def test_package_dependency_not_missing(self):
255254
test_file = self.get_test_loc('maven2/log4j/log4j-pom.xml')
256255
self.check_parse_to_package(test_file, regen=REGEN_TEST_FIXTURES)
@@ -275,7 +274,7 @@ def test_get_top_level_resources(self):
275274
pom_resource = codebase.get_resource(
276275
'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator/pom.xml'
277276
)
278-
self.assertTrue(pom_resource)
277+
assert pom_resource
279278
top_level_resources_paths = [
280279
r.path for r in maven.MavenPomXmlHandler.get_top_level_resources(pom_resource, codebase)
281280
]
@@ -288,7 +287,7 @@ def test_get_top_level_resources(self):
288287
'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator/pom.properties',
289288
'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator/pom.xml',
290289
]
291-
self.assertEquals(expected_resource_paths, top_level_resources_paths)
290+
assert top_level_resources_paths == expected_resource_paths
292291

293292

294293
class TestPomProperties(testcase.FileBasedTesting):

‎tests/packagedcode/test_pypi.py

+24-48
Original file line numberDiff line numberDiff line change
@@ -282,10 +282,10 @@ def test_parse_metadata_prefer_pkg_info_from_egg_info_from_command_line(self):
282282
# `celery/celery.egg-info/PKG-INFO`
283283
vc = VirtualCodebase(location=result_file)
284284
for dep in vc.attributes.dependencies:
285-
self.assertEqual(dep['datafile_path'], 'celery/celery.egg-info/PKG-INFO')
285+
assert dep['datafile_path'] == 'celery/celery.egg-info/PKG-INFO'
286286
for pkg in vc.attributes.packages:
287287
for path in pkg['datafile_paths']:
288-
self.assertEqual(path, 'celery/celery.egg-info/PKG-INFO')
288+
assert path == 'celery/celery.egg-info/PKG-INFO'
289289

290290

291291
class TestPipRequirementsFileHandler(PackageTester):
@@ -417,47 +417,23 @@ def test_parse_dependency_file_with_invalid_does_not_fail(self):
417417
expected_loc = self.get_test_loc('pypi/requirements_txt/invalid_spec/output.expected.json')
418418
self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
419419

420-
def test_PipRequirementsFileHandler_is_datafile(self):
421-
self.assertEqual(
422-
pypi.PipRequirementsFileHandler.is_datafile('dev-requirements.txt', _bare_filename=True),
423-
True
424-
)
425-
self.assertEqual(
426-
pypi.PipRequirementsFileHandler.is_datafile('requirements.txt', _bare_filename=True),
427-
True
428-
)
429-
self.assertEqual(
430-
pypi.PipRequirementsFileHandler.is_datafile('requirement.txt', _bare_filename=True),
431-
True
432-
)
433-
self.assertEqual(
434-
pypi.PipRequirementsFileHandler.is_datafile('requirements.in', _bare_filename=True),
435-
True
436-
)
437-
self.assertEqual(
438-
pypi.PipRequirementsFileHandler.is_datafile('requirements.pip', _bare_filename=True),
439-
True
440-
)
441-
self.assertEqual(
442-
pypi.PipRequirementsFileHandler.is_datafile('requirements-dev.txt', _bare_filename=True),
443-
True
444-
)
445-
self.assertEqual(
446-
pypi.PipRequirementsFileHandler.is_datafile('some-requirements-dev.txt', _bare_filename=True),
447-
True
448-
)
449-
self.assertEqual(
450-
pypi.PipRequirementsFileHandler.is_datafile('requires.txt', _bare_filename=True),
451-
True
452-
)
453-
self.assertEqual(
454-
pypi.PipRequirementsFileHandler.is_datafile('requirements/base.txt', _bare_filename=True),
455-
True
456-
)
457-
self.assertEqual(
458-
pypi.PipRequirementsFileHandler.is_datafile('reqs.txt', _bare_filename=True),
459-
True
460-
)
420+
@pytest.mark.parametrize(
421+
'filename',
422+
[
423+
'dev-requirements.txt',
424+
'reqs.txt',
425+
'requirements/base.txt',
426+
'requirements-dev.txt',
427+
'requirements.in',
428+
'requirements.pip',
429+
'requirements.txt',
430+
'requirement.txt',
431+
'requires.txt',
432+
'some-requirements-dev.txt',
433+
]
434+
)
435+
def test_PipRequirementsFileHandler_is_datafile(filename):
436+
assert pypi.PipRequirementsFileHandler.is_datafile(location=filename, _bare_filename=True)
461437

462438

463439
class TestPyPiPipfile(PackageTester):
@@ -607,36 +583,36 @@ def check_setup_py_parsing(test_loc):
607583

608584
expected_loc2 = f'{test_loc}-expected.json'
609585
packages_data = pypi.PythonSetupPyHandler.parse(test_loc)
610-
test_envt.check_packages_data(
586+
env.check_packages_data(
611587
packages_data=packages_data,
612588
expected_loc=expected_loc2,
613589
regen=REGEN_TEST_FIXTURES,
614590
must_exist=False,
615591
)
616592

617593

618-
test_envt = PackageTester()
594+
env = PackageTester()
619595

620596

621597
@pytest.mark.parametrize(
622598
'test_loc',
623-
get_setup_py_test_files(os.path.abspath(os.path.join(test_envt.test_data_dir, 'pypi', 'setup.py-versions'))),
599+
get_setup_py_test_files(os.path.abspath(os.path.join(env.test_data_dir, 'pypi', 'setup.py-versions'))),
624600
)
625601
def test_parse_setup_py_with_computed_versions(test_loc):
626602
check_setup_py_parsing(test_loc)
627603

628604

629605
@pytest.mark.parametrize(
630606
'test_loc',
631-
get_setup_py_test_files(os.path.abspath(os.path.join(test_envt.test_data_dir, 'pypi', 'setup.py')))
607+
get_setup_py_test_files(os.path.abspath(os.path.join(env.test_data_dir, 'pypi', 'setup.py')))
632608
)
633609
def test_parse_setup_py(test_loc):
634610
check_setup_py_parsing(test_loc)
635611

636612

637613
@pytest.mark.parametrize(
638614
'test_loc',
639-
get_setup_py_test_files(os.path.abspath(os.path.join(test_envt.test_data_dir, 'pypi', 'more_setup.py'))),
615+
get_setup_py_test_files(os.path.abspath(os.path.join(env.test_data_dir, 'pypi', 'more_setup.py'))),
640616
)
641617
def test_parse_more_setup_py(test_loc):
642618
check_setup_py_parsing(test_loc)

‎tests/packagedcode/test_win_reg.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,14 @@ def test_win_reg_remove_drive_letter(self):
6060
test_path = 'C:\\Users\\Test\\Desktop'
6161
expected_path = 'Users/Test/Desktop'
6262
result = remove_drive_letter(test_path)
63-
self.assertEqual(result, expected_path)
63+
assert result == expected_path
6464

6565
def test_win_reg_create_absolute_installed_file_path(self):
6666
root_dir = '/home/test/c/'
6767
test_path = 'C:\\Program Files\\Test Program\\'
6868
result = create_absolute_installed_file_path(root_dir, test_path)
6969
expected_path = '/home/test/c/Program Files/Test Program'
70-
self.assertEqual(result, expected_path)
70+
assert result == expected_path
7171

7272
def test_scan_system_package_end_to_end_installed_win_reg(self):
7373
test_dir = self.get_test_loc('win_reg/get_installed_packages_docker/layer')

0 commit comments

Comments
 (0)