Skip to content

Commit ac530bb

Browse files
Add poetry and pyproject.toml support
* Add poetry.lock support * Add pyproject.toml dependencies support for poetry * Add pyproject.toml dependencies support for standard python projects * Add poetry package assembly Reference: #3753 Reference: #2109 Signed-off-by: Ayan Sinha Mahapatra <[email protected]>
1 parent 6a7b359 commit ac530bb

15 files changed

+4748
-9
lines changed

src/packagedcode/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@
174174
pypi.PypiWheelHandler,
175175
pypi.PyprojectTomlHandler,
176176
pypi.PoetryPyprojectTomlHandler,
177+
pypi.PoetryLockHandler,
177178
pypi.PythonEditableInstallationPkgInfoFile,
178179
pypi.PythonEggPkgInfoFile,
179180
pypi.PythonInstalledWheelMetadataFile,

src/packagedcode/pypi.py

+258-2
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,19 @@ def parse(cls, location, package_only=False):
484484
if license_file:
485485
extra_data['license_file'] = license_file
486486

487+
dependencies = []
488+
parsed_dependencies = get_requires_dependencies(
489+
requires=project_data.get("dependencies", []),
490+
)
491+
dependencies.extend(parsed_dependencies)
492+
493+
for dep_type, deps in project_data.get("optional-dependencies", {}).items():
494+
parsed_dependencies = get_requires_dependencies(
495+
requires=deps,
496+
default_scope=dep_type,
497+
)
498+
dependencies.extend(parsed_dependencies)
499+
487500
package_data = dict(
488501
datasource_id=cls.datasource_id,
489502
type=cls.default_package_type,
@@ -494,6 +507,7 @@ def parse(cls, location, package_only=False):
494507
description=description,
495508
keywords=get_keywords(project_data),
496509
parties=get_pyproject_toml_parties(project_data),
510+
dependencies=dependencies,
497511
extra_data=extra_data,
498512
**urls,
499513
)
@@ -510,7 +524,75 @@ def is_poetry_pyproject_toml(location):
510524
return False
511525

512526

513-
class PoetryPyprojectTomlHandler(BaseExtractedPythonLayout):
527+
class BasePoetryPythonLayout(BaseExtractedPythonLayout):
528+
"""
529+
Base class for poetry python projects.
530+
"""
531+
532+
@classmethod
533+
def assemble(cls, package_data, resource, codebase, package_adder):
534+
535+
package_resource = None
536+
if resource.name == 'pyproject.toml':
537+
package_resource = resource
538+
elif resource.name == 'poetry.lock':
539+
if resource.has_parent():
540+
siblings = resource.siblings(codebase)
541+
package_resource = [r for r in siblings if r.name == 'pyproject.toml']
542+
if package_resource:
543+
package_resource = package_resource[0]
544+
545+
if not package_resource:
546+
# we do not have a pyproject.toml
547+
yield from yield_dependencies_from_package_resource(resource)
548+
return
549+
550+
if codebase.has_single_resource:
551+
yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder)
552+
return
553+
554+
assert len(package_resource.package_data) == 1, f'Invalid pyproject.toml for {package_resource.path}'
555+
pkg_data = package_resource.package_data[0]
556+
pkg_data = models.PackageData.from_dict(pkg_data)
557+
558+
if pkg_data.purl:
559+
package = models.Package.from_package_data(
560+
package_data=pkg_data,
561+
datafile_path=package_resource.path,
562+
)
563+
package_uid = package.package_uid
564+
package.populate_license_fields()
565+
yield package
566+
567+
root = package_resource.parent(codebase)
568+
if root:
569+
for pypi_res in cls.walk_pypi(resource=root, codebase=codebase):
570+
if package_uid and package_uid not in pypi_res.for_packages:
571+
package_adder(package_uid, pypi_res, codebase)
572+
yield pypi_res
573+
574+
yield package_resource
575+
576+
else:
577+
# we have no package, so deps are not for a specific package uid
578+
package_uid = None
579+
580+
# in all cases yield possible dependencies
581+
yield from yield_dependencies_from_package_data(pkg_data, package_resource.path, package_uid)
582+
583+
# we yield this as we do not want this further processed
584+
yield package_resource
585+
586+
for lock_file in package_resource.siblings(codebase):
587+
if lock_file.name == 'poetry.lock':
588+
yield from yield_dependencies_from_package_resource(lock_file, package_uid)
589+
590+
if package_uid and package_uid not in lock_file.for_packages:
591+
package_adder(package_uid, lock_file, codebase)
592+
yield lock_file
593+
594+
595+
class PoetryPyprojectTomlHandler(BasePoetryPythonLayout):
514596
datasource_id = 'pypi_poetry_pyproject_toml'
515597
path_patterns = ('*pyproject.toml',)
516598
default_package_type = 'pypi'
@@ -525,14 +607,53 @@ def is_datafile(cls, location, filetypes=tuple()):
525607
and is_poetry_pyproject_toml(location)
526608
)
527609

610+
@classmethod
611+
def parse_non_group_dependencies(cls, dependencies, dev=False):
612+
dependency_mappings = []
613+
for dep_name, requirement in dependencies.items():
614+
if not dev and dep_name == "python":
615+
continue
616+
617+
purl = PackageURL(
618+
type=cls.default_package_type,
619+
name=dep_name,
620+
)
621+
is_optional = False
622+
if dev:
623+
is_optional = True
624+
625+
extra_data = {}
626+
if isinstance(requirement, str):
627+
extracted_requirement = requirement
628+
elif isinstance(requirement, dict):
629+
extracted_requirement = requirement.get("version")
630+
is_optional = requirement.get("optional", is_optional)
631+
python_version = requirement.get("python")
632+
if python_version:
633+
extra_data["python_version"] = python_version
634+
635+
dependency = models.DependentPackage(
636+
purl=purl.to_string(),
637+
extracted_requirement=extracted_requirement,
638+
scope="install",
639+
is_runtime=True,
640+
is_optional=is_optional,
641+
is_direct=True,
642+
is_resolved=False,
643+
extra_data=extra_data,
644+
)
645+
dependency_mappings.append(dependency.to_dict())
646+
647+
return dependency_mappings
648+
528649
@classmethod
529650
def parse(cls, location, package_only=False):
530651
toml_data = toml.load(location, _dict=dict)
531652

532653
tool_data = toml_data.get('tool')
533654
if not tool_data:
534655
return
535-
656+
536657
poetry_data = tool_data.get('poetry')
537658
if not poetry_data:
538659
return
@@ -548,6 +669,34 @@ def parse(cls, location, package_only=False):
548669
if license_file:
549670
extra_data['license_file'] = license_file
550671

672+
dependencies = []
673+
parsed_deps = cls.parse_non_group_dependencies(
674+
dependencies=poetry_data.get("dependencies", {}),
675+
)
676+
dependencies.extend(parsed_deps)
677+
parsed_deps = cls.parse_non_group_dependencies(
678+
dependencies=poetry_data.get("dev-dependencies", {}),
679+
dev=True,
680+
)
681+
dependencies.extend(parsed_deps)
682+
683+
for group_name, group_deps in poetry_data.get("group", {}).items():
684+
for name, requirement in group_deps.get("dependencies", {}).items():
685+
purl = PackageURL(
686+
type=cls.default_package_type,
687+
name=name,
688+
)
689+
dependency = models.DependentPackage(
690+
purl=purl.to_string(),
691+
extracted_requirement=requirement,
692+
scope=group_name,
693+
is_runtime=True,
694+
is_optional=False,
695+
is_direct=True,
696+
is_resolved=False,
697+
)
698+
dependencies.append(dependency.to_dict())
699+
551700
package_data = dict(
552701
datasource_id=cls.datasource_id,
553702
type=cls.default_package_type,
@@ -559,11 +708,118 @@ def parse(cls, location, package_only=False):
559708
keywords=get_keywords(poetry_data),
560709
parties=get_pyproject_toml_parties(poetry_data),
561710
extra_data=extra_data,
711+
dependencies=dependencies,
562712
**urls,
563713
)
564714
yield models.PackageData.from_data(package_data, package_only)
565715

566716

717+
class PoetryLockHandler(BasePoetryPythonLayout):
718+
datasource_id = 'pypi_poetry_lock'
719+
path_patterns = ('*poetry.lock',)
720+
default_package_type = 'pypi'
721+
default_primary_language = 'Python'
722+
description = 'Python poetry lockfile'
723+
documentation_url = 'https://python-poetry.org/docs/basic-usage/#installing-with-poetrylock'
724+
725+
@classmethod
726+
def parse(cls, location, package_only=False):
727+
toml_data = toml.load(location, _dict=dict)
728+
729+
packages = toml_data.get('package')
730+
if not packages:
731+
return
732+
733+
metadata = toml_data.get('metadata')
734+
735+
dependencies = []
736+
for package in packages:
737+
dependencies_for_resolved = []
738+
739+
deps = package.get("dependencies") or {}
740+
for name, requirement in deps.items():
741+
purl = PackageURL(
742+
type=cls.default_package_type,
743+
name=name,
744+
)
745+
dependency = models.DependentPackage(
746+
purl=purl.to_string(),
747+
extracted_requirement=requirement,
748+
scope="install",
749+
is_runtime=True,
750+
is_optional=False,
751+
is_direct=True,
752+
is_resolved=False,
753+
)
754+
dependencies_for_resolved.append(dependency.to_dict())
755+
756+
extra_deps = package.get("extras") or {}
757+
for group_name, group_deps in extra_deps.items():
758+
for dep in group_deps:
759+
if " (" in dep and ")" in dep:
760+
name, requirement = dep.split(" (")
761+
requirement = requirement.rstrip(")")
762+
else:
763+
requirement = None
764+
name = dep
765+
purl = PackageURL(
766+
type=cls.default_package_type,
767+
name=name,
768+
)
769+
dependency = models.DependentPackage(
770+
purl=purl.to_string(),
771+
extracted_requirement=requirement,
772+
scope=group_name,
773+
is_runtime=True,
774+
is_optional=True,
775+
is_direct=True,
776+
is_resolved=False,
777+
)
778+
dependencies_for_resolved.append(dependency.to_dict())
779+
780+
name = package.get('name')
781+
version = package.get('version')
782+
urls = get_pypi_urls(name, version)
783+
package_data = dict(
784+
datasource_id=cls.datasource_id,
785+
type=cls.default_package_type,
786+
primary_language='Python',
787+
name=name,
788+
version=version,
789+
description=metadata.get('description'),
790+
is_virtual=True,
791+
dependencies=dependencies_for_resolved,
792+
**urls,
793+
)
794+
resolved_package = models.PackageData.from_data(package_data, package_only)
795+
796+
is_optional = package.get("is_optional") or True
797+
dependency = models.DependentPackage(
798+
purl=resolved_package.purl,
799+
extracted_requirement=None,
800+
scope=None,
801+
is_runtime=True,
802+
is_optional=is_optional,
803+
is_direct=False,
804+
is_resolved=True,
805+
resolved_package=resolved_package.to_dict()
806+
)
807+
dependencies.append(dependency.to_dict())
808+
809+
extra_data = {}
810+
extra_data['python_version'] = metadata.get("python-versions")
811+
extra_data['lock_version'] = metadata.get("lock-version")
812+
813+
package_data = dict(
814+
datasource_id=cls.datasource_id,
815+
type=cls.default_package_type,
816+
primary_language='Python',
817+
extra_data=extra_data,
818+
dependencies=dependencies,
819+
)
820+
yield models.PackageData.from_data(package_data, package_only)
821+
822+
567823
class PipInspectDeplockHandler(models.DatafileHandler):
568824
datasource_id = 'pypi_inspect_deplock'
569825
path_patterns = ('*pip-inspect.deplock',)

0 commit comments

Comments
 (0)