From d2c9b2aeae7f3168dc34fc65899130c759b39d00 Mon Sep 17 00:00:00 2001 From: Olivier Ramonat Date: Fri, 10 Feb 2023 16:09:05 +0100 Subject: [PATCH] Implement a first version of the SPDX generator This is meant only for generating simple SPDX files in the context of e3-based builds. For #554 --- README.md | 1 + src/e3/spdx.py | 519 ++++++++++++++++++++++++++++++++++++ tests/tests_e3/spdx_test.py | 103 +++++++ 3 files changed, 623 insertions(+) create mode 100644 src/e3/spdx.py create mode 100644 tests/tests_e3/spdx_test.py diff --git a/README.md b/README.md index 3c9988d1..43bf8726 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ and sharing the same namespace: `e3`. - *os.timezone*: platform independent interface to get the machine timezone - *platform*: generic interface for providing platform information - *platform_db*: knowledge base for computing platform information +- *spdx*: simple interface for generating SPDX files - *store*: interface to download and store resources in a store - *sys*: `e3` information, sanity check, ... - *text*: text formatting and transformation diff --git a/src/e3/spdx.py b/src/e3/spdx.py new file mode 100644 index 00000000..a0fb70c4 --- /dev/null +++ b/src/e3/spdx.py @@ -0,0 +1,519 @@ +"""Generate a SPDX file. + +This is following the specification from https://spdx.github.io/spdx-spec/v2.3/ +a simple example can be found at ./tests/tests_e3/spdx_test.py +""" +from __future__ import annotations + +from enum import Enum, auto +from abc import ABCMeta, abstractmethod +from dataclasses import dataclass, astuple, field +from datetime import datetime, timezone +from uuid import uuid4 + +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from typing import Literal, Union + +NOASSERTION: Literal["NOASSERTION"] = "NOASSERTION" +NONE_VALUE: Literal["NONE"] = "NONE" + +if TYPE_CHECKING: + MAYBE_STR = Union[str, Literal["NOASSERTION"], Literal["NONE"]] + + +class SPDXEntry(metaclass=ABCMeta): + """Describe a SPDX Entry.""" + + @property + def entry_key(self) -> str: + return self.__class__.__name__ + + @abstractmethod + def __str__(self) -> str: + pass + + def __format__(self, format_spec: str) -> str: + return self.__str__() + + def to_tagvalue(self) -> str: + return f"{self.entry_key}: {self}" + + +class SPDXEntryStr(SPDXEntry): + """Describe a SPDX Entry accepting a string.""" + + def __init__(self, value: str) -> None: + self.value = value + + def __str__(self) -> str: + return self.value + + +class SPDXEntryMaybeStr(SPDXEntry): + """Describe a SPDX Entry accepting a string, NOASSERTION, or NONE.""" + + def __init__(self, value: MAYBE_STR) -> None: + self.value = value + + def __str__(self) -> str: + return self.value + + +class SPDXEntryBool(SPDXEntry): + """Describe a SPDX Entry accepting a boolean.""" + + def __init__(self, value: bool) -> None: + self.value: bool = value + + def __str__(self) -> str: + return "true" if self.value else "false" + + +class SPDXSection: + """Describe a SPDX section.""" + + def to_tagvalue(self) -> list[str]: + """Generate a chunk of a SPDX tag:value document.""" + output = [] + for section_field in astuple(self): + if isinstance(section_field, list): + for extra_field in section_field: + output.append(extra_field.to_tagvalue()) + else: + output.append(section_field.to_tagvalue()) + + return output + + +class SPDXVersion(SPDXEntryStr): + """Provide the SPDX version used to generate the document. + + See 6.1 SPDX version field + """ + + pass + + +class DataLicense(SPDXEntryStr): + """License of the SPDX Metadata. + + See 6.2 Data license field + """ + + pass + + +class SPDXID(SPDXEntryStr): + """Identify a SPDX Document, or Package. + + See 6.3 SPDX identifier field and 7.2 Package SPDX identifier field + The value is a unique string containing letters, numbers, ., and/or -. + """ + + def __str__(self) -> str: + return f"SPDXRef-{self.value}" + + +class DocumentName(SPDXEntryStr): + """Identify name of this document. + + See 6.4 Document name field + """ + + pass + + +class DocumentNamespace(SPDXEntryStr): + """Provide a unique URI for this document. + + See 6.5 SPDX document namespace field + """ + + pass + + +class LicenseListVersion(SPDXEntryStr): + """Provide the version of the SPDX License List used. + + See 6.7 License list version field + """ + + pass + + +class Creator(SPDXEntryStr): + """Identify who (or what, in the case of a tool) created the SPDX document. + + See 6.8 Creator field + """ + + pass + + +class Created(SPDXEntryStr): + """Identify when the SPDX document was originally created. + + See 6.9 Created field + """ + + pass + + +class Organization(Creator): + def __str__(self) -> str: + return f"Organization: {self.value}" + + +class Person(Creator): + def __str__(self) -> str: + return f"Person: {self.value}" + + +class Tool(Creator): + def __str__(self) -> str: + return f"Tool: {self.value}" + + +class PackageName(SPDXEntryStr): + """Identify the full name of the package. + + See 7.1 Package name field + """ + + pass + + +class PackageVersion(SPDXEntryStr): + """Identify the version of the package. + + See 7.3 Package version field + """ + + pass + + +class PackageFileName(SPDXEntryStr): + """Provide the actual file name of the package. + + See 7.4 Package file name field + """ + + pass + + +class PackageDownloadLocation(SPDXEntryMaybeStr): + """Identifies the download location of the package. + + See 7.7 Package download location field + """ + + pass + + +class FilesAnalyzed(SPDXEntryBool): + """Indicates whether the file content of this package have been analyzed. + + See 7.8 Files analyzed field + """ + + pass + + +class ChecksumAlgorithm(Enum): + MD5 = auto() + SHA1 = auto() + SHA256 = auto() + + +class PackageChecksum(SPDXEntry): + """Provide a mechanism that permits unique identification of the package. + + See 7.10 Package checksum field + """ + + def __init__(self, algorithm: ChecksumAlgorithm, value: str) -> None: + self.algorithm = algorithm + self.value = value + + def __str__(self) -> str: + return "{0}: {1}".format(self.algorithm.name, self.value) + + +class PackageLicenseConcluded(SPDXEntryMaybeStr): + """Contain the license concluded as governing the package. + + See 7.13 Concluded license field + """ + + pass + + +class PackageCopyrightText(SPDXEntryMaybeStr): + """Identify the copyright holders of the package. + + See 7.17 Copyright text field + """ + + def to_tagvalue(self) -> str: + """Return the content that can span to multiple lines. + + In tag:value format multiple lines are delimited by .... + """ + return f"{self.entry_key}: {self}" + + +class RelationshipType(Enum): + # Is to be used when SPDXRef-DOCUMENT describes SPDXRef-A + DESCRIBES = auto() + # Is to be used when SPDXRef-A is described by SPDXREF-Document + DESCRIBED_BY = auto() + # Is to be used when SPDXRef-A contains SPDXRef-B + CONTAINS = auto() + # Is to be used when SPDXRef-A is contained by SPDXRef-B + CONTAINED_BY = auto() + # Is to be used when SPDXRef-A depends on SPDXRef-B + DEPENDS_ON = auto() + # Is to be used when SPDXRef-A is dependency of SPDXRef-B + DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is a manifest file that lists + # a set of dependencies for SPDXRef-B + DEPENDENCY_MANIFEST_OF = auto() + # Is to be used when SPDXRef-A is a build dependency of SPDXRef-B + BUILD_DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is a development dependency of SPDXRef-B + DEV_DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is an optional dependency of SPDXRef-B + OPTIONAL_DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is a to be provided dependency of SPDXRef-B + PROVIDED_DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is a test dependency of SPDXRef-B + TEST_DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is a dependency required for the + # execution of SPDXRef-B + RUNTIME_DEPENDENCY_OF = auto() + # Is to be used when SPDXRef-A is an example of SPDXRef-B + EXAMPLE_OF = auto() + # Is to be used when SPDXRef-A generates SPDXRef-B + GENERATES = auto() + # Is to be used when SPDXRef-A was generated from SPDXRef-B + GENERATED_FROM = auto() + # Is to be used when SPDXRef-A is an ancestor + # (same lineage but pre-dates) SPDXRef-B + ANCESTOR_OF = auto() + # Is to be used when SPDXRef-A is a descendant of + # (same lineage but postdates) SPDXRef-B + DESCENDANT_OF = auto() + # Is to be used when SPDXRef-A is a variant of + # (same lineage but not clear which came first) SPDXRef-B + VARIANT_OF = auto() + # Is to be used when distributing SPDXRef-A requires that SPDXRef-B + # also be distributed + DISTRIBUTION_ARTIFACT = auto() + # Is to be used when SPDXRef-A is a patch file for + # (to be applied to) SPDXRef-B + PATCH_FOR = auto() + # Is to be used when SPDXRef-A is a patch file that has been applied + # to SPDXRef-B + PATCH_APPLIED = auto() + # Is to be used when SPDXRef-A is an exact copy of SPDXRef-B + COPY_OF = auto() + # Is to be used when SPDXRef-A is a file that was added to SPDXRef-B + FILE_ADDED = auto() + # Is to be used when SPDXRef-A is a file that was deleted from SPDXRef-B + FILE_DELETED = auto() + # Is to be used when SPDXRef-A is a file that was modified from SPDXRef-B + FILE_MODIFIED = auto() + # Is to be used when SPDXRef-A is expanded from the archive SPDXRef-B + EXPANDED_FROM_ARCHIVE = auto() + # Is to be used when SPDXRef-A dynamically links to SPDXRef-B + DYNAMIC_LINK = auto() + # Is to be used when SPDXRef-A statically links to SPDXRef-B + STATIC_LINK = auto() + # Is to be used when SPDXRef-A is a data file used in SPDXRef-B + DATA_FILE_OF = auto() + # Is to be used when SPDXRef-A is a test case used in testing SPDXRef-B + TEST_CASE_OF = auto() + # Is to be used when SPDXRef-A is used to build SPDXRef-B + BUILD_TOOL_OF = auto() + # Is to be used when SPDXRef-A is used as a development tool for SPDXRef-B + DEV_TOOL_OF = auto() + # Is to be used when SPDXRef-A is used for testing SPDXRef-B + TEST_OF = auto() + # Is to be used when SPDXRef-A is used as a test tool for SPDXRef-B + TEST_TOOL_OF = auto() + # Is to be used when SPDXRef-A provides documentation of SPDXRef-B + DOCUMENTATION_OF = auto() + # Is to be used when SPDXRef-A is an optional component of SPDXRef-B + OPTIONAL_COMPONENT_OF = auto() + # Is to be used when SPDXRef-A is a metafile of SPDXRef-B + METAFILE_OF = auto() + # Is to be used when SPDXRef-A is used as a package as part of SPDXRef-B + PACKAGE_OF = auto() + # Is to be used when (current) SPDXRef-DOCUMENT amends the SPDX + # information in SPDXRef-B + AMENDS = auto() + # Is to be used when SPDXRef-A is a prerequisite for SPDXRef-B + PREREQUISITE_FOR = auto() + # Is to be used when SPDXRef-A has as a prerequisite SPDXRef-B + HAS_PREREQUISITE = auto() + # Is to be used when SPDXRef-A describes, illustrates, or specifies + # a requirement statement for SPDXRef-B + REQUIREMENT_DESCRIPTION_FOR = auto() + # Is to be used when SPDXRef-A describes, illustrates, or defines a + # design specification for SPDXRef-B + SPECIFICATION_FOR = auto() + # Is to be used for a relationship which has not been defined + # in the formal SPDX specification + OTHER = auto() + + +class Relationship(SPDXEntry): + """Provides information about the relationship between two SPDX elements. + + See 11.1 Relationship field + """ + + def __init__( + self, left: str, right: str, relationship_type: RelationshipType + ) -> None: + self.left = SPDXID(left) + self.right = SPDXID(right) + self.relationship_type = relationship_type + + def __str__(self) -> str: + return f"{self.left} {self.relationship_type.name} {self.right}" + + +@dataclass +class Package(SPDXSection): + """Describe a package.""" + + name: PackageName + spdx_id: SPDXID + version: PackageVersion + file_name: PackageFileName + checksum: list[PackageChecksum] + supplier: Creator + originator: Creator + copyright_text: PackageCopyrightText + files_analyzed: FilesAnalyzed + license_concluded: PackageLicenseConcluded + download_location: PackageDownloadLocation + + +@dataclass +class DocumentInformation(SPDXSection): + """Describe the SPDX Document.""" + + document_name: DocumentName + document_namespace: DocumentNamespace = field(init=False) + version: SPDXVersion = SPDXVersion("SPDX-1.2") + data_license: DataLicense = DataLicense("CC0-1.0") + license_list_version: LicenseListVersion = LicenseListVersion("3.19") + spdx_id: SPDXID = SPDXID("DOCUMENT") + + def __post_init__(self) -> None: + self.document_namespace = DocumentNamespace(f"{self.document_name}-{uuid4()}") + + +@dataclass +class CreationInformation(SPDXSection): + """Document where and by who the SPDX document has been created.""" + + creators: list[Creator] + created_now: Created = field(init=False) + + def __post_init__(self) -> None: + self.created_now = Created( + datetime.now(tz=timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + +class Document: + """Describe the SPDX Document.""" + + def __init__( + self, + document_name: str, + creators: list[Creator], + ) -> None: + """Initialize the SPDX Document. + + :param doc_info: A DocumentInformation instance + :param creation_info: A CreationInformation instance + """ + self.doc_info = DocumentInformation(document_name=DocumentName(document_name)) + self.creation_info = CreationInformation(creators=creators) + self.packages: list[Package] = [] + self.relationships: list[Relationship] = [] + + def add_package( + self, + name: str, + version: str, + spdx_id: str, + file_name: str, + checksum: list[PackageChecksum], + license_concluded: str, + supplier: Creator, + originator: Creator, + download_location: str, + files_analyzed: bool, + copyright_text: str, + relationship: Relationship, + ) -> None: + """Add a new Package and describe its relationship to other elements. + + :param package: the new package to add to the document + :param relationship: a Relationship instance describing how the new + package is linked to the other elements + """ + self.packages.append( + Package( + name=PackageName(name), + version=PackageVersion(version), + spdx_id=SPDXID(spdx_id), + file_name=PackageFileName(file_name), + checksum=checksum, + license_concluded=PackageLicenseConcluded(license_concluded), + supplier=supplier, + originator=originator, + download_location=PackageDownloadLocation(download_location), + files_analyzed=FilesAnalyzed(files_analyzed), + copyright_text=PackageCopyrightText(copyright_text), + ) + ) + self.relationships.append(relationship) + + def to_tagvalue(self) -> list[str]: + """Generate a list of tag:value lines describing the SPDX document.""" + output: list[str] = [] + is_first_section = True + + def add_section(section: str) -> None: + nonlocal is_first_section + nonlocal output + if not is_first_section: + output += ["", ""] + is_first_section = False + output += [f"# {section}", ""] + + add_section("Document Information") + output += self.doc_info.to_tagvalue() + + add_section("Creation Info") + output += self.creation_info.to_tagvalue() + + add_section("Relationships") + for rel in self.relationships: + output.append(rel.to_tagvalue()) + for pkg in self.packages: + add_section("Package") + output += pkg.to_tagvalue() + return output diff --git a/tests/tests_e3/spdx_test.py b/tests/tests_e3/spdx_test.py new file mode 100644 index 00000000..8ebe04f2 --- /dev/null +++ b/tests/tests_e3/spdx_test.py @@ -0,0 +1,103 @@ +from e3.spdx import ( + Document, + Organization, + Tool, + Person, + PackageChecksum, + ChecksumAlgorithm, + NOASSERTION, + Relationship, + RelationshipType, +) + + +def test_spdx(): + """Test a SPDX document creation.""" + doc = Document( + document_name="my-spdx-test", + creators=[ + Organization("AdaCore"), + Tool("e3-core"), + Person("e3-core maintainer"), + ], + ) + + doc.add_package( + name="my-spdx-test-main-pkg", + version="2.2.2", + spdx_id="main-pkg", + file_name="main-pkg.zip", + checksum=[ + PackageChecksum( + ChecksumAlgorithm.SHA1, "6476df3aac780622368173fe6e768a2edc3932c8" + ), + PackageChecksum( + ChecksumAlgorithm.SHA256, + "91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada", + ), + ], + license_concluded="GPL-3.0-or-later", + supplier=Organization("AdaCore"), + originator=Organization("AdaCore"), + download_location=NOASSERTION, + files_analyzed=False, + copyright_text="2023 AdaCore", + relationship=Relationship( + left="DOCUMENT", + right="main-pkg", + relationship_type=RelationshipType.DESCRIBES, + ), + ) + + tagvalue_content = doc.to_tagvalue() + + # Change fields that are not stable: DocumentNamespace containing an UUID + # and Created timestamp + for idx, field in enumerate(tagvalue_content): + if field.startswith("DocumentNamespace: my-spdx-test-"): + tagvalue_content[ + idx + ] = "DocumentNamespace: my-spdx-test-c5c1e261-fb57-474a-b3c3-dc2adf3a4e06" + if field.startswith("Created: "): + tagvalue_content[idx] = "Created: 2023-02-10T14:54:01Z" + + assert tagvalue_content == [ + "# Document Information", + "", + "DocumentName: my-spdx-test", + "DocumentNamespace: my-spdx-test-c5c1e261-fb57-474a-b3c3-dc2adf3a4e06", + "SPDXVersion: SPDX-1.2", + "DataLicense: CC0-1.0", + "LicenseListVersion: 3.19", + "SPDXID: SPDXRef-DOCUMENT", + "", + "", + "# Creation Info", + "", + "Organization: Organization: AdaCore", + "Tool: Tool: e3-core", + "Person: Person: e3-core maintainer", + "Created: 2023-02-10T14:54:01Z", + "", + "", + "# Relationships", + "", + "Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-main-pkg", + "", + "", + "# Package", + "", + "PackageName: my-spdx-test-main-pkg", + "SPDXID: SPDXRef-main-pkg", + "PackageVersion: 2.2.2", + "PackageFileName: main-pkg.zip", + "PackageChecksum: SHA1: 6476df3aac780622368173fe6e768a2edc3932c8", + "PackageChecksum: SHA256: " + "91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada", + "Organization: Organization: AdaCore", + "Organization: Organization: AdaCore", + "PackageCopyrightText: 2023 AdaCore", + "FilesAnalyzed: false", + "PackageLicenseConcluded: GPL-3.0-or-later", + "PackageDownloadLocation: NOASSERTION", + ]