Include static libraries information in Android SBOM.
Bug: 280852724
Test: CIs
Test: lunch aosp_cf_x86_64_phone-userdebug && m sbom
Change-Id: Ie2365d79ba24910b7ace132b578589be10a17d78
diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py
index 2415f7e..b19be87 100755
--- a/tools/sbom/generate-sbom.py
+++ b/tools/sbom/generate-sbom.py
@@ -332,14 +332,6 @@
return external_doc_ref, packages, relationships
-def generate_package_verification_code(files):
- checksums = [file.checksum for file in files]
- checksums.sort()
- h = hashlib.sha1()
- h.update(''.join(checksums).encode(encoding='utf-8'))
- return h.hexdigest()
-
-
def save_report(report_file_path, report):
with open(report_file_path, 'w', encoding='utf-8') as report_file:
for type, issues in report.items():
@@ -487,20 +479,32 @@
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
build_output_path = installed_file_metadata['build_output_path']
+ is_static_lib = installed_file_metadata['is_static_lib']
if not installed_file_has_metadata(installed_file_metadata, report):
continue
- if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
+ if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
+ # Ignore non-existing static library files for now since they are not shipped on devices.
report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
continue
file_id = new_file_id(installed_file)
- doc.files.append(
- sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))
- if not args.unbundled_apex:
- product_package.file_ids.append(file_id)
- elif len(doc.files) > 1:
- doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
+ # TODO(b/285453664): Soong should report the information of statically linked libraries to Make.
+ # This happens when a different sanitized version of static libraries is used in linking.
+ # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be
+ # located correctly because Soong doesn't report the information to Make.
+ sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string
+ if os.path.islink(build_output_path) or os.path.isfile(build_output_path):
+ sha1 = checksum(build_output_path)
+ doc.files.append(sbom_data.File(id=file_id,
+ name=installed_file,
+ checksum=sha1))
+
+ if not is_static_lib:
+ if not args.unbundled_apex:
+ product_package.file_ids.append(file_id)
+ elif len(doc.files) > 1:
+ doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
metadata_file_path = get_metadata_file_path(installed_file_metadata)
@@ -544,13 +548,21 @@
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
- if not args.unbundled_apex:
- product_package.verification_code = generate_package_verification_code(doc.files)
+ # Process static libraries and whole static libraries the installed file links to
+ static_libs = installed_file_metadata['static_libraries']
+ whole_static_libs = installed_file_metadata['whole_static_libraries']
+ all_static_libs = (static_libs + ' ' + whole_static_libs).strip()
+ if all_static_libs:
+ for lib in all_static_libs.split(' '):
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.STATIC_LINK,
+ id2=new_file_id(lib + '.a')))
if args.unbundled_apex:
doc.describes = doc.files[0].id
# Save SBOM records to output file
+ doc.generate_packages_verification_code()
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
prefix = args.output_file
if prefix.endswith('.spdx'):
diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py
index 14c4eb2..ea38e36 100644
--- a/tools/sbom/sbom_data.py
+++ b/tools/sbom/sbom_data.py
@@ -25,6 +25,7 @@
from dataclasses import dataclass, field
from typing import List
+import hashlib
SPDXID_DOC = 'SPDXRef-DOCUMENT'
SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
@@ -81,6 +82,7 @@
VARIANT_OF = 'VARIANT_OF'
GENERATED_FROM = 'GENERATED_FROM'
CONTAINS = 'CONTAINS'
+ STATIC_LINK = 'STATIC_LINK'
@dataclass
@@ -122,3 +124,17 @@
if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
for r in self.relationships):
self.relationships.append(rel)
+
+ def generate_packages_verification_code(self):
+ for package in self.packages:
+ if not package.file_ids:
+ continue
+
+ checksums = []
+ for file in self.files:
+ if file.id in package.file_ids:
+ checksums.append(file.checksum)
+ checksums.sort()
+ h = hashlib.sha1()
+ h.update(''.join(checksums).encode(encoding='utf-8'))
+ package.verification_code = h.hexdigest()
diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py
index 85dee9d..1cb864d 100644
--- a/tools/sbom/sbom_writers.py
+++ b/tools/sbom/sbom_writers.py
@@ -85,7 +85,7 @@
return headers
@staticmethod
- def marshal_package(package):
+ def marshal_package(sbom_doc, package, fragment):
download_location = sbom_data.VALUE_NOASSERTION
if package.download_location:
download_location = package.download_location
@@ -107,50 +107,32 @@
f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')
tagvalues.append('')
+
+ if package.id == sbom_doc.describes and not fragment:
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
+ tagvalues.append('')
+
+ for file in sbom_doc.files:
+ if file.id in package.file_ids:
+ tagvalues += TagValueWriter.marshal_file(file)
+
return tagvalues
@staticmethod
- def marshal_described_element(sbom_doc, fragment):
- if not sbom_doc.describes:
- return None
-
- product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
- if product_package:
- tagvalues = TagValueWriter.marshal_package(product_package[0])
- if not fragment:
- tagvalues.append(
- f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
-
- tagvalues.append('')
- return tagvalues
-
- file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
- if file:
- tagvalues = TagValueWriter.marshal_file(file[0])
- if not fragment:
- tagvalues.append(
- f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
-
- return tagvalues
-
- return None
-
- @staticmethod
- def marshal_packages(sbom_doc):
+ def marshal_packages(sbom_doc, fragment):
tagvalues = []
marshaled_relationships = []
i = 0
packages = sbom_doc.packages
while i < len(packages):
- if packages[i].id == sbom_doc.describes:
- i += 1
- continue
-
- if i + 1 < len(packages) \
- and packages[i].id.startswith('SPDXRef-SOURCE-') \
- and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
- tagvalues += TagValueWriter.marshal_package(packages[i])
- tagvalues += TagValueWriter.marshal_package(packages[i + 1])
+ if (i + 1 < len(packages)
+ and packages[i].id.startswith('SPDXRef-SOURCE-')
+ and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')):
+ # Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other
+ # in SBOMs in tagvalue format.
+ tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
+ tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment)
rel = next((r for r in sbom_doc.relationships if
r.id1 == packages[i].id and
r.id2 == packages[i + 1].id and
@@ -162,7 +144,7 @@
i += 2
else:
- tagvalues += TagValueWriter.marshal_package(packages[i])
+ tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
i += 1
return tagvalues, marshaled_relationships
@@ -179,12 +161,20 @@
return tagvalues
@staticmethod
- def marshal_files(sbom_doc):
+ def marshal_files(sbom_doc, fragment):
tagvalues = []
+ files_in_packages = []
+ for package in sbom_doc.packages:
+ files_in_packages += package.file_ids
for file in sbom_doc.files:
- if file.id == sbom_doc.describes:
+ if file.id in files_in_packages:
continue
tagvalues += TagValueWriter.marshal_file(file)
+ if file.id == sbom_doc.describes and not fragment:
+ # Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable.
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
+ tagvalues.append('')
return tagvalues
@staticmethod
@@ -208,11 +198,8 @@
content = []
if not fragment:
content += TagValueWriter.marshal_doc_headers(sbom_doc)
- described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)
- if described_element:
- content += described_element
- content += TagValueWriter.marshal_files(sbom_doc)
- tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
+ content += TagValueWriter.marshal_files(sbom_doc, fragment)
+ tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)
content += tagvalues
content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
file.write('\n'.join(content))
diff --git a/tools/sbom/sbom_writers_test.py b/tools/sbom/sbom_writers_test.py
index 361dae6..cf85e01 100644
--- a/tools/sbom/sbom_writers_test.py
+++ b/tools/sbom/sbom_writers_test.py
@@ -31,6 +31,7 @@
SPDXID_FILE1 = 'SPDXRef-file1'
SPDXID_FILE2 = 'SPDXRef-file2'
SPDXID_FILE3 = 'SPDXRef-file3'
+SPDXID_FILE4 = 'SPDXRef-file4'
class SBOMWritersTest(unittest.TestCase):
@@ -101,6 +102,8 @@
sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
+ self.sbom_doc.files.append(
+ sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444'))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
@@ -112,6 +115,10 @@
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_SOURCE_PACKAGE1
))
+ self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
+ relationship=sbom_data.RelationshipType.STATIC_LINK,
+ id2=SPDXID_FILE4
+ ))
# SBOM fragment of a APK
self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
@@ -139,6 +146,14 @@
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
+ def test_tagvalue_writer_doc_describes_file(self):
+ with io.StringIO() as output:
+ self.sbom_doc.describes = SPDXID_FILE4
+ sbom_writers.TagValueWriter.write(self.sbom_doc, output)
+ expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_doc_describes_file.spdx').read_text()
+ self.maxDiff = None
+ self.assertEqual(expected_output, output.getvalue())
+
def test_tagvalue_writer_unbundled(self):
with io.StringIO() as output:
sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True)
diff --git a/tools/sbom/testdata/expected_json_sbom.spdx.json b/tools/sbom/testdata/expected_json_sbom.spdx.json
index 32715a5..53936c5 100644
--- a/tools/sbom/testdata/expected_json_sbom.spdx.json
+++ b/tools/sbom/testdata/expected_json_sbom.spdx.json
@@ -110,6 +110,16 @@
"checksumValue": "33333"
}
]
+ },
+ {
+ "fileName": "file4.a",
+ "SPDXID": "SPDXRef-file4",
+ "checksums": [
+ {
+ "algorithm": "SHA1",
+ "checksumValue": "44444"
+ }
+ ]
}
],
"relationships": [
@@ -129,6 +139,11 @@
"relationshipType": "GENERATED_FROM"
},
{
+ "spdxElementId": "SPDXRef-file1",
+ "relatedSpdxElement": "SPDXRef-file4",
+ "relationshipType": "STATIC_LINK"
+ },
+ {
"spdxElementId": "SPDXRef-SOURCE-package1",
"relatedSpdxElement": "SPDXRef-UPSTREAM-package1",
"relationshipType": "VARIANT_OF"
diff --git a/tools/sbom/testdata/expected_tagvalue_sbom.spdx b/tools/sbom/testdata/expected_tagvalue_sbom.spdx
index ee39e82..e6fd17e 100644
--- a/tools/sbom/testdata/expected_tagvalue_sbom.spdx
+++ b/tools/sbom/testdata/expected_tagvalue_sbom.spdx
@@ -7,6 +7,10 @@
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
+FileName: file4.a
+SPDXID: SPDXRef-file4
+FileChecksum: SHA1: 44444
+
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
@@ -63,3 +67,4 @@
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
+Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4
diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx
new file mode 100644
index 0000000..428d7e3
--- /dev/null
+++ b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx
@@ -0,0 +1,70 @@
+SPDXVersion: SPDX-2.3
+DataLicense: CC0-1.0
+SPDXID: SPDXRef-DOCUMENT
+DocumentName: test doc
+DocumentNamespace: http://www.google.com/sbom/spdx/android
+Creator: Organization: Google
+Created: 2023-03-31T22:17:58Z
+ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
+
+FileName: file4.a
+SPDXID: SPDXRef-file4
+FileChecksum: SHA1: 44444
+
+Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-file4
+
+PackageName: PRODUCT
+SPDXID: SPDXRef-PRODUCT
+PackageDownloadLocation: NONE
+FilesAnalyzed: true
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+PackageVerificationCode: 123456
+
+FileName: /bin/file1
+SPDXID: SPDXRef-file1
+FileChecksum: SHA1: 11111
+
+FileName: /bin/file2
+SPDXID: SPDXRef-file2
+FileChecksum: SHA1: 22222
+
+FileName: /bin/file3
+SPDXID: SPDXRef-file3
+FileChecksum: SHA1: 33333
+
+PackageName: PLATFORM
+SPDXID: SPDXRef-PLATFORM
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+PackageName: Prebuilt package1
+SPDXID: SPDXRef-PREBUILT-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+PackageName: Source package1
+SPDXID: SPDXRef-SOURCE-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4
+
+PackageName: Upstream package1
+SPDXID: SPDXRef-UPSTREAM-package1
+PackageDownloadLocation: NOASSERTION
+FilesAnalyzed: false
+PackageVersion: 1.1
+PackageSupplier: Organization: upstream
+
+Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
+
+Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
+Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
+Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
+Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4