Merge "Create separate python libraries for the following logic and refactor SBOM generation script accordingly."
diff --git a/tools/Android.bp b/tools/Android.bp
index e325f6b..bea0602 100644
--- a/tools/Android.bp
+++ b/tools/Android.bp
@@ -71,22 +71,6 @@
}
python_binary_host {
- name: "generate-sbom",
- srcs: [
- "generate-sbom.py",
- ],
- version: {
- py3: {
- embedded_launcher: true,
- },
- },
- libs: [
- "metadata_file_proto_py",
- "libprotobuf-python",
- ],
-}
-
-python_binary_host {
name: "list_files",
main: "list_files.py",
srcs: [
diff --git a/tools/generate-sbom.py b/tools/generate-sbom.py
deleted file mode 100755
index 9583395..0000000
--- a/tools/generate-sbom.py
+++ /dev/null
@@ -1,690 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2023 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Generate the SBOM of the current target product in SPDX format.
-Usage example:
- generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \
- --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \
- --product_out_dir=out/target/product/vsoc_x86_64 \
- --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
- --product_mfr=Google
-"""
-
-import argparse
-import csv
-import datetime
-import google.protobuf.text_format as text_format
-import hashlib
-import json
-import os
-import metadata_file_pb2
-
-# Common
-SPDXID = 'SPDXID'
-SPDX_VERSION = 'SPDXVersion'
-DATA_LICENSE = 'DataLicense'
-DOCUMENT_NAME = 'DocumentName'
-DOCUMENT_NAMESPACE = 'DocumentNamespace'
-CREATED = 'Created'
-CREATOR = 'Creator'
-EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef'
-
-# Package
-PACKAGE_NAME = 'PackageName'
-PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation'
-PACKAGE_VERSION = 'PackageVersion'
-PACKAGE_SUPPLIER = 'PackageSupplier'
-FILES_ANALYZED = 'FilesAnalyzed'
-PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode'
-PACKAGE_EXTERNAL_REF = 'ExternalRef'
-# Package license
-PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded'
-PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles'
-PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared'
-PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments'
-
-# File
-FILE_NAME = 'FileName'
-FILE_CHECKSUM = 'FileChecksum'
-# File license
-FILE_LICENSE_CONCLUDED = 'LicenseConcluded'
-FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile'
-FILE_LICENSE_COMMENTS = 'LicenseComments'
-FILE_COPYRIGHT_TEXT = 'FileCopyrightText'
-FILE_NOTICE = 'FileNotice'
-FILE_ATTRIBUTION_TEXT = 'FileAttributionText'
-
-# Relationship
-RELATIONSHIP = 'Relationship'
-REL_DESCRIBES = 'DESCRIBES'
-REL_VARIANT_OF = 'VARIANT_OF'
-REL_GENERATED_FROM = 'GENERATED_FROM'
-
-# Package type
-PKG_SOURCE = 'SOURCE'
-PKG_UPSTREAM = 'UPSTREAM'
-PKG_PREBUILT = 'PREBUILT'
-
-# Security tag
-NVD_CPE23 = 'NVD-CPE2.3:'
-
-# Report
-ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:'
-ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:'
-ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:'
-ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:'
-ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-exist installed files:'
-INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:'
-
-
-def get_args():
- parser = argparse.ArgumentParser()
- parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
- parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
- parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.')
- parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.')
- parser.add_argument('--build_version', required=True, help='The build version.')
- parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
- parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
- parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module')
-
- return parser.parse_args()
-
-
-def log(*info):
- if args.verbose:
- for i in info:
- print(i)
-
-
-def new_doc_header(doc_id):
- return {
- SPDX_VERSION: 'SPDX-2.3',
- DATA_LICENSE: 'CC0-1.0',
- SPDXID: doc_id,
- DOCUMENT_NAME: args.build_version,
- DOCUMENT_NAMESPACE: f'https://www.google.com/sbom/spdx/android/{args.build_version}',
- CREATOR: 'Organization: Google, LLC',
- CREATED: '<timestamp>',
- EXTERNAL_DOCUMENT_REF: [],
- }
-
-
-def new_package_record(id, name, version, supplier, download_location=None, files_analyzed='false', external_refs=[]):
- package = {
- PACKAGE_NAME: name,
- SPDXID: id,
- PACKAGE_DOWNLOAD_LOCATION: download_location if download_location else 'NONE',
- FILES_ANALYZED: files_analyzed,
- }
- if version:
- package[PACKAGE_VERSION] = version
- if supplier:
- package[PACKAGE_SUPPLIER] = f'Organization: {supplier}'
- if external_refs:
- package[PACKAGE_EXTERNAL_REF] = external_refs
-
- return package
-
-
-def new_file_record(id, name, checksum):
- return {
- FILE_NAME: name,
- SPDXID: id,
- FILE_CHECKSUM: checksum
- }
-
-
-def encode_for_spdxid(s):
- """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-"""
- result = ''
- for c in s:
- if c.isalnum() or c in '.-':
- result += c
- elif c in '_@/':
- result += '-'
- else:
- result += '0x' + c.encode('utf-8').hex()
-
- return result.lstrip('-')
-
-
-def new_package_id(package_name, type):
- return f'SPDXRef-{type}-{encode_for_spdxid(package_name)}'
-
-
-def new_external_doc_ref(package_name, sbom_url, sbom_checksum):
- doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(package_name)}'
- return f'{EXTERNAL_DOCUMENT_REF}: {doc_ref_id} {sbom_url} {sbom_checksum}', doc_ref_id
-
-
-def new_file_id(file_path):
- return f'SPDXRef-{encode_for_spdxid(file_path)}'
-
-
-def new_relationship_record(id1, relationship, id2):
- return f'{RELATIONSHIP}: {id1} {relationship} {id2}'
-
-
-def checksum(file_path):
- file_path = args.product_out_dir + '/' + file_path
- h = hashlib.sha1()
- if os.path.islink(file_path):
- h.update(os.readlink(file_path).encode('utf-8'))
- else:
- with open(file_path, 'rb') as f:
- h.update(f.read())
- return f'SHA1: {h.hexdigest()}'
-
-
-def is_soong_prebuilt_module(file_metadata):
- return file_metadata['soong_module_type'] and file_metadata['soong_module_type'] in [
- 'android_app_import', 'android_library_import', 'cc_prebuilt_binary', 'cc_prebuilt_library',
- 'cc_prebuilt_library_headers', 'cc_prebuilt_library_shared', 'cc_prebuilt_library_static', 'cc_prebuilt_object',
- 'dex_import', 'java_import', 'java_sdk_library_import', 'java_system_modules_import',
- 'libclang_rt_prebuilt_library_static', 'libclang_rt_prebuilt_library_shared', 'llvm_prebuilt_library_static',
- 'ndk_prebuilt_object', 'ndk_prebuilt_shared_stl', 'nkd_prebuilt_static_stl', 'prebuilt_apex',
- 'prebuilt_bootclasspath_fragment', 'prebuilt_dsp', 'prebuilt_firmware', 'prebuilt_kernel_modules',
- 'prebuilt_rfsa', 'prebuilt_root', 'rust_prebuilt_dylib', 'rust_prebuilt_library', 'rust_prebuilt_rlib',
- 'vndk_prebuilt_shared',
-
- # 'android_test_import',
- # 'cc_prebuilt_test_library_shared',
- # 'java_import_host',
- # 'java_test_import',
- # 'llvm_host_prebuilt_library_shared',
- # 'prebuilt_apis',
- # 'prebuilt_build_tool',
- # 'prebuilt_defaults',
- # 'prebuilt_etc',
- # 'prebuilt_etc_host',
- # 'prebuilt_etc_xml',
- # 'prebuilt_font',
- # 'prebuilt_hidl_interfaces',
- # 'prebuilt_platform_compat_config',
- # 'prebuilt_stubs_sources',
- # 'prebuilt_usr_share',
- # 'prebuilt_usr_share_host',
- # 'soong_config_module_type_import',
- ]
-
-
-def is_source_package(file_metadata):
- module_path = file_metadata['module_path']
- return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)
-
-
-def is_prebuilt_package(file_metadata):
- module_path = file_metadata['module_path']
- if module_path:
- return (module_path.startswith('prebuilts/') or
- is_soong_prebuilt_module(file_metadata) or
- file_metadata['is_prebuilt_make_module'])
-
- kernel_module_copy_files = file_metadata['kernel_module_copy_files']
- if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
- return True
-
- return False
-
-
-def get_source_package_info(file_metadata, metadata_file_path):
- if not metadata_file_path:
- return file_metadata['module_path'], []
-
- metadata_proto = metadata_file_protos[metadata_file_path]
- external_refs = []
- for tag in metadata_proto.third_party.security.tag:
- if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
- external_refs.append(f'{PACKAGE_EXTERNAL_REF}: SECURITY cpe23Type {tag.removeprefix(NVD_CPE23)}')
- elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
- external_refs.append(f'{PACKAGE_EXTERNAL_REF}: SECURITY cpe22Type {tag.removeprefix(NVD_CPE23)}')
-
- if metadata_proto.name:
- return metadata_proto.name, external_refs
- else:
- return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name
-
-
-def get_prebuilt_package_name(file_metadata, metadata_file_path):
- name = None
- if metadata_file_path:
- metadata_proto = metadata_file_protos[metadata_file_path]
- if metadata_proto.name:
- name = metadata_proto.name
- else:
- name = metadata_file_path
- elif file_metadata['module_path']:
- name = file_metadata['module_path']
- elif file_metadata['kernel_module_copy_files']:
- src_path = file_metadata['kernel_module_copy_files'].split(':')[0]
- name = os.path.dirname(src_path)
-
- return name.removeprefix('prebuilts/').replace('/', '-')
-
-
-def get_metadata_file_path(file_metadata):
- metadata_path = ''
- if file_metadata['module_path']:
- metadata_path = file_metadata['module_path']
- elif file_metadata['kernel_module_copy_files']:
- metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0])
-
- while metadata_path and not os.path.exists(metadata_path + '/METADATA'):
- metadata_path = os.path.dirname(metadata_path)
-
- return metadata_path
-
-
-def get_package_version(metadata_file_path):
- if not metadata_file_path:
- return None
- metadata_proto = metadata_file_protos[metadata_file_path]
- return metadata_proto.third_party.version
-
-
-def get_package_homepage(metadata_file_path):
- if not metadata_file_path:
- return None
- metadata_proto = metadata_file_protos[metadata_file_path]
- if metadata_proto.third_party.homepage:
- return metadata_proto.third_party.homepage
- for url in metadata_proto.third_party.url:
- if url.type == metadata_file_pb2.URL.Type.HOMEPAGE:
- return url.value
-
- return None
-
-
-def get_package_download_location(metadata_file_path):
- if not metadata_file_path:
- return None
- metadata_proto = metadata_file_protos[metadata_file_path]
- if metadata_proto.third_party.url:
- urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type)
- if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE:
- return urls[0].value
- elif len(urls) > 1:
- return urls[1].value
-
- return None
-
-
-def get_sbom_fragments(installed_file_metadata, metadata_file_path):
- external_doc_ref = None
- packages = []
- relationships = []
-
- # Info from METADATA file
- homepage = get_package_homepage(metadata_file_path)
- version = get_package_version(metadata_file_path)
- download_location = get_package_download_location(metadata_file_path)
-
- if is_source_package(installed_file_metadata):
- # Source fork packages
- name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
- source_package_id = new_package_id(name, PKG_SOURCE)
- source_package = new_package_record(source_package_id, name, args.build_version, args.product_mfr,
- external_refs=external_refs)
-
- upstream_package_id = new_package_id(name, PKG_UPSTREAM)
- upstream_package = new_package_record(upstream_package_id, name, version, homepage, download_location)
- packages += [source_package, upstream_package]
- relationships.append(new_relationship_record(source_package_id, REL_VARIANT_OF, upstream_package_id))
- elif is_prebuilt_package(installed_file_metadata):
- # Prebuilt fork packages
- name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
- prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
- prebuilt_package = new_package_record(prebuilt_package_id, name, args.build_version, args.product_mfr)
- packages.append(prebuilt_package)
-
- if metadata_file_path:
- metadata_proto = metadata_file_protos[metadata_file_path]
- if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
- sbom_url = metadata_proto.third_party.sbom_ref.url
- sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
- upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
- if sbom_url and sbom_checksum and upstream_element_id:
- external_doc_ref, doc_ref_id = new_external_doc_ref(name, sbom_url, sbom_checksum)
- relationships.append(
- new_relationship_record(prebuilt_package_id, REL_VARIANT_OF, doc_ref_id + ':' + upstream_element_id))
-
- return external_doc_ref, packages, relationships
-
-
-def generate_package_verification_code(files):
- checksums = [file[FILE_CHECKSUM] for file in files]
- checksums.sort()
- h = hashlib.sha1()
- h.update(''.join(checksums).encode(encoding='utf-8'))
- return h.hexdigest()
-
-
-def write_record(f, record):
- if record.__class__.__name__ == 'dict':
- for k, v in record.items():
- if k == EXTERNAL_DOCUMENT_REF or k == PACKAGE_EXTERNAL_REF:
- for ref in v:
- f.write(ref + '\n')
- else:
- f.write('{}: {}\n'.format(k, v))
- elif record.__class__.__name__ == 'str':
- f.write(record + '\n')
- f.write('\n')
-
-
-def write_tagvalue_sbom(all_records):
- with open(args.output_file, 'w', encoding="utf-8") as output_file:
- for rec in all_records:
- write_record(output_file, rec)
-
-
-def write_json_sbom(all_records, product_package_id):
- doc = {}
- product_package = None
- for r in all_records:
- if r.__class__.__name__ == 'dict':
- if DOCUMENT_NAME in r: # Doc header
- doc['spdxVersion'] = r[SPDX_VERSION]
- doc['dataLicense'] = r[DATA_LICENSE]
- doc[SPDXID] = r[SPDXID]
- doc['name'] = r[DOCUMENT_NAME]
- doc['documentNamespace'] = r[DOCUMENT_NAMESPACE]
- doc['creationInfo'] = {
- 'creators': [r[CREATOR]],
- 'created': r[CREATED],
- }
- doc['externalDocumentRefs'] = []
- for ref in r[EXTERNAL_DOCUMENT_REF]:
- # ref is 'ExternalDocumentRef: <doc id> <doc url> SHA1: xxxxx'
- fields = ref.split(' ')
- doc_ref = {
- 'externalDocumentId': fields[1],
- 'spdxDocument': fields[2],
- 'checksum': {
- 'algorithm': fields[3][:-1],
- 'checksumValue': fields[4]
- }
- }
- doc['externalDocumentRefs'].append(doc_ref)
- doc['documentDescribes'] = []
- doc['packages'] = []
- doc['files'] = []
- doc['relationships'] = []
-
- elif PACKAGE_NAME in r: # packages
- package = {
- 'name': r[PACKAGE_NAME],
- SPDXID: r[SPDXID],
- 'downloadLocation': r[PACKAGE_DOWNLOAD_LOCATION],
- 'filesAnalyzed': r[FILES_ANALYZED] == "true"
- }
- if PACKAGE_VERSION in r:
- package['versionInfo'] = r[PACKAGE_VERSION]
- if PACKAGE_SUPPLIER in r:
- package['supplier'] = r[PACKAGE_SUPPLIER]
- if PACKAGE_VERIFICATION_CODE in r:
- package['packageVerificationCode'] = {
- 'packageVerificationCodeValue': r[PACKAGE_VERIFICATION_CODE]
- }
- if PACKAGE_EXTERNAL_REF in r:
- package['externalRefs'] = []
- for ref in r[PACKAGE_EXTERNAL_REF]:
- # ref is 'ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4'
- fields = ref.split(' ')
- ext_ref = {
- 'referenceCategory': fields[1],
- 'referenceType': fields[2],
- 'referenceLocator': fields[3],
- }
- package['externalRefs'].append(ext_ref)
-
- doc['packages'].append(package)
- if r[SPDXID] == product_package_id:
- product_package = package
- product_package['hasFiles'] = []
-
- elif FILE_NAME in r: # files
- file = {
- 'fileName': r[FILE_NAME],
- SPDXID: r[SPDXID]
- }
- checksum = r[FILE_CHECKSUM].split(': ')
- file['checksums'] = [{
- 'algorithm': checksum[0],
- 'checksumValue': checksum[1],
- }]
- doc['files'].append(file)
- product_package['hasFiles'].append(r[SPDXID])
-
- elif r.__class__.__name__ == 'str':
- if r.startswith(RELATIONSHIP):
- # r is 'Relationship: <spdxid> <relationship> <spdxid>'
- fields = r.split(' ')
- rel = {
- 'spdxElementId': fields[1],
- 'relatedSpdxElement': fields[3],
- 'relationshipType': fields[2],
- }
- if fields[2] == REL_DESCRIBES:
- doc['documentDescribes'].append(fields[3])
- else:
- doc['relationships'].append(rel)
-
- with open(args.output_file + '.json', 'w', encoding="utf-8") as output_file:
- output_file.write(json.dumps(doc, indent=4))
-
-
-def save_report(report):
- prefix, _ = os.path.splitext(args.output_file)
- with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file:
- for type, issues in report.items():
- report_file.write(type + '\n')
- for issue in issues:
- report_file.write('\t' + issue + '\n')
- report_file.write('\n')
-
-
-def sort_rels(rel):
- # rel = 'Relationship file_id GENERATED_FROM package_id'
- fields = rel.split(' ')
- return fields[3] + fields[1]
-
-
-# Validate the metadata generated by Make for installed files and report if there is no metadata.
-def installed_file_has_metadata(installed_file_metadata, report):
- installed_file = installed_file_metadata['installed_file']
- module_path = installed_file_metadata['module_path']
- product_copy_files = installed_file_metadata['product_copy_files']
- kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
- is_platform_generated = installed_file_metadata['is_platform_generated']
-
- if (not module_path and
- not product_copy_files and
- not kernel_module_copy_files and
- not is_platform_generated and
- not installed_file.endswith('.fsv_meta')):
- report[ISSUE_NO_METADATA].append(installed_file)
- return False
-
- return True
-
-
-def report_metadata_file(metadata_file_path, installed_file_metadata, report):
- if metadata_file_path:
- report[INFO_METADATA_FOUND_FOR_PACKAGE].append(
- 'installed_file: {}, module_path: {}, METADATA file: {}'.format(
- installed_file_metadata['installed_file'],
- installed_file_metadata['module_path'],
- metadata_file_path + '/METADATA'))
-
- package_metadata = metadata_file_pb2.Metadata()
- with open(metadata_file_path + '/METADATA', 'rt') as f:
- text_format.Parse(f.read(), package_metadata)
-
- if not metadata_file_path in metadata_file_protos:
- metadata_file_protos[metadata_file_path] = package_metadata
- if not package_metadata.name:
- report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"')
-
- if not package_metadata.third_party.version:
- report[ISSUE_METADATA_FILE_INCOMPLETE].append(
- f'{metadata_file_path}/METADATA does not has "third_party.version"')
-
- for tag in package_metadata.third_party.security.tag:
- if not tag.startswith(NVD_CPE23):
- report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append(
- f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA')
- else:
- report[ISSUE_NO_METADATA_FILE].append(
- "installed_file: {}, module_path: {}".format(
- installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
-
-
-def generate_fragment():
- with open(args.metadata, newline='') as sbom_metadata_file:
- reader = csv.DictReader(sbom_metadata_file)
- for installed_file_metadata in reader:
- installed_file = installed_file_metadata['installed_file']
- if args.output_file != args.product_out_dir + installed_file + ".spdx":
- continue
-
- module_path = installed_file_metadata['module_path']
- package_id = new_package_id(encode_for_spdxid(module_path), PKG_PREBUILT)
- package = new_package_record(package_id, module_path, args.build_version, args.product_mfr)
- file_id = new_file_id(installed_file)
- file = new_file_record(file_id, installed_file, checksum(installed_file))
- relationship = new_relationship_record(file_id, REL_GENERATED_FROM, package_id)
- records = [package, file, relationship]
- write_tagvalue_sbom(records)
- break
-
-
-def main():
- global args
- args = get_args()
- log('Args:', vars(args))
-
- if args.unbundled:
- generate_fragment()
- return
-
- global metadata_file_protos
- metadata_file_protos = {}
-
- doc_id = 'SPDXRef-DOCUMENT'
- doc_header = new_doc_header(doc_id)
-
- product_package_id = 'SPDXRef-PRODUCT'
- product_package = new_package_record(product_package_id, 'PRODUCT', args.build_version, args.product_mfr,
- files_analyzed='true')
-
- platform_package_id = 'SPDXRef-PLATFORM'
- platform_package = new_package_record(platform_package_id, 'PLATFORM', args.build_version, args.product_mfr)
-
- # Report on some issues and information
- report = {
- ISSUE_NO_METADATA: [],
- ISSUE_NO_METADATA_FILE: [],
- ISSUE_METADATA_FILE_INCOMPLETE: [],
- ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [],
- ISSUE_INSTALLED_FILE_NOT_EXIST: [],
- INFO_METADATA_FOUND_FOR_PACKAGE: [],
- }
-
- # Scan the metadata in CSV file and create the corresponding package and file records in SPDX
- product_files = []
- package_ids = []
- package_records = []
- rels_file_gen_from = []
- with open(args.metadata, newline='') as sbom_metadata_file:
- reader = csv.DictReader(sbom_metadata_file)
- for installed_file_metadata in reader:
- installed_file = installed_file_metadata['installed_file']
- module_path = installed_file_metadata['module_path']
- product_copy_files = installed_file_metadata['product_copy_files']
- kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
-
- if not installed_file_has_metadata(installed_file_metadata, report):
- continue
- file_path = args.product_out_dir + '/' + installed_file
- if not (os.path.islink(file_path) or os.path.isfile(file_path)):
- report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
- continue
-
- file_id = new_file_id(installed_file)
- product_files.append(new_file_record(file_id, installed_file, checksum(installed_file)))
-
- if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
- metadata_file_path = get_metadata_file_path(installed_file_metadata)
- report_metadata_file(metadata_file_path, installed_file_metadata, report)
-
- # File from source fork packages or prebuilt fork packages
- external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path)
- if len(pkgs) > 0:
- if external_doc_ref and external_doc_ref not in doc_header[EXTERNAL_DOCUMENT_REF]:
- doc_header[EXTERNAL_DOCUMENT_REF].append(external_doc_ref)
- for p in pkgs:
- if not p[SPDXID] in package_ids:
- package_ids.append(p[SPDXID])
- package_records.append(p)
- for rel in rels:
- if not rel in package_records:
- package_records.append(rel)
- fork_package_id = pkgs[0][SPDXID] # The first package should be the source/prebuilt fork package
- rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, fork_package_id))
- elif module_path or installed_file_metadata['is_platform_generated']:
- # File from PLATFORM package
- rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
- elif product_copy_files:
- # Format of product_copy_files: <source path>:<dest path>
- src_path = product_copy_files.split(':')[0]
- # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
- # so process them as files from PLATFORM package
- rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
- elif installed_file.endswith('.fsv_meta'):
- # See build/make/core/Makefile:2988
- rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
- elif kernel_module_copy_files.startswith('ANDROID-GEN'):
- # For the four files generated for _dlkm, _ramdisk partitions
- # See build/make/core/Makefile:323
- rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
-
- product_package[PACKAGE_VERIFICATION_CODE] = generate_package_verification_code(product_files)
-
- all_records = [
- doc_header,
- product_package,
- new_relationship_record(doc_id, REL_DESCRIBES, product_package_id),
- ]
- all_records += product_files
- all_records.append(platform_package)
- all_records += package_records
- rels_file_gen_from.sort(key=sort_rels)
- all_records += rels_file_gen_from
-
- # Save SBOM records to output file
- doc_header[CREATED] = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
- write_tagvalue_sbom(all_records)
- if args.json:
- write_json_sbom(all_records, product_package_id)
-
- save_report(report)
-
-
-if __name__ == '__main__':
- main()
diff --git a/tools/sbom/Android.bp b/tools/sbom/Android.bp
new file mode 100644
index 0000000..f6c0190
--- /dev/null
+++ b/tools/sbom/Android.bp
@@ -0,0 +1,53 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+python_binary_host {
+ name: "generate-sbom",
+ srcs: [
+ "generate-sbom.py",
+ ],
+ version: {
+ py3: {
+ embedded_launcher: true,
+ },
+ },
+ libs: [
+ "metadata_file_proto_py",
+ "libprotobuf-python",
+ "sbom_lib",
+ ],
+}
+
+python_library_host {
+ name: "sbom_lib",
+ srcs: [
+ "sbom_data.py",
+ "sbom_writers.py",
+ ],
+}
+
+python_test_host {
+ name: "sbom_writers_test",
+ main: "sbom_writers_test.py",
+ srcs: [
+ "sbom_writers_test.py",
+ ],
+ data: [
+ "testdata/*",
+ ],
+ libs: [
+ "sbom_lib",
+ ],
+ test_suites: ["general-tests"],
+}
diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py
new file mode 100755
index 0000000..0c5deb2
--- /dev/null
+++ b/tools/sbom/generate-sbom.py
@@ -0,0 +1,536 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generate the SBOM of the current target product in SPDX format.
+Usage example:
+ generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \
+ --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \
+ --product_out_dir=out/target/product/vsoc_x86_64 \
+ --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
+ --product_mfr=Google
+"""
+
+import argparse
+import csv
+import datetime
+import google.protobuf.text_format as text_format
+import hashlib
+import os
+import metadata_file_pb2
+import sbom_data
+import sbom_writers
+
+
+# Package type
+PKG_SOURCE = 'SOURCE'
+PKG_UPSTREAM = 'UPSTREAM'
+PKG_PREBUILT = 'PREBUILT'
+
+# Security tag
+NVD_CPE23 = 'NVD-CPE2.3:'
+
+# Report
+ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:'
+ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:'
+ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:'
+ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:'
+ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-exist installed files:'
+INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:'
+
+
+def get_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
+ parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
+ parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.')
+ parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.')
+ parser.add_argument('--build_version', required=True, help='The build version.')
+ parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
+ parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
+ parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module')
+
+ return parser.parse_args()
+
+
+def log(*info):
+ if args.verbose:
+ for i in info:
+ print(i)
+
+
+def encode_for_spdxid(s):
+ """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-"""
+ result = ''
+ for c in s:
+ if c.isalnum() or c in '.-':
+ result += c
+ elif c in '_@/':
+ result += '-'
+ else:
+ result += '0x' + c.encode('utf-8').hex()
+
+ return result.lstrip('-')
+
+
+def new_package_id(package_name, type):
+ return f'SPDXRef-{type}-{encode_for_spdxid(package_name)}'
+
+
+def new_file_id(file_path):
+ return f'SPDXRef-{encode_for_spdxid(file_path)}'
+
+
+def checksum(file_path):
+ file_path = args.product_out_dir + '/' + file_path
+ h = hashlib.sha1()
+ if os.path.islink(file_path):
+ h.update(os.readlink(file_path).encode('utf-8'))
+ else:
+ with open(file_path, 'rb') as f:
+ h.update(f.read())
+ return f'SHA1: {h.hexdigest()}'
+
+
+def is_soong_prebuilt_module(file_metadata):
+ return file_metadata['soong_module_type'] and file_metadata['soong_module_type'] in [
+ 'android_app_import', 'android_library_import', 'cc_prebuilt_binary', 'cc_prebuilt_library',
+ 'cc_prebuilt_library_headers', 'cc_prebuilt_library_shared', 'cc_prebuilt_library_static', 'cc_prebuilt_object',
+ 'dex_import', 'java_import', 'java_sdk_library_import', 'java_system_modules_import',
+ 'libclang_rt_prebuilt_library_static', 'libclang_rt_prebuilt_library_shared', 'llvm_prebuilt_library_static',
+ 'ndk_prebuilt_object', 'ndk_prebuilt_shared_stl', 'nkd_prebuilt_static_stl', 'prebuilt_apex',
+ 'prebuilt_bootclasspath_fragment', 'prebuilt_dsp', 'prebuilt_firmware', 'prebuilt_kernel_modules',
+ 'prebuilt_rfsa', 'prebuilt_root', 'rust_prebuilt_dylib', 'rust_prebuilt_library', 'rust_prebuilt_rlib',
+ 'vndk_prebuilt_shared',
+
+ # 'android_test_import',
+ # 'cc_prebuilt_test_library_shared',
+ # 'java_import_host',
+ # 'java_test_import',
+ # 'llvm_host_prebuilt_library_shared',
+ # 'prebuilt_apis',
+ # 'prebuilt_build_tool',
+ # 'prebuilt_defaults',
+ # 'prebuilt_etc',
+ # 'prebuilt_etc_host',
+ # 'prebuilt_etc_xml',
+ # 'prebuilt_font',
+ # 'prebuilt_hidl_interfaces',
+ # 'prebuilt_platform_compat_config',
+ # 'prebuilt_stubs_sources',
+ # 'prebuilt_usr_share',
+ # 'prebuilt_usr_share_host',
+ # 'soong_config_module_type_import',
+ ]
+
+
+def is_source_package(file_metadata):
+ module_path = file_metadata['module_path']
+ return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)
+
+
+def is_prebuilt_package(file_metadata):
+ module_path = file_metadata['module_path']
+ if module_path:
+ return (module_path.startswith('prebuilts/') or
+ is_soong_prebuilt_module(file_metadata) or
+ file_metadata['is_prebuilt_make_module'])
+
+ kernel_module_copy_files = file_metadata['kernel_module_copy_files']
+ if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
+ return True
+
+ return False
+
+
+def get_source_package_info(file_metadata, metadata_file_path):
+ """Return source package info exists in its METADATA file, currently including name, security tag
+ and external SBOM reference.
+
+ See go/android-spdx and go/android-sbom-gen for more details.
+ """
+ if not metadata_file_path:
+ return file_metadata['module_path'], []
+
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ external_refs = []
+ for tag in metadata_proto.third_party.security.tag:
+ if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
+ external_refs.append(
+ sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
+ type=sbom_data.PackageExternalRefType.cpe23Type,
+ locator=tag.removeprefix(NVD_CPE23)))
+ elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
+ external_refs.append(
+ sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
+ type=sbom_data.PackageExternalRefType.cpe22Type,
+ locator=tag.removeprefix(NVD_CPE23)))
+
+ if metadata_proto.name:
+ return metadata_proto.name, external_refs
+ else:
+ return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name
+
+
+def get_prebuilt_package_name(file_metadata, metadata_file_path):
+ """Return name of a prebuilt package, which can be from the METADATA file, metadata file path,
+ module path or kernel module's source path if the installed file is a kernel module.
+
+ See go/android-spdx and go/android-sbom-gen for more details.
+ """
+ name = None
+ if metadata_file_path:
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.name:
+ name = metadata_proto.name
+ else:
+ name = metadata_file_path
+ elif file_metadata['module_path']:
+ name = file_metadata['module_path']
+ elif file_metadata['kernel_module_copy_files']:
+ src_path = file_metadata['kernel_module_copy_files'].split(':')[0]
+ name = os.path.dirname(src_path)
+
+ return name.removeprefix('prebuilts/').replace('/', '-')
+
+
+def get_metadata_file_path(file_metadata):
+ """Search for METADATA file of a package and return its path."""
+ metadata_path = ''
+ if file_metadata['module_path']:
+ metadata_path = file_metadata['module_path']
+ elif file_metadata['kernel_module_copy_files']:
+ metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0])
+
+ while metadata_path and not os.path.exists(metadata_path + '/METADATA'):
+ metadata_path = os.path.dirname(metadata_path)
+
+ return metadata_path
+
+
+def get_package_version(metadata_file_path):
+ """Return a package's version in its METADATA file."""
+ if not metadata_file_path:
+ return None
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ return metadata_proto.third_party.version
+
+
+def get_package_homepage(metadata_file_path):
+ """Return a package's homepage URL in its METADATA file."""
+ if not metadata_file_path:
+ return None
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.homepage:
+ return metadata_proto.third_party.homepage
+ for url in metadata_proto.third_party.url:
+ if url.type == metadata_file_pb2.URL.Type.HOMEPAGE:
+ return url.value
+
+ return None
+
+
+def get_package_download_location(metadata_file_path):
+ """Return a package's code repository URL in its METADATA file."""
+ if not metadata_file_path:
+ return None
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.url:
+ urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type)
+ if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE:
+ return urls[0].value
+ elif len(urls) > 1:
+ return urls[1].value
+
+ return None
+
+
+def get_sbom_fragments(installed_file_metadata, metadata_file_path):
+ """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT
+ package, a UPSTREAM package if it's a source package and a external SBOM document reference if
+ it's a prebuilt package with sbom_ref defined in its METADATA file.
+
+ See go/android-spdx and go/android-sbom-gen for more details.
+ """
+ external_doc_ref = None
+ packages = []
+ relationships = []
+
+ # Info from METADATA file
+ homepage = get_package_homepage(metadata_file_path)
+ version = get_package_version(metadata_file_path)
+ download_location = get_package_download_location(metadata_file_path)
+
+ if is_source_package(installed_file_metadata):
+ # Source fork packages
+ name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
+ source_package_id = new_package_id(name, PKG_SOURCE)
+ source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version,
+ supplier='Organization: ' + args.product_mfr,
+ external_refs=external_refs)
+
+ upstream_package_id = new_package_id(name, PKG_UPSTREAM)
+ upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
+ supplier='Organization: ' + homepage if homepage else None,
+ download_location=download_location)
+ packages += [source_package, upstream_package]
+ relationships.append(sbom_data.Relationship(id1=source_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=upstream_package_id))
+ elif is_prebuilt_package(installed_file_metadata):
+ # Prebuilt fork packages
+ name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
+ prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
+ prebuilt_package = sbom_data.Package(id=prebuilt_package_id,
+ name=name,
+ version=args.build_version,
+ supplier='Organization: ' + args.product_mfr)
+ packages.append(prebuilt_package)
+
+ if metadata_file_path:
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
+ sbom_url = metadata_proto.third_party.sbom_ref.url
+ sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
+ upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
+ if sbom_url and sbom_checksum and upstream_element_id:
+ doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}'
+ external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
+ uri=sbom_url,
+ checksum=sbom_checksum)
+ relationships.append(
+ sbom_data.Relationship(id1=prebuilt_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=doc_ref_id + ':' + upstream_element_id))
+
+ return external_doc_ref, packages, relationships
+
+
+def generate_package_verification_code(files):
+ checksums = [file.checksum for file in files]
+ checksums.sort()
+ h = hashlib.sha1()
+ h.update(''.join(checksums).encode(encoding='utf-8'))
+ return h.hexdigest()
+
+
+def save_report(report):
+ prefix, _ = os.path.splitext(args.output_file)
+ with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file:
+ for type, issues in report.items():
+ report_file.write(type + '\n')
+ for issue in issues:
+ report_file.write('\t' + issue + '\n')
+ report_file.write('\n')
+
+
+# Validate the metadata generated by Make for installed files and report if there is no metadata.
+def installed_file_has_metadata(installed_file_metadata, report):
+ installed_file = installed_file_metadata['installed_file']
+ module_path = installed_file_metadata['module_path']
+ product_copy_files = installed_file_metadata['product_copy_files']
+ kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
+ is_platform_generated = installed_file_metadata['is_platform_generated']
+
+ if (not module_path and
+ not product_copy_files and
+ not kernel_module_copy_files and
+ not is_platform_generated and
+ not installed_file.endswith('.fsv_meta')):
+ report[ISSUE_NO_METADATA].append(installed_file)
+ return False
+
+ return True
+
+
+def report_metadata_file(metadata_file_path, installed_file_metadata, report):
+ if metadata_file_path:
+ report[INFO_METADATA_FOUND_FOR_PACKAGE].append(
+ 'installed_file: {}, module_path: {}, METADATA file: {}'.format(
+ installed_file_metadata['installed_file'],
+ installed_file_metadata['module_path'],
+ metadata_file_path + '/METADATA'))
+
+ package_metadata = metadata_file_pb2.Metadata()
+ with open(metadata_file_path + '/METADATA', 'rt') as f:
+ text_format.Parse(f.read(), package_metadata)
+
+ if not metadata_file_path in metadata_file_protos:
+ metadata_file_protos[metadata_file_path] = package_metadata
+ if not package_metadata.name:
+ report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"')
+
+ if not package_metadata.third_party.version:
+ report[ISSUE_METADATA_FILE_INCOMPLETE].append(
+ f'{metadata_file_path}/METADATA does not has "third_party.version"')
+
+ for tag in package_metadata.third_party.security.tag:
+ if not tag.startswith(NVD_CPE23):
+ report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append(
+ f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA')
+ else:
+ report[ISSUE_NO_METADATA_FILE].append(
+ "installed_file: {}, module_path: {}".format(
+ installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
+
+
+def generate_sbom_for_unbundled():
+ with open(args.metadata, newline='') as sbom_metadata_file:
+ reader = csv.DictReader(sbom_metadata_file)
+ doc = sbom_data.Document(name=args.build_version,
+ namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
+ creators=['Organization: ' + args.product_mfr])
+ for installed_file_metadata in reader:
+ installed_file = installed_file_metadata['installed_file']
+ if args.output_file != args.product_out_dir + installed_file + ".spdx":
+ continue
+
+ module_path = installed_file_metadata['module_path']
+ package_id = new_package_id(module_path, PKG_PREBUILT)
+ package = sbom_data.Package(id=package_id,
+ name=module_path,
+ version=args.build_version,
+ supplier='Organization: ' + args.product_mfr)
+ file_id = new_file_id(installed_file)
+ file = sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file))
+ relationship = sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=package_id)
+ doc.add_package(package)
+ doc.files.append(file)
+ doc.describes = file_id
+ doc.add_relationship(relationship)
+ doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+ break
+
+ with open(args.output_file, 'w', encoding="utf-8") as file:
+ sbom_writers.TagValueWriter.write(doc, file, fragment=True)
+
+
+def main():
+ global args
+ args = get_args()
+ log('Args:', vars(args))
+
+ if args.unbundled:
+ generate_sbom_for_unbundled()
+ return
+
+ global metadata_file_protos
+ metadata_file_protos = {}
+
+ doc = sbom_data.Document(name=args.build_version,
+ namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
+ creators=['Organization: ' + args.product_mfr])
+
+ product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,
+ name=sbom_data.PACKAGE_NAME_PRODUCT,
+ version=args.build_version,
+ supplier='Organization: ' + args.product_mfr,
+ files_analyzed=True)
+ doc.packages.append(product_package)
+
+ doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
+ name=sbom_data.PACKAGE_NAME_PLATFORM,
+ version=args.build_version,
+ supplier='Organization: ' + args.product_mfr))
+
+ # Report on some issues and information
+ report = {
+ ISSUE_NO_METADATA: [],
+ ISSUE_NO_METADATA_FILE: [],
+ ISSUE_METADATA_FILE_INCOMPLETE: [],
+ ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [],
+ ISSUE_INSTALLED_FILE_NOT_EXIST: [],
+ INFO_METADATA_FOUND_FOR_PACKAGE: [],
+ }
+
+ # Scan the metadata in CSV file and create the corresponding package and file records in SPDX
+ with open(args.metadata, newline='') as sbom_metadata_file:
+ reader = csv.DictReader(sbom_metadata_file)
+ for installed_file_metadata in reader:
+ installed_file = installed_file_metadata['installed_file']
+ module_path = installed_file_metadata['module_path']
+ product_copy_files = installed_file_metadata['product_copy_files']
+ kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
+
+ if not installed_file_has_metadata(installed_file_metadata, report):
+ continue
+ file_path = args.product_out_dir + '/' + installed_file
+ if not (os.path.islink(file_path) or os.path.isfile(file_path)):
+ report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
+ continue
+
+ file_id = new_file_id(installed_file)
+ doc.files.append(
+ sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file)))
+ product_package.file_ids.append(file_id)
+
+ if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
+ metadata_file_path = get_metadata_file_path(installed_file_metadata)
+ report_metadata_file(metadata_file_path, installed_file_metadata, report)
+
+ # File from source fork packages or prebuilt fork packages
+ external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path)
+ if len(pkgs) > 0:
+ if external_doc_ref:
+ doc.add_external_ref(external_doc_ref)
+ for p in pkgs:
+ doc.add_package(p)
+ for rel in rels:
+ doc.add_relationship(rel)
+ fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=fork_package_id))
+ elif module_path or installed_file_metadata['is_platform_generated']:
+ # File from PLATFORM package
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ elif product_copy_files:
+ # Format of product_copy_files: <source path>:<dest path>
+ src_path = product_copy_files.split(':')[0]
+ # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
+ # so process them as files from PLATFORM package
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ elif installed_file.endswith('.fsv_meta'):
+ # See build/make/core/Makefile:2988
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ elif kernel_module_copy_files.startswith('ANDROID-GEN'):
+ # For the four files generated for _dlkm, _ramdisk partitions
+ # See build/make/core/Makefile:323
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+
+ product_package.verification_code = generate_package_verification_code(doc.files)
+
+ # Save SBOM records to output file
+ doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+ with open(args.output_file, 'w', encoding="utf-8") as file:
+ sbom_writers.TagValueWriter.write(doc, file)
+ if args.json:
+ with open(args.output_file+'.json', 'w', encoding="utf-8") as file:
+ sbom_writers.JSONWriter.write(doc, file)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py
new file mode 100644
index 0000000..0c380f6
--- /dev/null
+++ b/tools/sbom/sbom_data.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Define data classes that model SBOMs defined by SPDX. The data classes could be
+written out to different formats (tagvalue, JSON, etc) of SPDX with corresponding
+writer utilities.
+
+Rrefer to SPDX 2.3 spec: https://spdx.github.io/spdx-spec/v2.3/ and go/android-spdx for details of
+fields in each data class.
+"""
+
+from dataclasses import dataclass, field
+from typing import List
+
+SPDXID_DOC = 'SPDXRef-DOCUMENT'
+SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
+SPDXID_PLATFORM = 'SPDXRef-PLATFORM'
+
+PACKAGE_NAME_PRODUCT = 'PRODUCT'
+PACKAGE_NAME_PLATFORM = 'PLATFORM'
+
+
+class PackageExternalRefCategory:
+ SECURITY = 'SECURITY'
+ PACKAGE_MANAGER = 'PACKAGE-MANAGER'
+ PERSISTENT_ID = 'PERSISTENT-ID'
+ OTHER = 'OTHER'
+
+
+class PackageExternalRefType:
+ cpe22Type = 'cpe22Type'
+ cpe23Type = 'cpe23Type'
+
+
+@dataclass
+class PackageExternalRef:
+ category: PackageExternalRefCategory
+ type: PackageExternalRefType
+ locator: str
+
+
+@dataclass
+class Package:
+ name: str
+ id: str
+ version: str = None
+ supplier: str = None
+ download_location: str = None
+ files_analyzed: bool = False
+ verification_code: str = None
+ file_ids: List[str] = field(default_factory=list)
+ external_refs: List[PackageExternalRef] = field(default_factory=list)
+
+
+@dataclass
+class File:
+ id: str
+ name: str
+ checksum: str
+
+
+class RelationshipType:
+ DESCRIBES = 'DESCRIBES'
+ VARIANT_OF = 'VARIANT_OF'
+ GENERATED_FROM = 'GENERATED_FROM'
+
+
+@dataclass
+class Relationship:
+ id1: str
+ relationship: RelationshipType
+ id2: str
+
+
+@dataclass
+class DocumentExternalReference:
+ id: str
+ uri: str
+ checksum: str
+
+
+@dataclass
+class Document:
+ name: str
+ namespace: str
+ id: str = SPDXID_DOC
+ describes: str = SPDXID_PRODUCT
+ creators: List[str] = field(default_factory=list)
+ created: str = None
+ external_refs: List[DocumentExternalReference] = field(default_factory=list)
+ packages: List[Package] = field(default_factory=list)
+ files: List[File] = field(default_factory=list)
+ relationships: List[Relationship] = field(default_factory=list)
+
+ def add_external_ref(self, external_ref):
+ if not any(external_ref.uri == ref.uri for ref in self.external_refs):
+ self.external_refs.append(external_ref)
+
+ def add_package(self, package):
+ if not any(package.id == p.id for p in self.packages):
+ self.packages.append(package)
+
+ def add_relationship(self, rel):
+ if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
+ for r in self.relationships):
+ self.relationships.append(rel)
diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py
new file mode 100644
index 0000000..66aa6b4
--- /dev/null
+++ b/tools/sbom/sbom_writers.py
@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Serialize objects defined in package sbom_data to SPDX format: tagvalue, JSON.
+"""
+
+import json
+import sbom_data
+
+SPDX_VER = 'SPDX-2.3'
+DATA_LIC = 'CC0-1.0'
+
+
+class Tags:
+ # Common
+ SPDXID = 'SPDXID'
+ SPDX_VERSION = 'SPDXVersion'
+ DATA_LICENSE = 'DataLicense'
+ DOCUMENT_NAME = 'DocumentName'
+ DOCUMENT_NAMESPACE = 'DocumentNamespace'
+ CREATED = 'Created'
+ CREATOR = 'Creator'
+ EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef'
+
+ # Package
+ PACKAGE_NAME = 'PackageName'
+ PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation'
+ PACKAGE_VERSION = 'PackageVersion'
+ PACKAGE_SUPPLIER = 'PackageSupplier'
+ FILES_ANALYZED = 'FilesAnalyzed'
+ PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode'
+ PACKAGE_EXTERNAL_REF = 'ExternalRef'
+ # Package license
+ PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded'
+ PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles'
+ PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared'
+ PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments'
+
+ # File
+ FILE_NAME = 'FileName'
+ FILE_CHECKSUM = 'FileChecksum'
+ # File license
+ FILE_LICENSE_CONCLUDED = 'LicenseConcluded'
+ FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile'
+ FILE_LICENSE_COMMENTS = 'LicenseComments'
+ FILE_COPYRIGHT_TEXT = 'FileCopyrightText'
+ FILE_NOTICE = 'FileNotice'
+ FILE_ATTRIBUTION_TEXT = 'FileAttributionText'
+
+ # Relationship
+ RELATIONSHIP = 'Relationship'
+
+
+class TagValueWriter:
+ @staticmethod
+ def marshal_doc_headers(sbom_doc):
+ headers = [
+ f'{Tags.SPDX_VERSION}: {SPDX_VER}',
+ f'{Tags.DATA_LICENSE}: {DATA_LIC}',
+ f'{Tags.SPDXID}: {sbom_doc.id}',
+ f'{Tags.DOCUMENT_NAME}: {sbom_doc.name}',
+ f'{Tags.DOCUMENT_NAMESPACE}: {sbom_doc.namespace}',
+ ]
+ for creator in sbom_doc.creators:
+ headers.append(f'{Tags.CREATOR}: {creator}')
+ headers.append(f'{Tags.CREATED}: {sbom_doc.created}')
+ for doc_ref in sbom_doc.external_refs:
+ headers.append(
+ f'{Tags.EXTERNAL_DOCUMENT_REF}: {doc_ref.id} {doc_ref.uri} {doc_ref.checksum}')
+ headers.append('')
+ return headers
+
+ @staticmethod
+ def marshal_package(package):
+ download_location = 'NONE'
+ if package.download_location:
+ download_location = package.download_location
+ tagvalues = [
+ f'{Tags.PACKAGE_NAME}: {package.name}',
+ f'{Tags.SPDXID}: {package.id}',
+ f'{Tags.PACKAGE_DOWNLOAD_LOCATION}: {download_location}',
+ f'{Tags.FILES_ANALYZED}: {str(package.files_analyzed).lower()}',
+ ]
+ if package.version:
+ tagvalues.append(f'{Tags.PACKAGE_VERSION}: {package.version}')
+ if package.supplier:
+ tagvalues.append(f'{Tags.PACKAGE_SUPPLIER}: {package.supplier}')
+ if package.verification_code:
+ tagvalues.append(f'{Tags.PACKAGE_VERIFICATION_CODE}: {package.verification_code}')
+ if package.external_refs:
+ for external_ref in package.external_refs:
+ tagvalues.append(
+ f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')
+
+ tagvalues.append('')
+ return tagvalues
+
+ @staticmethod
+ def marshal_described_element(sbom_doc):
+ if not sbom_doc.describes:
+ return None
+
+ product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
+ if product_package:
+ tagvalues = TagValueWriter.marshal_package(product_package[0])
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
+
+ tagvalues.append('')
+ return tagvalues
+
+ file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
+ if file:
+ tagvalues = [
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}'
+ ]
+
+ return tagvalues
+
+ return None
+
+ @staticmethod
+ def marshal_packages(sbom_doc):
+ tagvalues = []
+ marshaled_relationships = []
+ i = 0
+ packages = sbom_doc.packages
+ while i < len(packages):
+ if packages[i].id == sbom_doc.describes:
+ i += 1
+ continue
+
+ if i + 1 < len(packages) \
+ and packages[i].id.startswith('SPDXRef-SOURCE-') \
+ and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
+ tagvalues += TagValueWriter.marshal_package(packages[i])
+ tagvalues += TagValueWriter.marshal_package(packages[i + 1])
+ rel = next((r for r in sbom_doc.relationships if
+ r.id1 == packages[i].id and
+ r.id2 == packages[i + 1].id and
+ r.relationship == sbom_data.RelationshipType.VARIANT_OF), None)
+ if rel:
+ marshaled_relationships.append(rel)
+ tagvalues.append(TagValueWriter.marshal_relationship(rel))
+ tagvalues.append('')
+
+ i += 2
+ else:
+ tagvalues += TagValueWriter.marshal_package(packages[i])
+ i += 1
+
+ return tagvalues, marshaled_relationships
+
+ @staticmethod
+ def marshal_file(file):
+ tagvalues = [
+ f'{Tags.FILE_NAME}: {file.name}',
+ f'{Tags.SPDXID}: {file.id}',
+ f'{Tags.FILE_CHECKSUM}: {file.checksum}',
+ '',
+ ]
+
+ return tagvalues
+
+ @staticmethod
+ def marshal_files(sbom_doc):
+ tagvalues = []
+ for file in sbom_doc.files:
+ tagvalues += TagValueWriter.marshal_file(file)
+ return tagvalues
+
+ @staticmethod
+ def marshal_relationship(rel):
+ return f'{Tags.RELATIONSHIP}: {rel.id1} {rel.relationship} {rel.id2}'
+
+ @staticmethod
+ def marshal_relationships(sbom_doc, marshaled_rels):
+ tagvalues = []
+ sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.id2 + r.id1)
+ for rel in sorted_rels:
+ if any(r.id1 == rel.id1 and r.id2 == rel.id2 and r.relationship == rel.relationship
+ for r in marshaled_rels):
+ continue
+ tagvalues.append(TagValueWriter.marshal_relationship(rel))
+ tagvalues.append('')
+ return tagvalues
+
+ @staticmethod
+ def write(sbom_doc, file, fragment=False):
+ content = []
+ if not fragment:
+ content += TagValueWriter.marshal_doc_headers(sbom_doc)
+ described_element = TagValueWriter.marshal_described_element(sbom_doc)
+ if described_element:
+ content += described_element
+ content += TagValueWriter.marshal_files(sbom_doc)
+ tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
+ content += tagvalues
+ content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
+ file.write('\n'.join(content))
+
+
+class PropNames:
+ # Common
+ SPDXID = 'SPDXID'
+ SPDX_VERSION = 'spdxVersion'
+ DATA_LICENSE = 'dataLicense'
+ NAME = 'name'
+ DOCUMENT_NAMESPACE = 'documentNamespace'
+ CREATION_INFO = 'creationInfo'
+ CREATORS = 'creators'
+ CREATED = 'created'
+ EXTERNAL_DOCUMENT_REF = 'externalDocumentRefs'
+ DOCUMENT_DESCRIBES = 'documentDescribes'
+ EXTERNAL_DOCUMENT_ID = 'externalDocumentId'
+ EXTERNAL_DOCUMENT_URI = 'spdxDocument'
+ EXTERNAL_DOCUMENT_CHECKSUM = 'checksum'
+ ALGORITHM = 'algorithm'
+ CHECKSUM_VALUE = 'checksumValue'
+
+ # Package
+ PACKAGES = 'packages'
+ PACKAGE_DOWNLOAD_LOCATION = 'downloadLocation'
+ PACKAGE_VERSION = 'versionInfo'
+ PACKAGE_SUPPLIER = 'supplier'
+ FILES_ANALYZED = 'filesAnalyzed'
+ PACKAGE_VERIFICATION_CODE = 'packageVerificationCode'
+ PACKAGE_VERIFICATION_CODE_VALUE = 'packageVerificationCodeValue'
+ PACKAGE_EXTERNAL_REFS = 'externalRefs'
+ PACKAGE_EXTERNAL_REF_CATEGORY = 'referenceCategory'
+ PACKAGE_EXTERNAL_REF_TYPE = 'referenceType'
+ PACKAGE_EXTERNAL_REF_LOCATOR = 'referenceLocator'
+ PACKAGE_HAS_FILES = 'hasFiles'
+
+ # File
+ FILES = 'files'
+ FILE_NAME = 'fileName'
+ FILE_CHECKSUMS = 'checksums'
+
+ # Relationship
+ RELATIONSHIPS = 'relationships'
+ REL_ELEMENT_ID = 'spdxElementId'
+ REL_RELATED_ELEMENT_ID = 'relatedSpdxElement'
+ REL_TYPE = 'relationshipType'
+
+
+class JSONWriter:
+ @staticmethod
+ def marshal_doc_headers(sbom_doc):
+ headers = {
+ PropNames.SPDX_VERSION: SPDX_VER,
+ PropNames.DATA_LICENSE: DATA_LIC,
+ PropNames.SPDXID: sbom_doc.id,
+ PropNames.NAME: sbom_doc.name,
+ PropNames.DOCUMENT_NAMESPACE: sbom_doc.namespace,
+ PropNames.CREATION_INFO: {}
+ }
+ creators = [creator for creator in sbom_doc.creators]
+ headers[PropNames.CREATION_INFO][PropNames.CREATORS] = creators
+ headers[PropNames.CREATION_INFO][PropNames.CREATED] = sbom_doc.created
+ external_refs = []
+ for doc_ref in sbom_doc.external_refs:
+ checksum = doc_ref.checksum.split(': ')
+ external_refs.append({
+ PropNames.EXTERNAL_DOCUMENT_ID: f'{doc_ref.id}',
+ PropNames.EXTERNAL_DOCUMENT_URI: doc_ref.uri,
+ PropNames.EXTERNAL_DOCUMENT_CHECKSUM: {
+ PropNames.ALGORITHM: checksum[0],
+ PropNames.CHECKSUM_VALUE: checksum[1]
+ }
+ })
+ if external_refs:
+ headers[PropNames.EXTERNAL_DOCUMENT_REF] = external_refs
+ headers[PropNames.DOCUMENT_DESCRIBES] = [sbom_doc.describes]
+
+ return headers
+
+ @staticmethod
+ def marshal_packages(sbom_doc):
+ packages = []
+ for p in sbom_doc.packages:
+ package = {
+ PropNames.NAME: p.name,
+ PropNames.SPDXID: p.id,
+ PropNames.PACKAGE_DOWNLOAD_LOCATION: p.download_location if p.download_location else 'NONE',
+ PropNames.FILES_ANALYZED: p.files_analyzed
+ }
+ if p.version:
+ package[PropNames.PACKAGE_VERSION] = p.version
+ if p.supplier:
+ package[PropNames.PACKAGE_SUPPLIER] = p.supplier
+ if p.verification_code:
+ package[PropNames.PACKAGE_VERIFICATION_CODE] = {
+ PropNames.PACKAGE_VERIFICATION_CODE_VALUE: p.verification_code
+ }
+ if p.external_refs:
+ package[PropNames.PACKAGE_EXTERNAL_REFS] = []
+ for ref in p.external_refs:
+ ext_ref = {
+ PropNames.PACKAGE_EXTERNAL_REF_CATEGORY: ref.category,
+ PropNames.PACKAGE_EXTERNAL_REF_TYPE: ref.type,
+ PropNames.PACKAGE_EXTERNAL_REF_LOCATOR: ref.locator,
+ }
+ package[PropNames.PACKAGE_EXTERNAL_REFS].append(ext_ref)
+ if p.file_ids:
+ package[PropNames.PACKAGE_HAS_FILES] = []
+ for file_id in p.file_ids:
+ package[PropNames.PACKAGE_HAS_FILES].append(file_id)
+
+ packages.append(package)
+
+ return {PropNames.PACKAGES: packages}
+
+ @staticmethod
+ def marshal_files(sbom_doc):
+ files = []
+ for f in sbom_doc.files:
+ file = {
+ PropNames.FILE_NAME: f.name,
+ PropNames.SPDXID: f.id
+ }
+ checksum = f.checksum.split(': ')
+ file[PropNames.FILE_CHECKSUMS] = [{
+ PropNames.ALGORITHM: checksum[0],
+ PropNames.CHECKSUM_VALUE: checksum[1],
+ }]
+ files.append(file)
+ return {PropNames.FILES: files}
+
+ @staticmethod
+ def marshal_relationships(sbom_doc):
+ relationships = []
+ sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.relationship + r.id2 + r.id1)
+ for r in sorted_rels:
+ rel = {
+ PropNames.REL_ELEMENT_ID: r.id1,
+ PropNames.REL_RELATED_ELEMENT_ID: r.id2,
+ PropNames.REL_TYPE: r.relationship,
+ }
+ relationships.append(rel)
+
+ return {PropNames.RELATIONSHIPS: relationships}
+
+ @staticmethod
+ def write(sbom_doc, file):
+ doc = {}
+ doc.update(JSONWriter.marshal_doc_headers(sbom_doc))
+ doc.update(JSONWriter.marshal_packages(sbom_doc))
+ doc.update(JSONWriter.marshal_files(sbom_doc))
+ doc.update(JSONWriter.marshal_relationships(sbom_doc))
+ file.write(json.dumps(doc, indent=4))
diff --git a/tools/sbom/sbom_writers_test.py b/tools/sbom/sbom_writers_test.py
new file mode 100644
index 0000000..4db2bb7
--- /dev/null
+++ b/tools/sbom/sbom_writers_test.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import pathlib
+import unittest
+import sbom_data
+import sbom_writers
+
+BUILD_FINGER_PRINT = 'build_finger_print'
+SUPPLIER_GOOGLE = 'Organization: Google'
+SUPPLIER_UPSTREAM = 'Organization: upstream'
+
+SPDXID_PREBUILT_PACKAGE1 = 'SPDXRef-PREBUILT-package1'
+SPDXID_SOURCE_PACKAGE1 = 'SPDXRef-SOURCE-package1'
+SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1'
+
+SPDXID_FILE1 = 'SPDXRef-file1'
+SPDXID_FILE2 = 'SPDXRef-file2'
+SPDXID_FILE3 = 'SPDXRef-file3'
+
+
+class SBOMWritersTest(unittest.TestCase):
+
+ def setUp(self):
+ # SBOM of a product
+ self.sbom_doc = sbom_data.Document(name='test doc',
+ namespace='http://www.google.com/sbom/spdx/android',
+ creators=[SUPPLIER_GOOGLE],
+ created='2023-03-31T22:17:58Z',
+ describes=sbom_data.SPDXID_PRODUCT)
+ self.sbom_doc.add_external_ref(
+ sbom_data.DocumentExternalReference(id='DocumentRef-external_doc_ref',
+ uri='external_doc_uri',
+ checksum='SHA1: 1234567890'))
+ self.sbom_doc.add_package(
+ sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,
+ name=sbom_data.PACKAGE_NAME_PRODUCT,
+ supplier=SUPPLIER_GOOGLE,
+ version=BUILD_FINGER_PRINT,
+ files_analyzed=True,
+ verification_code='123456',
+ file_ids=[SPDXID_FILE1, SPDXID_FILE2, SPDXID_FILE3]))
+
+ self.sbom_doc.add_package(
+ sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
+ name=sbom_data.PACKAGE_NAME_PLATFORM,
+ supplier=SUPPLIER_GOOGLE,
+ version=BUILD_FINGER_PRINT,
+ ))
+
+ self.sbom_doc.add_package(
+ sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE1,
+ name='Prebuilt package1',
+ supplier=SUPPLIER_GOOGLE,
+ version=BUILD_FINGER_PRINT,
+ ))
+
+ self.sbom_doc.add_package(
+ sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1,
+ name='Source package1',
+ supplier=SUPPLIER_GOOGLE,
+ version=BUILD_FINGER_PRINT,
+ external_refs=[sbom_data.PackageExternalRef(
+ category=sbom_data.PackageExternalRefCategory.SECURITY,
+ type=sbom_data.PackageExternalRefType.cpe22Type,
+ locator='cpe:/a:jsoncpp_project:jsoncpp:1.9.4')]
+ ))
+
+ self.sbom_doc.add_package(
+ sbom_data.Package(id=SPDXID_UPSTREAM_PACKAGE1,
+ name='Upstream package1',
+ supplier=SUPPLIER_UPSTREAM,
+ version='1.1',
+ ))
+
+ self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_SOURCE_PACKAGE1,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=SPDXID_UPSTREAM_PACKAGE1))
+
+ self.sbom_doc.files.append(
+ sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', checksum='SHA1: 11111'))
+ self.sbom_doc.files.append(
+ sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
+ self.sbom_doc.files.append(
+ sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
+
+ self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE2,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=SPDXID_PREBUILT_PACKAGE1))
+ self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE3,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=SPDXID_SOURCE_PACKAGE1
+ ))
+
+ # SBOM fragment of a APK
+ self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
+ namespace='http://www.google.com/sbom/spdx/android',
+ creators=[SUPPLIER_GOOGLE],
+ created='2023-03-31T22:17:58Z',
+ describes=SPDXID_FILE1)
+
+ self.unbundled_sbom_doc.files.append(
+ sbom_data.File(id=SPDXID_FILE1, name='/bin/file1.apk', checksum='SHA1: 11111'))
+ self.unbundled_sbom_doc.add_package(
+ sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1,
+ name='Unbundled apk package',
+ supplier=SUPPLIER_GOOGLE,
+ version=BUILD_FINGER_PRINT))
+ self.unbundled_sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=SPDXID_SOURCE_PACKAGE1))
+
+ def test_tagvalue_writer(self):
+ with io.StringIO() as output:
+ sbom_writers.TagValueWriter.write(self.sbom_doc, output)
+ expected_output = pathlib.Path('testdata/expected_tagvalue_sbom.spdx').read_text()
+ self.maxDiff = None
+ self.assertEqual(expected_output, output.getvalue())
+
+ def test_tagvalue_writer_unbundled(self):
+ with io.StringIO() as output:
+ sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True)
+ expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_unbundled.spdx').read_text()
+ self.maxDiff = None
+ self.assertEqual(expected_output, output.getvalue())
+
+ def test_json_writer(self):
+ with io.StringIO() as output:
+ sbom_writers.JSONWriter.write(self.sbom_doc, output)
+ expected_output = pathlib.Path('testdata/expected_json_sbom.spdx.json').read_text()
+ self.maxDiff = None
+ self.assertEqual(expected_output, output.getvalue())
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)
diff --git a/tools/sbom/testdata/expected_json_sbom.spdx.json b/tools/sbom/testdata/expected_json_sbom.spdx.json
new file mode 100644
index 0000000..628615f
--- /dev/null
+++ b/tools/sbom/testdata/expected_json_sbom.spdx.json
@@ -0,0 +1,137 @@
+{
+ "spdxVersion": "SPDX-2.3",
+ "dataLicense": "CC0-1.0",
+ "SPDXID": "SPDXRef-DOCUMENT",
+ "name": "test doc",
+ "documentNamespace": "http://www.google.com/sbom/spdx/android",
+ "creationInfo": {
+ "creators": [
+ "Organization: Google"
+ ],
+ "created": "2023-03-31T22:17:58Z"
+ },
+ "externalDocumentRefs": [
+ {
+ "externalDocumentId": "DocumentRef-external_doc_ref",
+ "spdxDocument": "external_doc_uri",
+ "checksum": {
+ "algorithm": "SHA1",
+ "checksumValue": "1234567890"
+ }
+ }
+ ],
+ "documentDescribes": [
+ "SPDXRef-PRODUCT"
+ ],
+ "packages": [
+ {
+ "name": "PRODUCT",
+ "SPDXID": "SPDXRef-PRODUCT",
+ "downloadLocation": "NONE",
+ "filesAnalyzed": true,
+ "versionInfo": "build_finger_print",
+ "supplier": "Organization: Google",
+ "packageVerificationCode": {
+ "packageVerificationCodeValue": "123456"
+ },
+ "hasFiles": [
+ "SPDXRef-file1",
+ "SPDXRef-file2",
+ "SPDXRef-file3"
+ ]
+ },
+ {
+ "name": "PLATFORM",
+ "SPDXID": "SPDXRef-PLATFORM",
+ "downloadLocation": "NONE",
+ "filesAnalyzed": false,
+ "versionInfo": "build_finger_print",
+ "supplier": "Organization: Google"
+ },
+ {
+ "name": "Prebuilt package1",
+ "SPDXID": "SPDXRef-PREBUILT-package1",
+ "downloadLocation": "NONE",
+ "filesAnalyzed": false,
+ "versionInfo": "build_finger_print",
+ "supplier": "Organization: Google"
+ },
+ {
+ "name": "Source package1",
+ "SPDXID": "SPDXRef-SOURCE-package1",
+ "downloadLocation": "NONE",
+ "filesAnalyzed": false,
+ "versionInfo": "build_finger_print",
+ "supplier": "Organization: Google",
+ "externalRefs": [
+ {
+ "referenceCategory": "SECURITY",
+ "referenceType": "cpe22Type",
+ "referenceLocator": "cpe:/a:jsoncpp_project:jsoncpp:1.9.4"
+ }
+ ]
+ },
+ {
+ "name": "Upstream package1",
+ "SPDXID": "SPDXRef-UPSTREAM-package1",
+ "downloadLocation": "NONE",
+ "filesAnalyzed": false,
+ "versionInfo": "1.1",
+ "supplier": "Organization: upstream"
+ }
+ ],
+ "files": [
+ {
+ "fileName": "/bin/file1",
+ "SPDXID": "SPDXRef-file1",
+ "checksums": [
+ {
+ "algorithm": "SHA1",
+ "checksumValue": "11111"
+ }
+ ]
+ },
+ {
+ "fileName": "/bin/file2",
+ "SPDXID": "SPDXRef-file2",
+ "checksums": [
+ {
+ "algorithm": "SHA1",
+ "checksumValue": "22222"
+ }
+ ]
+ },
+ {
+ "fileName": "/bin/file3",
+ "SPDXID": "SPDXRef-file3",
+ "checksums": [
+ {
+ "algorithm": "SHA1",
+ "checksumValue": "33333"
+ }
+ ]
+ }
+ ],
+ "relationships": [
+ {
+ "spdxElementId": "SPDXRef-file1",
+ "relatedSpdxElement": "SPDXRef-PLATFORM",
+ "relationshipType": "GENERATED_FROM"
+ },
+ {
+ "spdxElementId": "SPDXRef-file2",
+ "relatedSpdxElement": "SPDXRef-PREBUILT-package1",
+ "relationshipType": "GENERATED_FROM"
+ },
+ {
+ "spdxElementId": "SPDXRef-file3",
+ "relatedSpdxElement": "SPDXRef-SOURCE-package1",
+ "relationshipType": "GENERATED_FROM"
+ },
+ {
+ "spdxElementId": "SPDXRef-SOURCE-package1",
+ "relatedSpdxElement": "SPDXRef-UPSTREAM-package1",
+ "relationshipType": "VARIANT_OF"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tools/sbom/testdata/expected_tagvalue_sbom.spdx b/tools/sbom/testdata/expected_tagvalue_sbom.spdx
new file mode 100644
index 0000000..0f1c6f8
--- /dev/null
+++ b/tools/sbom/testdata/expected_tagvalue_sbom.spdx
@@ -0,0 +1,65 @@
+SPDXVersion: SPDX-2.3
+DataLicense: CC0-1.0
+SPDXID: SPDXRef-DOCUMENT
+DocumentName: test doc
+DocumentNamespace: http://www.google.com/sbom/spdx/android
+Creator: Organization: Google
+Created: 2023-03-31T22:17:58Z
+ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
+
+PackageName: PRODUCT
+SPDXID: SPDXRef-PRODUCT
+PackageDownloadLocation: NONE
+FilesAnalyzed: true
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+PackageVerificationCode: 123456
+
+Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-PRODUCT
+
+FileName: /bin/file1
+SPDXID: SPDXRef-file1
+FileChecksum: SHA1: 11111
+
+FileName: /bin/file2
+SPDXID: SPDXRef-file2
+FileChecksum: SHA1: 22222
+
+FileName: /bin/file3
+SPDXID: SPDXRef-file3
+FileChecksum: SHA1: 33333
+
+PackageName: PLATFORM
+SPDXID: SPDXRef-PLATFORM
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+PackageName: Prebuilt package1
+SPDXID: SPDXRef-PREBUILT-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+PackageName: Source package1
+SPDXID: SPDXRef-SOURCE-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4
+
+PackageName: Upstream package1
+SPDXID: SPDXRef-UPSTREAM-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: 1.1
+PackageSupplier: Organization: upstream
+
+Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
+
+Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
+Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
+Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx
new file mode 100644
index 0000000..a00c291
--- /dev/null
+++ b/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx
@@ -0,0 +1,12 @@
+FileName: /bin/file1.apk
+SPDXID: SPDXRef-file1
+FileChecksum: SHA1: 11111
+
+PackageName: Unbundled apk package
+SPDXID: SPDXRef-SOURCE-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-SOURCE-package1