blob: 0a8f10a45cb8015401943a0126b77cdcd9173c56 [file] [log] [blame]
Wei Lidec97b12023-04-07 16:45:17 -07001#!/usr/bin/env python3
2#
3# Copyright (C) 2023 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Generate the SBOM of the current target product in SPDX format.
19Usage example:
20 generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \
21 --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \
Wei Lidec97b12023-04-07 16:45:17 -070022 --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
23 --product_mfr=Google
24"""
25
26import argparse
27import csv
28import datetime
29import google.protobuf.text_format as text_format
30import hashlib
31import os
32import metadata_file_pb2
33import sbom_data
34import sbom_writers
35
36
37# Package type
38PKG_SOURCE = 'SOURCE'
39PKG_UPSTREAM = 'UPSTREAM'
40PKG_PREBUILT = 'PREBUILT'
41
42# Security tag
43NVD_CPE23 = 'NVD-CPE2.3:'
44
45# Report
46ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:'
47ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:'
48ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:'
49ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:'
50ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-exist installed files:'
51INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:'
52
Wei Li6f407ba2023-04-19 12:39:07 -070053SOONG_PREBUILT_MODULE_TYPES = [
54 'android_app_import',
55 'android_library_import',
56 'cc_prebuilt_binary',
57 'cc_prebuilt_library',
58 'cc_prebuilt_library_headers',
59 'cc_prebuilt_library_shared',
60 'cc_prebuilt_library_static',
61 'cc_prebuilt_object',
62 'dex_import',
63 'java_import',
64 'java_sdk_library_import',
65 'java_system_modules_import',
66 'libclang_rt_prebuilt_library_static',
67 'libclang_rt_prebuilt_library_shared',
68 'llvm_prebuilt_library_static',
69 'ndk_prebuilt_object',
70 'ndk_prebuilt_shared_stl',
71 'nkd_prebuilt_static_stl',
72 'prebuilt_apex',
73 'prebuilt_bootclasspath_fragment',
74 'prebuilt_dsp',
75 'prebuilt_firmware',
76 'prebuilt_kernel_modules',
77 'prebuilt_rfsa',
78 'prebuilt_root',
79 'rust_prebuilt_dylib',
80 'rust_prebuilt_library',
81 'rust_prebuilt_rlib',
82 'vndk_prebuilt_shared',
83]
84
Wei Li427dacb2023-10-18 16:45:31 -070085THIRD_PARTY_IDENTIFIER_TYPES = [
86 # Types defined in metadata_file.proto
87 'Git',
88 'SVN',
89 'Hg',
90 'Darcs',
91 'VCS',
92 'Archive',
93 'PrebuiltByAlphabet',
94 'LocalSource',
95 'Other',
96 # OSV ecosystems defined at https://ossf.github.io/osv-schema/#affectedpackage-field.
97 'Go',
98 'npm',
99 'OSS-Fuzz',
100 'PyPI',
101 'RubyGems',
102 'crates.io',
103 'Hackage',
104 'GHC',
105 'Packagist',
106 'Maven',
107 'NuGet',
108 'Linux',
109 'Debian',
110 'Alpine',
111 'Hex',
112 'Android',
113 'GitHub Actions',
114 'Pub',
115 'ConanCenter',
116 'Rocky Linux',
117 'AlmaLinux',
118 'Bitnami',
119 'Photon OS',
120 'CRAN',
121 'Bioconductor',
122 'SwiftURL'
123]
124
Wei Lidec97b12023-04-07 16:45:17 -0700125
126def get_args():
127 parser = argparse.ArgumentParser()
128 parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
129 parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
130 parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.')
Wei Lidec97b12023-04-07 16:45:17 -0700131 parser.add_argument('--build_version', required=True, help='The build version.')
132 parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
133 parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
Wei Lifd7e6512023-05-05 10:49:28 -0700134 parser.add_argument('--unbundled_apk', action='store_true', default=False, help='Generate SBOM for unbundled APKs')
135 parser.add_argument('--unbundled_apex', action='store_true', default=False, help='Generate SBOM for unbundled APEXs')
Wei Lidec97b12023-04-07 16:45:17 -0700136
137 return parser.parse_args()
138
139
140def log(*info):
141 if args.verbose:
142 for i in info:
143 print(i)
144
145
146def encode_for_spdxid(s):
147 """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-"""
148 result = ''
149 for c in s:
150 if c.isalnum() or c in '.-':
151 result += c
152 elif c in '_@/':
153 result += '-'
154 else:
155 result += '0x' + c.encode('utf-8').hex()
156
157 return result.lstrip('-')
158
159
160def new_package_id(package_name, type):
161 return f'SPDXRef-{type}-{encode_for_spdxid(package_name)}'
162
163
164def new_file_id(file_path):
165 return f'SPDXRef-{encode_for_spdxid(file_path)}'
166
167
168def checksum(file_path):
Wei Lidec97b12023-04-07 16:45:17 -0700169 h = hashlib.sha1()
170 if os.path.islink(file_path):
171 h.update(os.readlink(file_path).encode('utf-8'))
172 else:
173 with open(file_path, 'rb') as f:
174 h.update(f.read())
175 return f'SHA1: {h.hexdigest()}'
176
177
178def is_soong_prebuilt_module(file_metadata):
Wei Li6f407ba2023-04-19 12:39:07 -0700179 return (file_metadata['soong_module_type'] and
180 file_metadata['soong_module_type'] in SOONG_PREBUILT_MODULE_TYPES)
Wei Lidec97b12023-04-07 16:45:17 -0700181
182
183def is_source_package(file_metadata):
184 module_path = file_metadata['module_path']
185 return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)
186
187
188def is_prebuilt_package(file_metadata):
189 module_path = file_metadata['module_path']
190 if module_path:
191 return (module_path.startswith('prebuilts/') or
192 is_soong_prebuilt_module(file_metadata) or
193 file_metadata['is_prebuilt_make_module'])
194
195 kernel_module_copy_files = file_metadata['kernel_module_copy_files']
196 if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
197 return True
198
199 return False
200
201
202def get_source_package_info(file_metadata, metadata_file_path):
203 """Return source package info exists in its METADATA file, currently including name, security tag
204 and external SBOM reference.
205
206 See go/android-spdx and go/android-sbom-gen for more details.
207 """
208 if not metadata_file_path:
209 return file_metadata['module_path'], []
210
211 metadata_proto = metadata_file_protos[metadata_file_path]
212 external_refs = []
213 for tag in metadata_proto.third_party.security.tag:
214 if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
215 external_refs.append(
216 sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
217 type=sbom_data.PackageExternalRefType.cpe23Type,
218 locator=tag.removeprefix(NVD_CPE23)))
219 elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
220 external_refs.append(
221 sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
222 type=sbom_data.PackageExternalRefType.cpe22Type,
223 locator=tag.removeprefix(NVD_CPE23)))
224
225 if metadata_proto.name:
226 return metadata_proto.name, external_refs
227 else:
228 return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name
229
230
231def get_prebuilt_package_name(file_metadata, metadata_file_path):
232 """Return name of a prebuilt package, which can be from the METADATA file, metadata file path,
233 module path or kernel module's source path if the installed file is a kernel module.
234
235 See go/android-spdx and go/android-sbom-gen for more details.
236 """
237 name = None
238 if metadata_file_path:
239 metadata_proto = metadata_file_protos[metadata_file_path]
240 if metadata_proto.name:
241 name = metadata_proto.name
242 else:
243 name = metadata_file_path
244 elif file_metadata['module_path']:
245 name = file_metadata['module_path']
246 elif file_metadata['kernel_module_copy_files']:
247 src_path = file_metadata['kernel_module_copy_files'].split(':')[0]
248 name = os.path.dirname(src_path)
249
250 return name.removeprefix('prebuilts/').replace('/', '-')
251
252
253def get_metadata_file_path(file_metadata):
254 """Search for METADATA file of a package and return its path."""
255 metadata_path = ''
256 if file_metadata['module_path']:
257 metadata_path = file_metadata['module_path']
258 elif file_metadata['kernel_module_copy_files']:
259 metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0])
260
261 while metadata_path and not os.path.exists(metadata_path + '/METADATA'):
262 metadata_path = os.path.dirname(metadata_path)
263
264 return metadata_path
265
266
267def get_package_version(metadata_file_path):
268 """Return a package's version in its METADATA file."""
269 if not metadata_file_path:
270 return None
271 metadata_proto = metadata_file_protos[metadata_file_path]
272 return metadata_proto.third_party.version
273
274
275def get_package_homepage(metadata_file_path):
276 """Return a package's homepage URL in its METADATA file."""
277 if not metadata_file_path:
278 return None
279 metadata_proto = metadata_file_protos[metadata_file_path]
280 if metadata_proto.third_party.homepage:
281 return metadata_proto.third_party.homepage
282 for url in metadata_proto.third_party.url:
283 if url.type == metadata_file_pb2.URL.Type.HOMEPAGE:
284 return url.value
285
286 return None
287
288
289def get_package_download_location(metadata_file_path):
290 """Return a package's code repository URL in its METADATA file."""
291 if not metadata_file_path:
292 return None
293 metadata_proto = metadata_file_protos[metadata_file_path]
294 if metadata_proto.third_party.url:
295 urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type)
296 if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE:
297 return urls[0].value
298 elif len(urls) > 1:
299 return urls[1].value
300
301 return None
302
303
304def get_sbom_fragments(installed_file_metadata, metadata_file_path):
305 """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT
Wei Li16e7aa32023-05-15 15:11:43 -0700306 package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its
307 METADATA file.
Wei Lidec97b12023-04-07 16:45:17 -0700308
309 See go/android-spdx and go/android-sbom-gen for more details.
310 """
311 external_doc_ref = None
312 packages = []
313 relationships = []
314
315 # Info from METADATA file
316 homepage = get_package_homepage(metadata_file_path)
317 version = get_package_version(metadata_file_path)
318 download_location = get_package_download_location(metadata_file_path)
319
320 if is_source_package(installed_file_metadata):
321 # Source fork packages
322 name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
323 source_package_id = new_package_id(name, PKG_SOURCE)
324 source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version,
Wei Li52908252023-04-14 18:49:42 -0700325 download_location=sbom_data.VALUE_NONE,
Wei Lidec97b12023-04-07 16:45:17 -0700326 supplier='Organization: ' + args.product_mfr,
327 external_refs=external_refs)
328
329 upstream_package_id = new_package_id(name, PKG_UPSTREAM)
330 upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
Wei Li52908252023-04-14 18:49:42 -0700331 supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
Wei Lidec97b12023-04-07 16:45:17 -0700332 download_location=download_location)
333 packages += [source_package, upstream_package]
334 relationships.append(sbom_data.Relationship(id1=source_package_id,
335 relationship=sbom_data.RelationshipType.VARIANT_OF,
336 id2=upstream_package_id))
337 elif is_prebuilt_package(installed_file_metadata):
338 # Prebuilt fork packages
339 name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
340 prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
341 prebuilt_package = sbom_data.Package(id=prebuilt_package_id,
342 name=name,
Wei Li52908252023-04-14 18:49:42 -0700343 download_location=sbom_data.VALUE_NONE,
Wei Li16e7aa32023-05-15 15:11:43 -0700344 version=version if version else args.build_version,
Wei Lidec97b12023-04-07 16:45:17 -0700345 supplier='Organization: ' + args.product_mfr)
Wei Lidec97b12023-04-07 16:45:17 -0700346
Wei Li16e7aa32023-05-15 15:11:43 -0700347 upstream_package_id = new_package_id(name, PKG_UPSTREAM)
348 upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version = version,
349 supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
350 download_location=download_location)
351 packages += [prebuilt_package, upstream_package]
352 relationships.append(sbom_data.Relationship(id1=prebuilt_package_id,
353 relationship=sbom_data.RelationshipType.VARIANT_OF,
354 id2=upstream_package_id))
355
356 if metadata_file_path:
357 metadata_proto = metadata_file_protos[metadata_file_path]
358 if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
359 sbom_url = metadata_proto.third_party.sbom_ref.url
360 sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
361 upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
362 if sbom_url and sbom_checksum and upstream_element_id:
363 doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}'
364 external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
365 uri=sbom_url,
366 checksum=sbom_checksum)
367 relationships.append(
368 sbom_data.Relationship(id1=upstream_package_id,
369 relationship=sbom_data.RelationshipType.VARIANT_OF,
370 id2=doc_ref_id + ':' + upstream_element_id))
Wei Lidec97b12023-04-07 16:45:17 -0700371
372 return external_doc_ref, packages, relationships
373
374
Wei Lifd7e6512023-05-05 10:49:28 -0700375def save_report(report_file_path, report):
376 with open(report_file_path, 'w', encoding='utf-8') as report_file:
Wei Lidec97b12023-04-07 16:45:17 -0700377 for type, issues in report.items():
378 report_file.write(type + '\n')
379 for issue in issues:
380 report_file.write('\t' + issue + '\n')
381 report_file.write('\n')
382
383
384# Validate the metadata generated by Make for installed files and report if there is no metadata.
385def installed_file_has_metadata(installed_file_metadata, report):
386 installed_file = installed_file_metadata['installed_file']
387 module_path = installed_file_metadata['module_path']
388 product_copy_files = installed_file_metadata['product_copy_files']
389 kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
390 is_platform_generated = installed_file_metadata['is_platform_generated']
391
392 if (not module_path and
393 not product_copy_files and
394 not kernel_module_copy_files and
395 not is_platform_generated and
396 not installed_file.endswith('.fsv_meta')):
397 report[ISSUE_NO_METADATA].append(installed_file)
398 return False
399
400 return True
401
402
Wei Li427dacb2023-10-18 16:45:31 -0700403# Validate identifiers in a package's METADATA.
404# 1) Only known identifier type is allowed
405# 2) Only one identifier's primary_source can be true
406def validate_package_metadata(metadata_file_path, package_metadata):
407 primary_source_found = False
408 for identifier in package_metadata.third_party.identifier:
409 if identifier.type not in THIRD_PARTY_IDENTIFIER_TYPES:
410 sys.exit(f'Unknown value of third_party.identifier.type in {metadata_file_path}/METADATA: {identifier.type}.')
411 if primary_source_found and identifier.primary_source:
412 sys.exit(
413 f'Field "primary_source" is set to true in multiple third_party.identifier in {metadata_file_path}/METADATA.')
414 primary_source_found = identifier.primary_source
415
416
Wei Lidec97b12023-04-07 16:45:17 -0700417def report_metadata_file(metadata_file_path, installed_file_metadata, report):
418 if metadata_file_path:
419 report[INFO_METADATA_FOUND_FOR_PACKAGE].append(
420 'installed_file: {}, module_path: {}, METADATA file: {}'.format(
421 installed_file_metadata['installed_file'],
422 installed_file_metadata['module_path'],
423 metadata_file_path + '/METADATA'))
424
425 package_metadata = metadata_file_pb2.Metadata()
426 with open(metadata_file_path + '/METADATA', 'rt') as f:
427 text_format.Parse(f.read(), package_metadata)
428
Wei Li427dacb2023-10-18 16:45:31 -0700429 validate_package_metadata(metadata_file_path, package_metadata)
430
Wei Lidec97b12023-04-07 16:45:17 -0700431 if not metadata_file_path in metadata_file_protos:
432 metadata_file_protos[metadata_file_path] = package_metadata
433 if not package_metadata.name:
434 report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"')
435
436 if not package_metadata.third_party.version:
437 report[ISSUE_METADATA_FILE_INCOMPLETE].append(
438 f'{metadata_file_path}/METADATA does not has "third_party.version"')
439
440 for tag in package_metadata.third_party.security.tag:
441 if not tag.startswith(NVD_CPE23):
442 report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append(
443 f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA')
444 else:
445 report[ISSUE_NO_METADATA_FILE].append(
446 "installed_file: {}, module_path: {}".format(
447 installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
448
449
Wei Lifd7e6512023-05-05 10:49:28 -0700450def generate_sbom_for_unbundled_apk():
Wei Lidec97b12023-04-07 16:45:17 -0700451 with open(args.metadata, newline='') as sbom_metadata_file:
452 reader = csv.DictReader(sbom_metadata_file)
453 doc = sbom_data.Document(name=args.build_version,
454 namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
455 creators=['Organization: ' + args.product_mfr])
456 for installed_file_metadata in reader:
457 installed_file = installed_file_metadata['installed_file']
Wei Lifd7e6512023-05-05 10:49:28 -0700458 if args.output_file != installed_file_metadata['build_output_path'] + '.spdx.json':
Wei Lidec97b12023-04-07 16:45:17 -0700459 continue
460
461 module_path = installed_file_metadata['module_path']
462 package_id = new_package_id(module_path, PKG_PREBUILT)
463 package = sbom_data.Package(id=package_id,
464 name=module_path,
465 version=args.build_version,
466 supplier='Organization: ' + args.product_mfr)
467 file_id = new_file_id(installed_file)
Wei Lifd7e6512023-05-05 10:49:28 -0700468 file = sbom_data.File(id=file_id,
469 name=installed_file,
470 checksum=checksum(installed_file_metadata['build_output_path']))
Wei Lidec97b12023-04-07 16:45:17 -0700471 relationship = sbom_data.Relationship(id1=file_id,
472 relationship=sbom_data.RelationshipType.GENERATED_FROM,
473 id2=package_id)
474 doc.add_package(package)
475 doc.files.append(file)
476 doc.describes = file_id
477 doc.add_relationship(relationship)
478 doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
479 break
480
Wei Li49af9392023-04-12 17:35:26 -0700481 with open(args.output_file, 'w', encoding='utf-8') as file:
482 sbom_writers.JSONWriter.write(doc, file)
483 fragment_file = args.output_file.removesuffix('.spdx.json') + '-fragment.spdx'
484 with open(fragment_file, 'w', encoding='utf-8') as file:
Wei Lidec97b12023-04-07 16:45:17 -0700485 sbom_writers.TagValueWriter.write(doc, file, fragment=True)
486
487
488def main():
489 global args
490 args = get_args()
491 log('Args:', vars(args))
492
Wei Lifd7e6512023-05-05 10:49:28 -0700493 if args.unbundled_apk:
494 generate_sbom_for_unbundled_apk()
Wei Lidec97b12023-04-07 16:45:17 -0700495 return
496
497 global metadata_file_protos
498 metadata_file_protos = {}
499
Wei Lidec97b12023-04-07 16:45:17 -0700500 product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,
501 name=sbom_data.PACKAGE_NAME_PRODUCT,
Wei Li52908252023-04-14 18:49:42 -0700502 download_location=sbom_data.VALUE_NONE,
Wei Lidec97b12023-04-07 16:45:17 -0700503 version=args.build_version,
504 supplier='Organization: ' + args.product_mfr,
505 files_analyzed=True)
Wei Lifd7e6512023-05-05 10:49:28 -0700506
507 doc = sbom_data.Document(name=args.build_version,
508 namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
509 creators=['Organization: ' + args.product_mfr])
510 if not args.unbundled_apex:
511 doc.packages.append(product_package)
Wei Lidec97b12023-04-07 16:45:17 -0700512
513 doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
514 name=sbom_data.PACKAGE_NAME_PLATFORM,
Wei Li52908252023-04-14 18:49:42 -0700515 download_location=sbom_data.VALUE_NONE,
Wei Lidec97b12023-04-07 16:45:17 -0700516 version=args.build_version,
517 supplier='Organization: ' + args.product_mfr))
518
519 # Report on some issues and information
520 report = {
521 ISSUE_NO_METADATA: [],
522 ISSUE_NO_METADATA_FILE: [],
523 ISSUE_METADATA_FILE_INCOMPLETE: [],
524 ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [],
525 ISSUE_INSTALLED_FILE_NOT_EXIST: [],
526 INFO_METADATA_FOUND_FOR_PACKAGE: [],
527 }
528
529 # Scan the metadata in CSV file and create the corresponding package and file records in SPDX
530 with open(args.metadata, newline='') as sbom_metadata_file:
531 reader = csv.DictReader(sbom_metadata_file)
532 for installed_file_metadata in reader:
533 installed_file = installed_file_metadata['installed_file']
534 module_path = installed_file_metadata['module_path']
535 product_copy_files = installed_file_metadata['product_copy_files']
536 kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
Wei Lifd7e6512023-05-05 10:49:28 -0700537 build_output_path = installed_file_metadata['build_output_path']
Wei Lid2636952023-05-30 15:03:03 -0700538 is_static_lib = installed_file_metadata['is_static_lib']
Wei Lidec97b12023-04-07 16:45:17 -0700539
540 if not installed_file_has_metadata(installed_file_metadata, report):
541 continue
Wei Lid2636952023-05-30 15:03:03 -0700542 if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
543 # Ignore non-existing static library files for now since they are not shipped on devices.
Wei Lidec97b12023-04-07 16:45:17 -0700544 report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
545 continue
546
547 file_id = new_file_id(installed_file)
Wei Lid2636952023-05-30 15:03:03 -0700548 # TODO(b/285453664): Soong should report the information of statically linked libraries to Make.
549 # This happens when a different sanitized version of static libraries is used in linking.
550 # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be
551 # located correctly because Soong doesn't report the information to Make.
552 sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string
553 if os.path.islink(build_output_path) or os.path.isfile(build_output_path):
554 sha1 = checksum(build_output_path)
555 doc.files.append(sbom_data.File(id=file_id,
556 name=installed_file,
557 checksum=sha1))
558
559 if not is_static_lib:
560 if not args.unbundled_apex:
561 product_package.file_ids.append(file_id)
562 elif len(doc.files) > 1:
563 doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
Wei Lidec97b12023-04-07 16:45:17 -0700564
565 if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
566 metadata_file_path = get_metadata_file_path(installed_file_metadata)
567 report_metadata_file(metadata_file_path, installed_file_metadata, report)
568
569 # File from source fork packages or prebuilt fork packages
570 external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path)
571 if len(pkgs) > 0:
572 if external_doc_ref:
573 doc.add_external_ref(external_doc_ref)
574 for p in pkgs:
575 doc.add_package(p)
576 for rel in rels:
577 doc.add_relationship(rel)
578 fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package
579 doc.add_relationship(sbom_data.Relationship(id1=file_id,
580 relationship=sbom_data.RelationshipType.GENERATED_FROM,
581 id2=fork_package_id))
582 elif module_path or installed_file_metadata['is_platform_generated']:
583 # File from PLATFORM package
584 doc.add_relationship(sbom_data.Relationship(id1=file_id,
585 relationship=sbom_data.RelationshipType.GENERATED_FROM,
586 id2=sbom_data.SPDXID_PLATFORM))
587 elif product_copy_files:
588 # Format of product_copy_files: <source path>:<dest path>
589 src_path = product_copy_files.split(':')[0]
590 # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
591 # so process them as files from PLATFORM package
592 doc.add_relationship(sbom_data.Relationship(id1=file_id,
593 relationship=sbom_data.RelationshipType.GENERATED_FROM,
594 id2=sbom_data.SPDXID_PLATFORM))
595 elif installed_file.endswith('.fsv_meta'):
596 # See build/make/core/Makefile:2988
597 doc.add_relationship(sbom_data.Relationship(id1=file_id,
598 relationship=sbom_data.RelationshipType.GENERATED_FROM,
599 id2=sbom_data.SPDXID_PLATFORM))
600 elif kernel_module_copy_files.startswith('ANDROID-GEN'):
601 # For the four files generated for _dlkm, _ramdisk partitions
602 # See build/make/core/Makefile:323
603 doc.add_relationship(sbom_data.Relationship(id1=file_id,
604 relationship=sbom_data.RelationshipType.GENERATED_FROM,
605 id2=sbom_data.SPDXID_PLATFORM))
606
Wei Lid2636952023-05-30 15:03:03 -0700607 # Process static libraries and whole static libraries the installed file links to
608 static_libs = installed_file_metadata['static_libraries']
609 whole_static_libs = installed_file_metadata['whole_static_libraries']
610 all_static_libs = (static_libs + ' ' + whole_static_libs).strip()
611 if all_static_libs:
612 for lib in all_static_libs.split(' '):
613 doc.add_relationship(sbom_data.Relationship(id1=file_id,
614 relationship=sbom_data.RelationshipType.STATIC_LINK,
615 id2=new_file_id(lib + '.a')))
Wei Lifd7e6512023-05-05 10:49:28 -0700616
617 if args.unbundled_apex:
618 doc.describes = doc.files[0].id
Wei Lidec97b12023-04-07 16:45:17 -0700619
620 # Save SBOM records to output file
Wei Lid2636952023-05-30 15:03:03 -0700621 doc.generate_packages_verification_code()
Wei Lidec97b12023-04-07 16:45:17 -0700622 doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
Wei Lifd7e6512023-05-05 10:49:28 -0700623 prefix = args.output_file
624 if prefix.endswith('.spdx'):
625 prefix = prefix.removesuffix('.spdx')
626 elif prefix.endswith('.spdx.json'):
627 prefix = prefix.removesuffix('.spdx.json')
628
629 output_file = prefix + '.spdx'
630 if args.unbundled_apex:
631 output_file = prefix + '-fragment.spdx'
632 with open(output_file, 'w', encoding="utf-8") as file:
633 sbom_writers.TagValueWriter.write(doc, file, fragment=args.unbundled_apex)
Wei Lidec97b12023-04-07 16:45:17 -0700634 if args.json:
Wei Lifd7e6512023-05-05 10:49:28 -0700635 with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file:
Wei Lidec97b12023-04-07 16:45:17 -0700636 sbom_writers.JSONWriter.write(doc, file)
637
Wei Lifd7e6512023-05-05 10:49:28 -0700638 save_report(prefix + '-gen-report.txt', report)
639
Wei Lidec97b12023-04-07 16:45:17 -0700640
641if __name__ == '__main__':
642 main()