Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Copyright (C) 2023 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """ |
| 18 | Generate the SBOM of the current target product in SPDX format. |
| 19 | Usage example: |
| 20 | generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \ |
| 21 | --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \ |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 22 | --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \ |
| 23 | --product_mfr=Google |
| 24 | """ |
| 25 | |
| 26 | import argparse |
| 27 | import csv |
| 28 | import datetime |
| 29 | import google.protobuf.text_format as text_format |
| 30 | import hashlib |
| 31 | import os |
| 32 | import metadata_file_pb2 |
| 33 | import sbom_data |
| 34 | import sbom_writers |
| 35 | |
| 36 | |
| 37 | # Package type |
| 38 | PKG_SOURCE = 'SOURCE' |
| 39 | PKG_UPSTREAM = 'UPSTREAM' |
| 40 | PKG_PREBUILT = 'PREBUILT' |
| 41 | |
| 42 | # Security tag |
| 43 | NVD_CPE23 = 'NVD-CPE2.3:' |
| 44 | |
| 45 | # Report |
| 46 | ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:' |
| 47 | ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:' |
| 48 | ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:' |
| 49 | ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:' |
| 50 | ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-exist installed files:' |
| 51 | INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:' |
| 52 | |
Wei Li | 6f407ba | 2023-04-19 12:39:07 -0700 | [diff] [blame] | 53 | SOONG_PREBUILT_MODULE_TYPES = [ |
| 54 | 'android_app_import', |
| 55 | 'android_library_import', |
| 56 | 'cc_prebuilt_binary', |
| 57 | 'cc_prebuilt_library', |
| 58 | 'cc_prebuilt_library_headers', |
| 59 | 'cc_prebuilt_library_shared', |
| 60 | 'cc_prebuilt_library_static', |
| 61 | 'cc_prebuilt_object', |
| 62 | 'dex_import', |
| 63 | 'java_import', |
| 64 | 'java_sdk_library_import', |
| 65 | 'java_system_modules_import', |
| 66 | 'libclang_rt_prebuilt_library_static', |
| 67 | 'libclang_rt_prebuilt_library_shared', |
| 68 | 'llvm_prebuilt_library_static', |
| 69 | 'ndk_prebuilt_object', |
| 70 | 'ndk_prebuilt_shared_stl', |
| 71 | 'nkd_prebuilt_static_stl', |
| 72 | 'prebuilt_apex', |
| 73 | 'prebuilt_bootclasspath_fragment', |
| 74 | 'prebuilt_dsp', |
| 75 | 'prebuilt_firmware', |
| 76 | 'prebuilt_kernel_modules', |
| 77 | 'prebuilt_rfsa', |
| 78 | 'prebuilt_root', |
| 79 | 'rust_prebuilt_dylib', |
| 80 | 'rust_prebuilt_library', |
| 81 | 'rust_prebuilt_rlib', |
| 82 | 'vndk_prebuilt_shared', |
| 83 | ] |
| 84 | |
Wei Li | 427dacb | 2023-10-18 16:45:31 -0700 | [diff] [blame] | 85 | THIRD_PARTY_IDENTIFIER_TYPES = [ |
| 86 | # Types defined in metadata_file.proto |
| 87 | 'Git', |
| 88 | 'SVN', |
| 89 | 'Hg', |
| 90 | 'Darcs', |
| 91 | 'VCS', |
| 92 | 'Archive', |
| 93 | 'PrebuiltByAlphabet', |
| 94 | 'LocalSource', |
| 95 | 'Other', |
| 96 | # OSV ecosystems defined at https://ossf.github.io/osv-schema/#affectedpackage-field. |
| 97 | 'Go', |
| 98 | 'npm', |
| 99 | 'OSS-Fuzz', |
| 100 | 'PyPI', |
| 101 | 'RubyGems', |
| 102 | 'crates.io', |
| 103 | 'Hackage', |
| 104 | 'GHC', |
| 105 | 'Packagist', |
| 106 | 'Maven', |
| 107 | 'NuGet', |
| 108 | 'Linux', |
| 109 | 'Debian', |
| 110 | 'Alpine', |
| 111 | 'Hex', |
| 112 | 'Android', |
| 113 | 'GitHub Actions', |
| 114 | 'Pub', |
| 115 | 'ConanCenter', |
| 116 | 'Rocky Linux', |
| 117 | 'AlmaLinux', |
| 118 | 'Bitnami', |
| 119 | 'Photon OS', |
| 120 | 'CRAN', |
| 121 | 'Bioconductor', |
| 122 | 'SwiftURL' |
| 123 | ] |
| 124 | |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 125 | |
| 126 | def get_args(): |
| 127 | parser = argparse.ArgumentParser() |
| 128 | parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') |
| 129 | parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.') |
| 130 | parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.') |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 131 | parser.add_argument('--build_version', required=True, help='The build version.') |
| 132 | parser.add_argument('--product_mfr', required=True, help='The product manufacturer.') |
| 133 | parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format') |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 134 | parser.add_argument('--unbundled_apk', action='store_true', default=False, help='Generate SBOM for unbundled APKs') |
| 135 | parser.add_argument('--unbundled_apex', action='store_true', default=False, help='Generate SBOM for unbundled APEXs') |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 136 | |
| 137 | return parser.parse_args() |
| 138 | |
| 139 | |
| 140 | def log(*info): |
| 141 | if args.verbose: |
| 142 | for i in info: |
| 143 | print(i) |
| 144 | |
| 145 | |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 146 | def new_package_id(package_name, type): |
Wei Li | c134b76 | 2023-10-17 23:52:30 -0700 | [diff] [blame] | 147 | return f'SPDXRef-{type}-{sbom_data.encode_for_spdxid(package_name)}' |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 148 | |
| 149 | |
| 150 | def new_file_id(file_path): |
Wei Li | c134b76 | 2023-10-17 23:52:30 -0700 | [diff] [blame] | 151 | return f'SPDXRef-{sbom_data.encode_for_spdxid(file_path)}' |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 152 | |
| 153 | |
| 154 | def checksum(file_path): |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 155 | h = hashlib.sha1() |
| 156 | if os.path.islink(file_path): |
| 157 | h.update(os.readlink(file_path).encode('utf-8')) |
| 158 | else: |
| 159 | with open(file_path, 'rb') as f: |
| 160 | h.update(f.read()) |
| 161 | return f'SHA1: {h.hexdigest()}' |
| 162 | |
| 163 | |
| 164 | def is_soong_prebuilt_module(file_metadata): |
Wei Li | 6f407ba | 2023-04-19 12:39:07 -0700 | [diff] [blame] | 165 | return (file_metadata['soong_module_type'] and |
| 166 | file_metadata['soong_module_type'] in SOONG_PREBUILT_MODULE_TYPES) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 167 | |
| 168 | |
| 169 | def is_source_package(file_metadata): |
| 170 | module_path = file_metadata['module_path'] |
| 171 | return module_path.startswith('external/') and not is_prebuilt_package(file_metadata) |
| 172 | |
| 173 | |
| 174 | def is_prebuilt_package(file_metadata): |
| 175 | module_path = file_metadata['module_path'] |
| 176 | if module_path: |
| 177 | return (module_path.startswith('prebuilts/') or |
| 178 | is_soong_prebuilt_module(file_metadata) or |
| 179 | file_metadata['is_prebuilt_make_module']) |
| 180 | |
| 181 | kernel_module_copy_files = file_metadata['kernel_module_copy_files'] |
| 182 | if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'): |
| 183 | return True |
| 184 | |
| 185 | return False |
| 186 | |
| 187 | |
| 188 | def get_source_package_info(file_metadata, metadata_file_path): |
| 189 | """Return source package info exists in its METADATA file, currently including name, security tag |
| 190 | and external SBOM reference. |
| 191 | |
| 192 | See go/android-spdx and go/android-sbom-gen for more details. |
| 193 | """ |
| 194 | if not metadata_file_path: |
| 195 | return file_metadata['module_path'], [] |
| 196 | |
| 197 | metadata_proto = metadata_file_protos[metadata_file_path] |
| 198 | external_refs = [] |
| 199 | for tag in metadata_proto.third_party.security.tag: |
| 200 | if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()): |
| 201 | external_refs.append( |
| 202 | sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, |
| 203 | type=sbom_data.PackageExternalRefType.cpe23Type, |
| 204 | locator=tag.removeprefix(NVD_CPE23))) |
| 205 | elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()): |
| 206 | external_refs.append( |
| 207 | sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, |
| 208 | type=sbom_data.PackageExternalRefType.cpe22Type, |
| 209 | locator=tag.removeprefix(NVD_CPE23))) |
| 210 | |
| 211 | if metadata_proto.name: |
| 212 | return metadata_proto.name, external_refs |
| 213 | else: |
| 214 | return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name |
| 215 | |
| 216 | |
| 217 | def get_prebuilt_package_name(file_metadata, metadata_file_path): |
| 218 | """Return name of a prebuilt package, which can be from the METADATA file, metadata file path, |
| 219 | module path or kernel module's source path if the installed file is a kernel module. |
| 220 | |
| 221 | See go/android-spdx and go/android-sbom-gen for more details. |
| 222 | """ |
| 223 | name = None |
| 224 | if metadata_file_path: |
| 225 | metadata_proto = metadata_file_protos[metadata_file_path] |
| 226 | if metadata_proto.name: |
| 227 | name = metadata_proto.name |
| 228 | else: |
| 229 | name = metadata_file_path |
| 230 | elif file_metadata['module_path']: |
| 231 | name = file_metadata['module_path'] |
| 232 | elif file_metadata['kernel_module_copy_files']: |
| 233 | src_path = file_metadata['kernel_module_copy_files'].split(':')[0] |
| 234 | name = os.path.dirname(src_path) |
| 235 | |
| 236 | return name.removeprefix('prebuilts/').replace('/', '-') |
| 237 | |
| 238 | |
| 239 | def get_metadata_file_path(file_metadata): |
| 240 | """Search for METADATA file of a package and return its path.""" |
| 241 | metadata_path = '' |
| 242 | if file_metadata['module_path']: |
| 243 | metadata_path = file_metadata['module_path'] |
| 244 | elif file_metadata['kernel_module_copy_files']: |
| 245 | metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) |
| 246 | |
| 247 | while metadata_path and not os.path.exists(metadata_path + '/METADATA'): |
| 248 | metadata_path = os.path.dirname(metadata_path) |
| 249 | |
| 250 | return metadata_path |
| 251 | |
| 252 | |
| 253 | def get_package_version(metadata_file_path): |
| 254 | """Return a package's version in its METADATA file.""" |
| 255 | if not metadata_file_path: |
| 256 | return None |
| 257 | metadata_proto = metadata_file_protos[metadata_file_path] |
| 258 | return metadata_proto.third_party.version |
| 259 | |
| 260 | |
| 261 | def get_package_homepage(metadata_file_path): |
| 262 | """Return a package's homepage URL in its METADATA file.""" |
| 263 | if not metadata_file_path: |
| 264 | return None |
| 265 | metadata_proto = metadata_file_protos[metadata_file_path] |
| 266 | if metadata_proto.third_party.homepage: |
| 267 | return metadata_proto.third_party.homepage |
| 268 | for url in metadata_proto.third_party.url: |
| 269 | if url.type == metadata_file_pb2.URL.Type.HOMEPAGE: |
| 270 | return url.value |
| 271 | |
| 272 | return None |
| 273 | |
| 274 | |
| 275 | def get_package_download_location(metadata_file_path): |
| 276 | """Return a package's code repository URL in its METADATA file.""" |
| 277 | if not metadata_file_path: |
| 278 | return None |
| 279 | metadata_proto = metadata_file_protos[metadata_file_path] |
| 280 | if metadata_proto.third_party.url: |
| 281 | urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type) |
| 282 | if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE: |
| 283 | return urls[0].value |
| 284 | elif len(urls) > 1: |
| 285 | return urls[1].value |
| 286 | |
| 287 | return None |
| 288 | |
| 289 | |
| 290 | def get_sbom_fragments(installed_file_metadata, metadata_file_path): |
| 291 | """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT |
Wei Li | 16e7aa3 | 2023-05-15 15:11:43 -0700 | [diff] [blame] | 292 | package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its |
| 293 | METADATA file. |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 294 | |
| 295 | See go/android-spdx and go/android-sbom-gen for more details. |
| 296 | """ |
| 297 | external_doc_ref = None |
| 298 | packages = [] |
| 299 | relationships = [] |
| 300 | |
| 301 | # Info from METADATA file |
| 302 | homepage = get_package_homepage(metadata_file_path) |
| 303 | version = get_package_version(metadata_file_path) |
| 304 | download_location = get_package_download_location(metadata_file_path) |
| 305 | |
| 306 | if is_source_package(installed_file_metadata): |
| 307 | # Source fork packages |
| 308 | name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path) |
| 309 | source_package_id = new_package_id(name, PKG_SOURCE) |
| 310 | source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version, |
Wei Li | 5290825 | 2023-04-14 18:49:42 -0700 | [diff] [blame] | 311 | download_location=sbom_data.VALUE_NONE, |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 312 | supplier='Organization: ' + args.product_mfr, |
| 313 | external_refs=external_refs) |
| 314 | |
| 315 | upstream_package_id = new_package_id(name, PKG_UPSTREAM) |
| 316 | upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version, |
Wei Li | 5290825 | 2023-04-14 18:49:42 -0700 | [diff] [blame] | 317 | supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION, |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 318 | download_location=download_location) |
| 319 | packages += [source_package, upstream_package] |
| 320 | relationships.append(sbom_data.Relationship(id1=source_package_id, |
| 321 | relationship=sbom_data.RelationshipType.VARIANT_OF, |
| 322 | id2=upstream_package_id)) |
| 323 | elif is_prebuilt_package(installed_file_metadata): |
| 324 | # Prebuilt fork packages |
| 325 | name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path) |
| 326 | prebuilt_package_id = new_package_id(name, PKG_PREBUILT) |
| 327 | prebuilt_package = sbom_data.Package(id=prebuilt_package_id, |
| 328 | name=name, |
Wei Li | 5290825 | 2023-04-14 18:49:42 -0700 | [diff] [blame] | 329 | download_location=sbom_data.VALUE_NONE, |
Wei Li | 16e7aa3 | 2023-05-15 15:11:43 -0700 | [diff] [blame] | 330 | version=version if version else args.build_version, |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 331 | supplier='Organization: ' + args.product_mfr) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 332 | |
Wei Li | 16e7aa3 | 2023-05-15 15:11:43 -0700 | [diff] [blame] | 333 | upstream_package_id = new_package_id(name, PKG_UPSTREAM) |
| 334 | upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version = version, |
| 335 | supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION, |
| 336 | download_location=download_location) |
| 337 | packages += [prebuilt_package, upstream_package] |
| 338 | relationships.append(sbom_data.Relationship(id1=prebuilt_package_id, |
| 339 | relationship=sbom_data.RelationshipType.VARIANT_OF, |
| 340 | id2=upstream_package_id)) |
| 341 | |
| 342 | if metadata_file_path: |
| 343 | metadata_proto = metadata_file_protos[metadata_file_path] |
| 344 | if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref': |
| 345 | sbom_url = metadata_proto.third_party.sbom_ref.url |
| 346 | sbom_checksum = metadata_proto.third_party.sbom_ref.checksum |
| 347 | upstream_element_id = metadata_proto.third_party.sbom_ref.element_id |
| 348 | if sbom_url and sbom_checksum and upstream_element_id: |
| 349 | doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}' |
| 350 | external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id, |
| 351 | uri=sbom_url, |
| 352 | checksum=sbom_checksum) |
| 353 | relationships.append( |
| 354 | sbom_data.Relationship(id1=upstream_package_id, |
| 355 | relationship=sbom_data.RelationshipType.VARIANT_OF, |
| 356 | id2=doc_ref_id + ':' + upstream_element_id)) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 357 | |
| 358 | return external_doc_ref, packages, relationships |
| 359 | |
| 360 | |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 361 | def save_report(report_file_path, report): |
| 362 | with open(report_file_path, 'w', encoding='utf-8') as report_file: |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 363 | for type, issues in report.items(): |
| 364 | report_file.write(type + '\n') |
| 365 | for issue in issues: |
| 366 | report_file.write('\t' + issue + '\n') |
| 367 | report_file.write('\n') |
| 368 | |
| 369 | |
| 370 | # Validate the metadata generated by Make for installed files and report if there is no metadata. |
| 371 | def installed_file_has_metadata(installed_file_metadata, report): |
| 372 | installed_file = installed_file_metadata['installed_file'] |
| 373 | module_path = installed_file_metadata['module_path'] |
| 374 | product_copy_files = installed_file_metadata['product_copy_files'] |
| 375 | kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] |
| 376 | is_platform_generated = installed_file_metadata['is_platform_generated'] |
| 377 | |
| 378 | if (not module_path and |
| 379 | not product_copy_files and |
| 380 | not kernel_module_copy_files and |
| 381 | not is_platform_generated and |
| 382 | not installed_file.endswith('.fsv_meta')): |
| 383 | report[ISSUE_NO_METADATA].append(installed_file) |
| 384 | return False |
| 385 | |
| 386 | return True |
| 387 | |
| 388 | |
Wei Li | 427dacb | 2023-10-18 16:45:31 -0700 | [diff] [blame] | 389 | # Validate identifiers in a package's METADATA. |
| 390 | # 1) Only known identifier type is allowed |
| 391 | # 2) Only one identifier's primary_source can be true |
| 392 | def validate_package_metadata(metadata_file_path, package_metadata): |
| 393 | primary_source_found = False |
| 394 | for identifier in package_metadata.third_party.identifier: |
| 395 | if identifier.type not in THIRD_PARTY_IDENTIFIER_TYPES: |
| 396 | sys.exit(f'Unknown value of third_party.identifier.type in {metadata_file_path}/METADATA: {identifier.type}.') |
| 397 | if primary_source_found and identifier.primary_source: |
| 398 | sys.exit( |
| 399 | f'Field "primary_source" is set to true in multiple third_party.identifier in {metadata_file_path}/METADATA.') |
| 400 | primary_source_found = identifier.primary_source |
| 401 | |
| 402 | |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 403 | def report_metadata_file(metadata_file_path, installed_file_metadata, report): |
| 404 | if metadata_file_path: |
| 405 | report[INFO_METADATA_FOUND_FOR_PACKAGE].append( |
| 406 | 'installed_file: {}, module_path: {}, METADATA file: {}'.format( |
| 407 | installed_file_metadata['installed_file'], |
| 408 | installed_file_metadata['module_path'], |
| 409 | metadata_file_path + '/METADATA')) |
| 410 | |
| 411 | package_metadata = metadata_file_pb2.Metadata() |
| 412 | with open(metadata_file_path + '/METADATA', 'rt') as f: |
| 413 | text_format.Parse(f.read(), package_metadata) |
| 414 | |
Wei Li | 427dacb | 2023-10-18 16:45:31 -0700 | [diff] [blame] | 415 | validate_package_metadata(metadata_file_path, package_metadata) |
| 416 | |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 417 | if not metadata_file_path in metadata_file_protos: |
| 418 | metadata_file_protos[metadata_file_path] = package_metadata |
| 419 | if not package_metadata.name: |
| 420 | report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"') |
| 421 | |
| 422 | if not package_metadata.third_party.version: |
| 423 | report[ISSUE_METADATA_FILE_INCOMPLETE].append( |
| 424 | f'{metadata_file_path}/METADATA does not has "third_party.version"') |
| 425 | |
| 426 | for tag in package_metadata.third_party.security.tag: |
| 427 | if not tag.startswith(NVD_CPE23): |
| 428 | report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append( |
| 429 | f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA') |
| 430 | else: |
| 431 | report[ISSUE_NO_METADATA_FILE].append( |
| 432 | "installed_file: {}, module_path: {}".format( |
| 433 | installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) |
| 434 | |
| 435 | |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 436 | def generate_sbom_for_unbundled_apk(): |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 437 | with open(args.metadata, newline='') as sbom_metadata_file: |
| 438 | reader = csv.DictReader(sbom_metadata_file) |
| 439 | doc = sbom_data.Document(name=args.build_version, |
| 440 | namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', |
| 441 | creators=['Organization: ' + args.product_mfr]) |
| 442 | for installed_file_metadata in reader: |
| 443 | installed_file = installed_file_metadata['installed_file'] |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 444 | if args.output_file != installed_file_metadata['build_output_path'] + '.spdx.json': |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 445 | continue |
| 446 | |
| 447 | module_path = installed_file_metadata['module_path'] |
| 448 | package_id = new_package_id(module_path, PKG_PREBUILT) |
| 449 | package = sbom_data.Package(id=package_id, |
| 450 | name=module_path, |
| 451 | version=args.build_version, |
| 452 | supplier='Organization: ' + args.product_mfr) |
| 453 | file_id = new_file_id(installed_file) |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 454 | file = sbom_data.File(id=file_id, |
| 455 | name=installed_file, |
| 456 | checksum=checksum(installed_file_metadata['build_output_path'])) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 457 | relationship = sbom_data.Relationship(id1=file_id, |
| 458 | relationship=sbom_data.RelationshipType.GENERATED_FROM, |
| 459 | id2=package_id) |
| 460 | doc.add_package(package) |
| 461 | doc.files.append(file) |
| 462 | doc.describes = file_id |
| 463 | doc.add_relationship(relationship) |
| 464 | doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') |
| 465 | break |
| 466 | |
Wei Li | 49af939 | 2023-04-12 17:35:26 -0700 | [diff] [blame] | 467 | with open(args.output_file, 'w', encoding='utf-8') as file: |
| 468 | sbom_writers.JSONWriter.write(doc, file) |
| 469 | fragment_file = args.output_file.removesuffix('.spdx.json') + '-fragment.spdx' |
| 470 | with open(fragment_file, 'w', encoding='utf-8') as file: |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 471 | sbom_writers.TagValueWriter.write(doc, file, fragment=True) |
| 472 | |
| 473 | |
| 474 | def main(): |
| 475 | global args |
| 476 | args = get_args() |
| 477 | log('Args:', vars(args)) |
| 478 | |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 479 | if args.unbundled_apk: |
| 480 | generate_sbom_for_unbundled_apk() |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 481 | return |
| 482 | |
| 483 | global metadata_file_protos |
| 484 | metadata_file_protos = {} |
| 485 | |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 486 | product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT, |
| 487 | name=sbom_data.PACKAGE_NAME_PRODUCT, |
Wei Li | 5290825 | 2023-04-14 18:49:42 -0700 | [diff] [blame] | 488 | download_location=sbom_data.VALUE_NONE, |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 489 | version=args.build_version, |
| 490 | supplier='Organization: ' + args.product_mfr, |
| 491 | files_analyzed=True) |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 492 | |
| 493 | doc = sbom_data.Document(name=args.build_version, |
| 494 | namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', |
| 495 | creators=['Organization: ' + args.product_mfr]) |
| 496 | if not args.unbundled_apex: |
| 497 | doc.packages.append(product_package) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 498 | |
| 499 | doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, |
| 500 | name=sbom_data.PACKAGE_NAME_PLATFORM, |
Wei Li | 5290825 | 2023-04-14 18:49:42 -0700 | [diff] [blame] | 501 | download_location=sbom_data.VALUE_NONE, |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 502 | version=args.build_version, |
| 503 | supplier='Organization: ' + args.product_mfr)) |
| 504 | |
| 505 | # Report on some issues and information |
| 506 | report = { |
| 507 | ISSUE_NO_METADATA: [], |
| 508 | ISSUE_NO_METADATA_FILE: [], |
| 509 | ISSUE_METADATA_FILE_INCOMPLETE: [], |
| 510 | ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], |
| 511 | ISSUE_INSTALLED_FILE_NOT_EXIST: [], |
| 512 | INFO_METADATA_FOUND_FOR_PACKAGE: [], |
| 513 | } |
| 514 | |
| 515 | # Scan the metadata in CSV file and create the corresponding package and file records in SPDX |
| 516 | with open(args.metadata, newline='') as sbom_metadata_file: |
| 517 | reader = csv.DictReader(sbom_metadata_file) |
| 518 | for installed_file_metadata in reader: |
| 519 | installed_file = installed_file_metadata['installed_file'] |
| 520 | module_path = installed_file_metadata['module_path'] |
| 521 | product_copy_files = installed_file_metadata['product_copy_files'] |
| 522 | kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 523 | build_output_path = installed_file_metadata['build_output_path'] |
Wei Li | d263695 | 2023-05-30 15:03:03 -0700 | [diff] [blame] | 524 | is_static_lib = installed_file_metadata['is_static_lib'] |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 525 | |
| 526 | if not installed_file_has_metadata(installed_file_metadata, report): |
| 527 | continue |
Wei Li | d263695 | 2023-05-30 15:03:03 -0700 | [diff] [blame] | 528 | if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)): |
| 529 | # Ignore non-existing static library files for now since they are not shipped on devices. |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 530 | report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file) |
| 531 | continue |
| 532 | |
| 533 | file_id = new_file_id(installed_file) |
Wei Li | d263695 | 2023-05-30 15:03:03 -0700 | [diff] [blame] | 534 | # TODO(b/285453664): Soong should report the information of statically linked libraries to Make. |
| 535 | # This happens when a different sanitized version of static libraries is used in linking. |
| 536 | # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be |
| 537 | # located correctly because Soong doesn't report the information to Make. |
| 538 | sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string |
| 539 | if os.path.islink(build_output_path) or os.path.isfile(build_output_path): |
| 540 | sha1 = checksum(build_output_path) |
| 541 | doc.files.append(sbom_data.File(id=file_id, |
| 542 | name=installed_file, |
| 543 | checksum=sha1)) |
| 544 | |
| 545 | if not is_static_lib: |
| 546 | if not args.unbundled_apex: |
| 547 | product_package.file_ids.append(file_id) |
| 548 | elif len(doc.files) > 1: |
| 549 | doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id)) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 550 | |
| 551 | if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): |
| 552 | metadata_file_path = get_metadata_file_path(installed_file_metadata) |
| 553 | report_metadata_file(metadata_file_path, installed_file_metadata, report) |
| 554 | |
| 555 | # File from source fork packages or prebuilt fork packages |
| 556 | external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path) |
| 557 | if len(pkgs) > 0: |
| 558 | if external_doc_ref: |
| 559 | doc.add_external_ref(external_doc_ref) |
| 560 | for p in pkgs: |
| 561 | doc.add_package(p) |
| 562 | for rel in rels: |
| 563 | doc.add_relationship(rel) |
| 564 | fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package |
| 565 | doc.add_relationship(sbom_data.Relationship(id1=file_id, |
| 566 | relationship=sbom_data.RelationshipType.GENERATED_FROM, |
| 567 | id2=fork_package_id)) |
| 568 | elif module_path or installed_file_metadata['is_platform_generated']: |
| 569 | # File from PLATFORM package |
| 570 | doc.add_relationship(sbom_data.Relationship(id1=file_id, |
| 571 | relationship=sbom_data.RelationshipType.GENERATED_FROM, |
| 572 | id2=sbom_data.SPDXID_PLATFORM)) |
| 573 | elif product_copy_files: |
| 574 | # Format of product_copy_files: <source path>:<dest path> |
| 575 | src_path = product_copy_files.split(':')[0] |
| 576 | # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device, |
| 577 | # so process them as files from PLATFORM package |
| 578 | doc.add_relationship(sbom_data.Relationship(id1=file_id, |
| 579 | relationship=sbom_data.RelationshipType.GENERATED_FROM, |
| 580 | id2=sbom_data.SPDXID_PLATFORM)) |
| 581 | elif installed_file.endswith('.fsv_meta'): |
| 582 | # See build/make/core/Makefile:2988 |
| 583 | doc.add_relationship(sbom_data.Relationship(id1=file_id, |
| 584 | relationship=sbom_data.RelationshipType.GENERATED_FROM, |
| 585 | id2=sbom_data.SPDXID_PLATFORM)) |
| 586 | elif kernel_module_copy_files.startswith('ANDROID-GEN'): |
| 587 | # For the four files generated for _dlkm, _ramdisk partitions |
| 588 | # See build/make/core/Makefile:323 |
| 589 | doc.add_relationship(sbom_data.Relationship(id1=file_id, |
| 590 | relationship=sbom_data.RelationshipType.GENERATED_FROM, |
| 591 | id2=sbom_data.SPDXID_PLATFORM)) |
| 592 | |
Wei Li | d263695 | 2023-05-30 15:03:03 -0700 | [diff] [blame] | 593 | # Process static libraries and whole static libraries the installed file links to |
| 594 | static_libs = installed_file_metadata['static_libraries'] |
| 595 | whole_static_libs = installed_file_metadata['whole_static_libraries'] |
| 596 | all_static_libs = (static_libs + ' ' + whole_static_libs).strip() |
| 597 | if all_static_libs: |
| 598 | for lib in all_static_libs.split(' '): |
| 599 | doc.add_relationship(sbom_data.Relationship(id1=file_id, |
| 600 | relationship=sbom_data.RelationshipType.STATIC_LINK, |
| 601 | id2=new_file_id(lib + '.a'))) |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 602 | |
| 603 | if args.unbundled_apex: |
| 604 | doc.describes = doc.files[0].id |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 605 | |
| 606 | # Save SBOM records to output file |
Wei Li | d263695 | 2023-05-30 15:03:03 -0700 | [diff] [blame] | 607 | doc.generate_packages_verification_code() |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 608 | doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 609 | prefix = args.output_file |
| 610 | if prefix.endswith('.spdx'): |
| 611 | prefix = prefix.removesuffix('.spdx') |
| 612 | elif prefix.endswith('.spdx.json'): |
| 613 | prefix = prefix.removesuffix('.spdx.json') |
| 614 | |
| 615 | output_file = prefix + '.spdx' |
| 616 | if args.unbundled_apex: |
| 617 | output_file = prefix + '-fragment.spdx' |
| 618 | with open(output_file, 'w', encoding="utf-8") as file: |
| 619 | sbom_writers.TagValueWriter.write(doc, file, fragment=args.unbundled_apex) |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 620 | if args.json: |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 621 | with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file: |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 622 | sbom_writers.JSONWriter.write(doc, file) |
| 623 | |
Wei Li | fd7e651 | 2023-05-05 10:49:28 -0700 | [diff] [blame] | 624 | save_report(prefix + '-gen-report.txt', report) |
| 625 | |
Wei Li | dec97b1 | 2023-04-07 16:45:17 -0700 | [diff] [blame] | 626 | |
| 627 | if __name__ == '__main__': |
| 628 | main() |