Merge "Add hyphenation files to layoutlib SBOM" into main
diff --git a/backported_fixes/Android.bp b/backported_fixes/Android.bp
new file mode 100644
index 0000000..a20f3fc
--- /dev/null
+++ b/backported_fixes/Android.bp
@@ -0,0 +1,115 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+ default_team: "trendy_team_android_media_reliability",
+}
+
+genrule {
+ name: "applied_backported_fixes",
+ tools: ["applied_backported_fixes_main"],
+ srcs: [":applied_backported_fix_binpbs"],
+ out: ["applied_backported_fixes.prop"],
+ cmd: "$(location applied_backported_fixes_main)" +
+ " -p $(location applied_backported_fixes.prop)" +
+ " $(in)",
+}
+
+java_library {
+ name: "backported_fixes_proto",
+ srcs: [
+ "backported_fixes.proto",
+ ],
+ host_supported: true,
+}
+
+java_library {
+ name: "backported_fixes_common",
+ srcs: ["src/java/com/android/build/backportedfixes/common/*.java"],
+ static_libs: [
+ "backported_fixes_proto",
+ "guava",
+ ],
+ host_supported: true,
+}
+
+java_test_host {
+ name: "backported_fixes_common_test",
+ srcs: ["tests/java/com/android/build/backportedfixes/common/*.java"],
+ static_libs: [
+ "backported_fixes_common",
+ "backported_fixes_proto",
+ "junit",
+ "truth",
+ "truth-liteproto-extension",
+ "truth-proto-extension",
+ ],
+ test_options: {
+ unit_test: true,
+ },
+ test_suites: ["general-tests"],
+}
+
+java_library {
+ name: "applied_backported_fixes_lib",
+ srcs: ["src/java/com/android/build/backportedfixes/*.java"],
+ static_libs: [
+ "backported_fixes_common",
+ "backported_fixes_proto",
+ "jcommander",
+ "guava",
+ ],
+ host_supported: true,
+}
+
+java_binary_host {
+ name: "applied_backported_fixes_main",
+ main_class: "com.android.build.backportedfixes.Main",
+ static_libs: [
+ "applied_backported_fixes_lib",
+ ],
+}
+
+java_test_host {
+ name: "applied_backported_fixes_test",
+ srcs: ["tests/java/com/android/build/backportedfixes/*.java"],
+ static_libs: [
+ "applied_backported_fixes_lib",
+ "backported_fixes_proto",
+ "junit",
+ "truth",
+ ],
+ test_options: {
+ unit_test: true,
+ },
+ test_suites: ["general-tests"],
+}
+
+gensrcs {
+ name: "applied_backported_fix_binpbs",
+ tools: ["aprotoc"],
+ srcs: [
+ "applied_fixes/*.txtpb",
+ ],
+ tool_files: [
+ "backported_fixes.proto",
+ ],
+ output_extension: "binpb",
+ cmd: "$(location aprotoc) " +
+ " --encode=com.android.build.backportedfixes.BackportedFix" +
+ " $(location backported_fixes.proto)" +
+ " < $(in)" +
+ " > $(out); echo $(out)",
+}
diff --git a/backported_fixes/OWNERS b/backported_fixes/OWNERS
new file mode 100644
index 0000000..ac176bf
--- /dev/null
+++ b/backported_fixes/OWNERS
@@ -0,0 +1,3 @@
+essick@google.com
+nchalko@google.com
+portmannc@google.com
diff --git a/backported_fixes/applied_fixes/ki350037023.txtpb b/backported_fixes/applied_fixes/ki350037023.txtpb
new file mode 100644
index 0000000..456a7ae
--- /dev/null
+++ b/backported_fixes/applied_fixes/ki350037023.txtpb
@@ -0,0 +1,19 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# proto-file: ../backported_fixes.proto
+# proto-message: BackportedFix
+
+known_issue: 350037023
+alias: 1
diff --git a/backported_fixes/backported_fixes.proto b/backported_fixes/backported_fixes.proto
new file mode 100644
index 0000000..91618ee
--- /dev/null
+++ b/backported_fixes/backported_fixes.proto
@@ -0,0 +1,37 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package com.android.build.backportedfixes;
+
+option java_multiple_files = true;
+
+// A list of backported fixes.
+message BackportedFixes {
+ repeated BackportedFix fixes = 1;
+}
+
+// A known issue approved for reporting Build.getBackportedFixStatus
+message BackportedFix {
+
+ // The issue id from the public bug tracker
+ // https://issuetracker.google.com/issues/{known_issue}
+ optional int64 known_issue = 1;
+ // The alias for the known issue.
+ // 1 - 1023 are valid aliases
+ // Must be unique across all backported fixes.
+ optional int32 alias = 2;
+}
+
diff --git a/backported_fixes/src/java/com/android/build/backportedfixes/Main.java b/backported_fixes/src/java/com/android/build/backportedfixes/Main.java
new file mode 100644
index 0000000..79148cc
--- /dev/null
+++ b/backported_fixes/src/java/com/android/build/backportedfixes/Main.java
@@ -0,0 +1,79 @@
+
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.build.backportedfixes;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.android.build.backportedfixes.common.ClosableCollection;
+import com.android.build.backportedfixes.common.Parser;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.converters.FileConverter;
+import com.google.common.io.Files;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public final class Main {
+ @Parameter(description = "BackportedFix proto binary files", converter = FileConverter.class,
+ required = true)
+ List<File> fixFiles;
+ @Parameter(description = "The file to write the property value to.",
+ names = {"--property_file", "-p"}, converter = FileConverter.class, required = true)
+ File propertyFile;
+
+ public static void main(String... argv) throws Exception {
+ Main main = new Main();
+ JCommander.newBuilder().addObject(main).build().parse(argv);
+ main.run();
+ }
+
+ Main() {
+ }
+
+ private void run() throws Exception {
+ try (var fixStreams = ClosableCollection.wrap(Parser.getFileInputStreams(fixFiles));
+ var out = Files.newWriter(propertyFile, UTF_8)) {
+ var fixes = Parser.parseBackportedFixes(fixStreams.getCollection());
+ writeFixesAsAliasBitSet(fixes, out);
+ }
+ }
+
+ static void writeFixesAsAliasBitSet(BackportedFixes fixes, Writer out) {
+ PrintWriter printWriter = new PrintWriter(out);
+ printWriter.println("# The following backported fixes have been applied");
+ for (var f : fixes.getFixesList()) {
+ printWriter.printf("# https://issuetracker.google.com/issues/%d with alias %d",
+ f.getKnownIssue(), f.getAlias());
+ printWriter.println();
+ }
+ var bsArray = Parser.getBitSetArray(
+ fixes.getFixesList().stream().mapToInt(BackportedFix::getAlias).toArray());
+ String bsString = Arrays.stream(bsArray).mapToObj(Long::toString).collect(
+ Collectors.joining(","));
+ printWriter.printf("ro.build.backported_fixes.alias_bitset.long_list=%s", bsString);
+ printWriter.println();
+ if (printWriter.checkError()) {
+ throw new RuntimeException("There was an error writing to " + out.toString());
+ }
+ }
+}
diff --git a/backported_fixes/src/java/com/android/build/backportedfixes/common/ClosableCollection.java b/backported_fixes/src/java/com/android/build/backportedfixes/common/ClosableCollection.java
new file mode 100644
index 0000000..75b6730
--- /dev/null
+++ b/backported_fixes/src/java/com/android/build/backportedfixes/common/ClosableCollection.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.build.backportedfixes.common;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/** An AutoCloseable holder for a collection of AutoCloseables. */
+public final class ClosableCollection<T extends AutoCloseable, C extends Collection<T>> implements
+ AutoCloseable {
+ C source;
+
+ /** Makes the collection AutoCloseable. */
+ public static <T extends AutoCloseable, C extends Collection<T>> ClosableCollection<T, C> wrap(
+ C source) {
+ return new ClosableCollection<>(source);
+ }
+
+ private ClosableCollection(C source) {
+ this.source = source;
+ }
+
+ /** Get the source collection. */
+ public C getCollection() {
+ return source;
+ }
+
+ /**
+ * Closes each item in the collection.
+ *
+ * @throws Exception if any close throws an an exception, a new exception is thrown with
+ * all the exceptions thrown closing the streams added as a suppressed
+ * exceptions.
+ */
+ @Override
+ public void close() throws Exception {
+ var failures = new ArrayList<Exception>();
+ for (T t : source) {
+ try {
+ t.close();
+ } catch (Exception e) {
+ failures.add(e);
+ }
+ }
+ if (!failures.isEmpty()) {
+ Exception e = new Exception(
+ "%d of %d failed while closing".formatted(failures.size(), source.size()));
+ failures.forEach(e::addSuppressed);
+ throw e;
+ }
+ }
+}
diff --git a/backported_fixes/src/java/com/android/build/backportedfixes/common/Parser.java b/backported_fixes/src/java/com/android/build/backportedfixes/common/Parser.java
new file mode 100644
index 0000000..6b08b8f
--- /dev/null
+++ b/backported_fixes/src/java/com/android/build/backportedfixes/common/Parser.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.build.backportedfixes.common;
+
+import com.android.build.backportedfixes.BackportedFix;
+import com.android.build.backportedfixes.BackportedFixes;
+
+import com.google.common.collect.ImmutableList;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.BitSet;
+import java.util.List;
+
+
+/** Static utilities for working with {@link BackportedFixes}. */
+public final class Parser {
+
+ /** Creates list of FileInputStreams for a list of files. */
+ public static ImmutableList<FileInputStream> getFileInputStreams(List<File> fixFiles) throws
+ FileNotFoundException {
+ var streams = ImmutableList.<FileInputStream>builder();
+ for (var f : fixFiles) {
+ streams.add(new FileInputStream(f));
+ }
+ return streams.build();
+ }
+
+ /** Converts a list of backported fix aliases into a long array representing a {@link BitSet} */
+ public static long[] getBitSetArray(int[] aliases) {
+ BitSet bs = new BitSet();
+ for (int a : aliases) {
+ bs.set(a);
+ }
+ return bs.toLongArray();
+ }
+
+ /**
+ * Creates a {@link BackportedFixes} from a list of {@link BackportedFix} binary proto streams.
+ */
+ public static BackportedFixes parseBackportedFixes(List<? extends InputStream> fixStreams)
+ throws
+ IOException {
+ var fixes = BackportedFixes.newBuilder();
+ for (var s : fixStreams) {
+ BackportedFix fix = BackportedFix.parseFrom(s);
+ fixes.addFixes(fix);
+ s.close();
+ }
+ return fixes.build();
+ }
+
+ private Parser() {
+ }
+}
diff --git a/backported_fixes/tests/java/com/android/build/backportedfixes/MainTest.java b/backported_fixes/tests/java/com/android/build/backportedfixes/MainTest.java
new file mode 100644
index 0000000..84061e1
--- /dev/null
+++ b/backported_fixes/tests/java/com/android/build/backportedfixes/MainTest.java
@@ -0,0 +1,64 @@
+
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.build.backportedfixes;
+
+import com.google.common.truth.Truth;
+
+import org.junit.Test;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/** Tests for {@link Main}. */
+public class MainTest {
+
+
+ @Test
+ public void writeFixesAsAliasBitSet_default() {
+ BackportedFixes fixes = BackportedFixes.newBuilder().build();
+ var result = new StringWriter();
+
+ Main.writeFixesAsAliasBitSet(fixes, new PrintWriter(result));
+
+ Truth.assertThat(result.toString())
+ .isEqualTo("""
+ # The following backported fixes have been applied
+ ro.build.backported_fixes.alias_bitset.long_list=
+ """);
+ }
+
+ @Test
+ public void writeFixesAsAliasBitSet_some() {
+ BackportedFixes fixes = BackportedFixes.newBuilder()
+ .addFixes(BackportedFix.newBuilder().setKnownIssue(1234L).setAlias(1))
+ .addFixes(BackportedFix.newBuilder().setKnownIssue(3L).setAlias(65))
+ .addFixes(BackportedFix.newBuilder().setKnownIssue(4L).setAlias(67))
+ .build();
+ var result = new StringWriter();
+
+ Main.writeFixesAsAliasBitSet(fixes, new PrintWriter(result));
+
+ Truth.assertThat(result.toString())
+ .isEqualTo("""
+ # The following backported fixes have been applied
+ # https://issuetracker.google.com/issues/1234 with alias 1
+ # https://issuetracker.google.com/issues/3 with alias 65
+ # https://issuetracker.google.com/issues/4 with alias 67
+ ro.build.backported_fixes.alias_bitset.long_list=2,10
+ """);
+ }
+}
diff --git a/backported_fixes/tests/java/com/android/build/backportedfixes/common/CloseableCollectionTest.java b/backported_fixes/tests/java/com/android/build/backportedfixes/common/CloseableCollectionTest.java
new file mode 100644
index 0000000..d3d84a8
--- /dev/null
+++ b/backported_fixes/tests/java/com/android/build/backportedfixes/common/CloseableCollectionTest.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.build.backportedfixes.common;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.truth.Correspondence;
+import com.google.common.truth.Truth;
+
+import org.junit.Test;
+
+/** Tests for {@link ClosableCollection}. */
+public class CloseableCollectionTest {
+
+ private static class FakeCloseable implements AutoCloseable {
+ private final boolean throwOnClose;
+ private final String name;
+
+
+ private boolean isClosed = false;
+
+ private FakeCloseable(String name, boolean throwOnClose) {
+ this.name = name;
+ this.throwOnClose = throwOnClose;
+
+ }
+
+ private static FakeCloseable named(String name) {
+ return new FakeCloseable(name, false);
+ }
+
+ private static FakeCloseable failing(String name) {
+ return new FakeCloseable(name, true);
+ }
+
+ public boolean isClosed() {
+ return isClosed;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (throwOnClose) {
+ throw new Exception(name + " close failed");
+ }
+ isClosed = true;
+ }
+ }
+
+
+ @Test
+ public void bothClosed() throws Exception {
+ var c = ImmutableSet.of(FakeCloseable.named("foo"), FakeCloseable.named("bar"));
+ try (var cc = ClosableCollection.wrap(c);) {
+ Truth.assertThat(cc.getCollection()).isSameInstanceAs(c);
+ }
+ Truth.assertThat(c)
+ .comparingElementsUsing(
+ Correspondence.transforming(FakeCloseable::isClosed, "is closed"))
+ .containsExactly(true, true);
+ }
+
+ @Test
+ public void bothFailed() {
+ var c = ImmutableSet.of(FakeCloseable.failing("foo"), FakeCloseable.failing("bar"));
+
+ try {
+ try (var cc = ClosableCollection.wrap(c);) {
+ Truth.assertThat(cc.getCollection()).isSameInstanceAs(c);
+ }
+ } catch (Exception e) {
+ Truth.assertThat(e).hasMessageThat().isEqualTo("2 of 2 failed while closing");
+ Truth.assertThat(e.getSuppressed())
+ .asList()
+ .comparingElementsUsing(
+ Correspondence.transforming(Exception::getMessage, "has a message of "))
+ .containsExactly("foo close failed", "bar close failed");
+ }
+ }
+}
diff --git a/backported_fixes/tests/java/com/android/build/backportedfixes/common/ParserTest.java b/backported_fixes/tests/java/com/android/build/backportedfixes/common/ParserTest.java
new file mode 100644
index 0000000..444e694
--- /dev/null
+++ b/backported_fixes/tests/java/com/android/build/backportedfixes/common/ParserTest.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.build.backportedfixes.common;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat;
+
+import com.android.build.backportedfixes.BackportedFix;
+import com.android.build.backportedfixes.BackportedFixes;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+
+/** Tests for {@link Parser}.*/
+public class ParserTest {
+
+ @Test
+ public void getFileInputStreams() throws IOException {
+ var results = Parser.getFileInputStreams(
+ ImmutableList.of(Files.createTempFile("test", null).toFile()));
+ assertThat(results).isNotEmpty();
+ }
+
+
+ @Test
+ public void getBitSetArray_empty() {
+ var results = Parser.getBitSetArray(new int[]{});
+ assertThat(results).isEmpty();
+ }
+
+ @Test
+ public void getBitSetArray_2_3_64() {
+ var results = Parser.getBitSetArray(new int[]{2,3,64});
+ assertThat(results).asList().containsExactly(12L,1L).inOrder();
+ }
+
+ @Test
+ public void parseBackportedFixes_empty() throws IOException {
+ var result = Parser.parseBackportedFixes(ImmutableList.of());
+ assertThat(result).isEqualTo(BackportedFixes.getDefaultInstance());
+ }
+
+ @Test
+ public void parseBackportedFixes_oneBlank() throws IOException {
+ var result = Parser.parseBackportedFixes(
+ ImmutableList.of(inputStream(BackportedFix.getDefaultInstance())));
+
+ assertThat(result).isEqualTo(
+ BackportedFixes.newBuilder()
+ .addFixes(BackportedFix.getDefaultInstance())
+ .build());
+ }
+
+ @Test
+ public void parseBackportedFixes_two() throws IOException {
+ BackportedFix ki123 = BackportedFix.newBuilder()
+ .setKnownIssue(123)
+ .setAlias(1)
+ .build();
+ BackportedFix ki456 = BackportedFix.newBuilder()
+ .setKnownIssue(456)
+ .setAlias(2)
+ .build();
+ var result = Parser.parseBackportedFixes(
+ ImmutableList.of(inputStream(ki123), inputStream(ki456)));
+ assertThat(result).isEqualTo(
+ BackportedFixes.newBuilder()
+ .addFixes(ki123)
+ .addFixes(ki456)
+ .build());
+ }
+
+ private static ByteArrayInputStream inputStream(BackportedFix f) {
+ return new ByteArrayInputStream(f.toByteArray());
+ }
+}
diff --git a/ci/Android.bp b/ci/Android.bp
index 2407c51..3f28be4 100644
--- a/ci/Android.bp
+++ b/ci/Android.bp
@@ -103,13 +103,10 @@
"test_mapping_module_retriever.py",
"build_context.py",
"test_discovery_agent.py",
+ "metrics_agent.py",
+ "buildbot.py",
],
main: "build_test_suites.py",
- version: {
- py3: {
- embedded_launcher: true,
- },
- },
libs: [
"soong-metrics-proto-py",
],
@@ -123,6 +120,11 @@
"test_mapping_module_retriever.py",
"build_context.py",
"test_discovery_agent.py",
+ "metrics_agent.py",
+ "buildbot.py",
+ ],
+ libs: [
+ "soong-metrics-proto-py",
],
}
diff --git a/ci/build_context.py b/ci/build_context.py
index cc48d53..c7a1def 100644
--- a/ci/build_context.py
+++ b/ci/build_context.py
@@ -47,6 +47,9 @@
self.is_test_mapping = False
self.test_mapping_test_groups = set()
self.file_download_options = set()
+ self.name = test_info_dict.get('name')
+ self.command = test_info_dict.get('command')
+ self.extra_options = test_info_dict.get('extraOptions')
for opt in test_info_dict.get('extraOptions', []):
key = opt.get('key')
if key == 'test-mapping-test-group':
diff --git a/ci/build_test_suites b/ci/build_test_suites
index 5aaf2f4..9d11268 100755
--- a/ci/build_test_suites
+++ b/ci/build_test_suites
@@ -1,4 +1,4 @@
-#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B
+#!/usr/bin/env bash
#
# Copyright 2024, The Android Open Source Project
#
@@ -14,7 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import build_test_suites
-import sys
-
-build_test_suites.main(sys.argv[1:])
+build/soong/soong_ui.bash --make-mode build_test_suites || exit $?
+$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ || exit $?
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
index 9e56a20..cd9d76d 100644
--- a/ci/build_test_suites.py
+++ b/ci/build_test_suites.py
@@ -20,11 +20,14 @@
import logging
import os
import pathlib
+import re
import subprocess
import sys
from typing import Callable
from build_context import BuildContext
import optimized_targets
+import metrics_agent
+import test_discovery_agent
REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP'])
@@ -70,7 +73,24 @@
build_targets = set()
packaging_commands_getters = []
+ test_discovery_zip_regexes = set()
+ optimization_rationale = ''
+ try:
+ # Do not use these regexes for now, only run this to collect data on what
+ # would be optimized.
+ test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()
+ logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')
+ except test_discovery_agent.TestDiscoveryError as e:
+ optimization_rationale = e.message
+ logging.warning(f'Unable to perform test discovery: {optimization_rationale}')
for target in self.args.extra_targets:
+ if optimization_rationale:
+ get_metrics_agent().report_unoptimized_target(target, optimization_rationale)
+ else:
+ regex = r'\b(%s)\b' % re.escape(target)
+ if any(re.search(regex, opt) for opt in test_discovery_zip_regexes):
+ get_metrics_agent().report_optimized_target(target)
+
if self._unused_target_exclusion_enabled(
target
) and not self.build_context.build_target_used(target):
@@ -97,6 +117,34 @@
in self.build_context.enabled_build_features
)
+ def _get_test_discovery_zip_regexes(self) -> set[str]:
+ build_target_regexes = set()
+ for test_info in self.build_context.test_infos:
+ tf_command = self._build_tf_command(test_info)
+ discovery_agent = test_discovery_agent.TestDiscoveryAgent(tradefed_args=tf_command)
+ for regex in discovery_agent.discover_test_zip_regexes():
+ build_target_regexes.add(regex)
+ return build_target_regexes
+
+
+ def _build_tf_command(self, test_info) -> list[str]:
+ command = [test_info.command]
+ for extra_option in test_info.extra_options:
+ if not extra_option.get('key'):
+ continue
+ arg_key = '--' + extra_option.get('key')
+ if arg_key == '--build-id':
+ command.append(arg_key)
+ command.append(os.environ.get('BUILD_NUMBER'))
+ continue
+ if extra_option.get('values'):
+ for value in extra_option.get('values'):
+ command.append(arg_key)
+ command.append(value)
+ else:
+ command.append(arg_key)
+
+ return command
@dataclass(frozen=True)
class BuildPlan:
@@ -113,19 +161,27 @@
Returns:
The exit code of the build.
"""
- args = parse_args(argv)
- check_required_env()
- build_context = BuildContext(load_build_context())
- build_planner = BuildPlanner(
- build_context, args, optimized_targets.OPTIMIZED_BUILD_TARGETS
- )
- build_plan = build_planner.create_build_plan()
+ get_metrics_agent().analysis_start()
+ try:
+ args = parse_args(argv)
+ check_required_env()
+ build_context = BuildContext(load_build_context())
+ build_planner = BuildPlanner(
+ build_context, args, optimized_targets.OPTIMIZED_BUILD_TARGETS
+ )
+ build_plan = build_planner.create_build_plan()
+ except:
+ raise
+ finally:
+ get_metrics_agent().analysis_end()
try:
execute_build_plan(build_plan)
except BuildFailureError as e:
logging.error('Build command failed! Check build_log for details.')
return e.return_code
+ finally:
+ get_metrics_agent().end_reporting()
return 0
@@ -183,12 +239,15 @@
except subprocess.CalledProcessError as e:
raise BuildFailureError(e.returncode) from e
- for packaging_commands_getter in build_plan.packaging_commands_getters:
- try:
+ get_metrics_agent().packaging_start()
+ try:
+ for packaging_commands_getter in build_plan.packaging_commands_getters:
for packaging_command in packaging_commands_getter():
run_command(packaging_command)
- except subprocess.CalledProcessError as e:
- raise BuildFailureError(e.returncode) from e
+ except subprocess.CalledProcessError as e:
+ raise BuildFailureError(e.returncode) from e
+ finally:
+ get_metrics_agent().packaging_end()
def get_top() -> pathlib.Path:
@@ -199,6 +258,10 @@
subprocess.run(args=args, check=True, stdout=stdout)
+def get_metrics_agent():
+ return metrics_agent.MetricsAgent.instance()
+
+
def main(argv):
dist_dir = os.environ.get('DIST_DIR')
if dist_dir:
diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py
index 2afaab7..26f4316 100644
--- a/ci/build_test_suites_test.py
+++ b/ci/build_test_suites_test.py
@@ -37,6 +37,8 @@
import ci_test_lib
import optimized_targets
from pyfakefs import fake_filesystem_unittest
+import metrics_agent
+import test_discovery_agent
class BuildTestSuitesTest(fake_filesystem_unittest.TestCase):
@@ -52,6 +54,10 @@
self.addCleanup(subprocess_run_patcher.stop)
self.mock_subprocess_run = subprocess_run_patcher.start()
+ metrics_agent_finalize_patcher = mock.patch('metrics_agent.MetricsAgent.end_reporting')
+ self.addCleanup(metrics_agent_finalize_patcher.stop)
+ self.mock_metrics_agent_end = metrics_agent_finalize_patcher.start()
+
self._setup_working_build_env()
def test_missing_target_release_env_var_raises(self):
@@ -256,6 +262,12 @@
def get_enabled_flag(self):
return f'{self.target}_enabled'
+ def setUp(self):
+ test_discovery_agent_patcher = mock.patch('test_discovery_agent.TestDiscoveryAgent.discover_test_zip_regexes')
+ self.addCleanup(test_discovery_agent_patcher.stop)
+ self.mock_test_discovery_agent_end = test_discovery_agent_patcher.start()
+
+
def test_build_optimization_off_builds_everything(self):
build_targets = {'target_1', 'target_2'}
build_planner = self.create_build_planner(
diff --git a/ci/metrics_agent.py b/ci/metrics_agent.py
new file mode 100644
index 0000000..bc2479e
--- /dev/null
+++ b/ci/metrics_agent.py
@@ -0,0 +1,116 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MetricsAgent is a singleton class that collects metrics for optimized build."""
+
+from enum import Enum
+import time
+import metrics_pb2
+import os
+import logging
+
+
+class MetricsAgent:
+ _SOONG_METRICS_PATH = 'logs/soong_metrics'
+ _DIST_DIR = 'DIST_DIR'
+ _instance = None
+
+ def __init__(self):
+ raise RuntimeError(
+ 'MetricsAgent cannot be instantialized, use instance() instead'
+ )
+
+ @classmethod
+ def instance(cls):
+ if not cls._instance:
+ cls._instance = cls.__new__(cls)
+ cls._instance._proto = metrics_pb2.OptimizedBuildMetrics()
+ cls._instance._init_proto()
+ cls._instance._target_results = dict()
+
+ return cls._instance
+
+ def _init_proto(self):
+ self._proto.analysis_perf.name = 'Optimized build analysis time.'
+ self._proto.packaging_perf.name = 'Optimized build total packaging time.'
+
+ def analysis_start(self):
+ self._proto.analysis_perf.start_time = time.time_ns()
+
+ def analysis_end(self):
+ self._proto.analysis_perf.real_time = (
+ time.time_ns() - self._proto.analysis_perf.start_time
+ )
+
+ def packaging_start(self):
+ self._proto.packaging_perf.start_time = time.time_ns()
+
+ def packaging_end(self):
+ self._proto.packaging_perf.real_time = (
+ time.time_ns() - self._proto.packaging_perf.start_time
+ )
+
+ def report_optimized_target(self, name: str):
+ target_result = metrics_pb2.OptimizedBuildMetrics.TargetOptimizationResult()
+ target_result.name = name
+ target_result.optimized = True
+ self._target_results[name] = target_result
+
+ def report_unoptimized_target(self, name: str, optimization_rationale: str):
+ target_result = metrics_pb2.OptimizedBuildMetrics.TargetOptimizationResult()
+ target_result.name = name
+ target_result.optimization_rationale = optimization_rationale
+ target_result.optimized = False
+ self._target_results[name] = target_result
+
+ def target_packaging_start(self, name: str):
+ target_result = self._target_results.get(name)
+ target_result.packaging_perf.start_time = time.time_ns()
+ self._target_results[name] = target_result
+
+ def target_packaging_end(self, name: str):
+ target_result = self._target_results.get(name)
+ target_result.packaging_perf.real_time = (
+ time.time_ns() - target_result.packaging_perf.start_time
+ )
+
+ def add_target_artifact(
+ self,
+ target_name: str,
+ artifact_name: str,
+ size: int,
+ included_modules: set[str],
+ ):
+ target_result = self.target_results.get(target_name)
+ artifact = (
+ metrics_pb2.OptimizedBuildMetrics.TargetOptimizationResult.OutputArtifact()
+ )
+ artifact.name = artifact_name
+ artifact.size = size
+ for module in included_modules:
+ artifact.included_modules.add(module)
+ target_result.output_artifacts.add(artifact)
+
+ def end_reporting(self):
+ for target_result in self._target_results.values():
+ self._proto.target_result.append(target_result)
+ soong_metrics_proto = metrics_pb2.MetricsBase()
+ # Read in existing metrics that should have been written out by the soong
+ # build command so that we don't overwrite them.
+ with open(os.path.join(os.environ[self._DIST_DIR], self._SOONG_METRICS_PATH), 'rb') as f:
+ soong_metrics_proto.ParseFromString(f.read())
+ soong_metrics_proto.optimized_build_metrics.CopyFrom(self._proto)
+ logging.info(soong_metrics_proto)
+ with open(os.path.join(os.environ[self._DIST_DIR], self._SOONG_METRICS_PATH), 'wb') as f:
+ f.write(soong_metrics_proto.SerializeToString())
diff --git a/ci/test_discovery_agent.py b/ci/test_discovery_agent.py
index 4eed28d..008ee47 100644
--- a/ci/test_discovery_agent.py
+++ b/ci/test_discovery_agent.py
@@ -17,7 +17,6 @@
import logging
import os
import subprocess
-import buildbot
class TestDiscoveryAgent:
@@ -50,7 +49,7 @@
A list of test zip regexes that TF is going to try to pull files from.
"""
test_discovery_output_file_name = os.path.join(
- buildbot.OutDir(), self._TRADEFED_DISCOVERY_OUTPUT_FILE_NAME
+ os.environ.get('TOP'), 'out', self._TRADEFED_DISCOVERY_OUTPUT_FILE_NAME
)
with open(
test_discovery_output_file_name, mode="w+t"
diff --git a/core/main.mk b/core/main.mk
index f96cf04..624df49 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -1399,6 +1399,7 @@
$(INSTALLED_RAMDISK_TARGET) \
$(INSTALLED_BOOTIMAGE_TARGET) \
$(INSTALLED_INIT_BOOT_IMAGE_TARGET) \
+ $(INSTALLED_DTBOIMAGE_TARGET) \
$(INSTALLED_RADIOIMAGE_TARGET) \
$(INSTALLED_DEBUG_RAMDISK_TARGET) \
$(INSTALLED_DEBUG_BOOTIMAGE_TARGET) \
diff --git a/core/soong_config.mk b/core/soong_config.mk
index ee6a9f6..0f82b68 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -397,6 +397,10 @@
$(call add_json_str, BoardSquashfsCompressor, $(BOARD_$(image_type)IMAGE_SQUASHFS_COMPRESSOR)) \
$(call add_json_str, BoardSquashfsCompressorOpt, $(BOARD_$(image_type)IMAGE_SQUASHFS_COMPRESSOR_OPT)) \
$(call add_json_str, BoardSquashfsDisable4kAlign, $(BOARD_$(image_type)IMAGE_SQUASHFS_DISABLE_4K_ALIGN)) \
+ $(call add_json_str, BoardAvbKeyPath, $(BOARD_AVB_$(image_type)_KEY_PATH)) \
+ $(call add_json_str, BoardAvbAlgorithm, $(BOARD_AVB_$(image_type)_ALGORITHM)) \
+ $(call add_json_str, BoardAvbRollbackIndex, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX)) \
+ $(call add_json_str, BoardAvbRollbackIndexLocation, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX_LOCATION)) \
$(call add_json_str, ProductBaseFsPath, $(PRODUCT_$(image_type)_BASE_FS_PATH)) \
$(call add_json_str, ProductHeadroom, $(PRODUCT_$(image_type)_HEADROOM)) \
$(call add_json_str, ProductVerityPartition, $(PRODUCT_$(image_type)_VERITY_PARTITION)) \
diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py
index c25ff27..30a6acc 100644
--- a/tools/releasetools/add_img_to_target_files.py
+++ b/tools/releasetools/add_img_to_target_files.py
@@ -1100,7 +1100,7 @@
vbmeta_partitions = common.AVB_PARTITIONS[:] + tuple(avb_custom_partitions)
vbmeta_system = OPTIONS.info_dict.get("avb_vbmeta_system", "").strip()
- if vbmeta_system:
+ if vbmeta_system and set(vbmeta_system.split()).intersection(partitions):
banner("vbmeta_system")
partitions["vbmeta_system"] = AddVBMeta(
output_zip, partitions, "vbmeta_system", vbmeta_system.split())
@@ -1110,7 +1110,7 @@
vbmeta_partitions.append("vbmeta_system")
vbmeta_vendor = OPTIONS.info_dict.get("avb_vbmeta_vendor", "").strip()
- if vbmeta_vendor:
+ if vbmeta_vendor and set(vbmeta_vendor.split()).intersection(partitions):
banner("vbmeta_vendor")
partitions["vbmeta_vendor"] = AddVBMeta(
output_zip, partitions, "vbmeta_vendor", vbmeta_vendor.split())
@@ -1137,7 +1137,7 @@
if item not in included_partitions]
vbmeta_partitions.append(partition_name)
- if OPTIONS.info_dict.get("avb_building_vbmeta_image") == "true":
+ if OPTIONS.info_dict.get("avb_building_vbmeta_image") == "true" and set(vbmeta_partitions).intersection(partitions):
banner("vbmeta")
AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions)