Add bloaty_merger
This script will be used to merge the csv outputs from bloaty into a
single proto file.
Bug: 172339795
Test: atest --host bloaty_merger_test
Change-Id: I9a3c12d7b0c4b5e347fec04e6758bcc1fbc8ab40
diff --git a/bloaty/Android.bp b/bloaty/Android.bp
new file mode 100644
index 0000000..0c4f603
--- /dev/null
+++ b/bloaty/Android.bp
@@ -0,0 +1,27 @@
+python_test_host {
+ name: "bloaty_merger_test",
+ srcs: [
+ "bloaty_merger_test.py",
+ "bloaty_merger.py",
+ "file_sections.proto",
+ ],
+ proto: {
+ canonical_path_from_root: false,
+ },
+ libs: [
+ "pyfakefs",
+ "ninja_rsp",
+ ],
+}
+
+python_binary_host {
+ name: "bloaty_merger",
+ srcs: [
+ "bloaty_merger.py",
+ "file_sections.proto",
+ ],
+ proto: {
+ canonical_path_from_root: false,
+ },
+ libs: ["ninja_rsp"],
+}
diff --git a/bloaty/bloaty_merger.py b/bloaty/bloaty_merger.py
new file mode 100644
index 0000000..c873fb8
--- /dev/null
+++ b/bloaty/bloaty_merger.py
@@ -0,0 +1,79 @@
+# Copyright 2021 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Bloaty CSV Merger
+
+Merges a list of .csv files from Bloaty into a protobuf. It takes the list as
+a first argument and the output as second. For instance:
+
+ $ bloaty_merger binary_sizes.lst binary_sizes.pb
+
+"""
+
+import argparse
+import csv
+
+import ninja_rsp
+
+import file_sections_pb2
+
+BLOATY_EXTENSION = ".bloaty.csv"
+
+def parse_csv(path):
+ """Parses a Bloaty-generated CSV file into a protobuf.
+
+ Args:
+ path: The filepath to the CSV file, relative to $ANDROID_TOP.
+
+ Returns:
+ A file_sections_pb2.File if the file was found; None otherwise.
+ """
+ file_proto = None
+ with open(path, newline='') as csv_file:
+ file_proto = file_sections_pb2.File()
+ if path.endswith(BLOATY_EXTENSION):
+ file_proto.path = path[:-len(BLOATY_EXTENSION)]
+ section_reader = csv.DictReader(csv_file)
+ for row in section_reader:
+ section = file_proto.sections.add()
+ section.name = row["sections"]
+ section.vm_size = int(row["vmsize"])
+ section.file_size = int(row["filesize"])
+ return file_proto
+
+def create_file_size_metrics(input_list, output_proto):
+ """Creates a FileSizeMetrics proto from a list of CSV files.
+
+ Args:
+ input_list: The path to the file which contains the list of CSV files. Each
+ filepath is separated by a space.
+ output_proto: The path for the output protobuf.
+ """
+ metrics = file_sections_pb2.FileSizeMetrics()
+ reader = ninja_rsp.NinjaRspFileReader(input_list)
+ for csv_path in reader:
+ file_proto = parse_csv(csv_path)
+ if file_proto:
+ metrics.files.append(file_proto)
+ with open(output_proto, "wb") as output:
+ output.write(metrics.SerializeToString())
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("input_list_file", help="List of bloaty csv files.")
+ parser.add_argument("output_proto", help="Output proto.")
+ args = parser.parse_args()
+ create_file_size_metrics(args.input_list_file, args.output_proto)
+
+if __name__ == '__main__':
+ main()
diff --git a/bloaty/bloaty_merger_test.py b/bloaty/bloaty_merger_test.py
new file mode 100644
index 0000000..0e3641d
--- /dev/null
+++ b/bloaty/bloaty_merger_test.py
@@ -0,0 +1,65 @@
+# Copyright 2021 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+from pyfakefs import fake_filesystem_unittest
+
+import bloaty_merger
+import file_sections_pb2
+
+
+class BloatyMergerTestCase(fake_filesystem_unittest.TestCase):
+ def setUp(self):
+ self.setUpPyfakefs()
+
+ def test_parse_csv(self):
+ csv_content = "sections,vmsize,filesize\nsection1,2,3\n"
+ self.fs.create_file("file1.bloaty.csv", contents=csv_content)
+ pb = bloaty_merger.parse_csv("file1.bloaty.csv")
+ self.assertEqual(pb.path, "file1")
+ self.assertEqual(len(pb.sections), 1)
+ s = pb.sections[0]
+ self.assertEqual(s.name, "section1")
+ self.assertEqual(s.vm_size, 2)
+ self.assertEqual(s.file_size, 3)
+
+ def test_missing_file(self):
+ with self.assertRaises(FileNotFoundError):
+ bloaty_merger.parse_csv("missing.bloaty.csv")
+
+ def test_malformed_csv(self):
+ csv_content = "header1,heaVder2,header3\n4,5,6\n"
+ self.fs.create_file("file1.bloaty.csv", contents=csv_content)
+ with self.assertRaises(KeyError):
+ bloaty_merger.parse_csv("file1.bloaty.csv")
+
+ def test_create_file_metrics(self):
+ file_list = "file1.bloaty.csv file2.bloaty.csv"
+ file1_content = "sections,vmsize,filesize\nsection1,2,3\nsection2,7,8"
+ file2_content = "sections,vmsize,filesize\nsection1,4,5\n"
+
+ self.fs.create_file("files.lst", contents=file_list)
+ self.fs.create_file("file1.bloaty.csv", contents=file1_content)
+ self.fs.create_file("file2.bloaty.csv", contents=file2_content)
+
+ bloaty_merger.create_file_size_metrics("files.lst", "output.pb")
+
+ metrics = file_sections_pb2.FileSizeMetrics()
+ with open("output.pb", "rb") as output:
+ metrics.ParseFromString(output.read())
+
+
+if __name__ == '__main__':
+ suite = unittest.TestLoader().loadTestsFromTestCase(BloatyMergerTestCase)
+ unittest.TextTestRunner(verbosity=2).run(suite)
diff --git a/bloaty/file_sections.proto b/bloaty/file_sections.proto
new file mode 100644
index 0000000..34a32db
--- /dev/null
+++ b/bloaty/file_sections.proto
@@ -0,0 +1,39 @@
+// Copyright 2021 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+syntax = "proto2";
+
+package file_sections;
+
+message SectionDescriptior {
+ // Name of the section (e.g. .rodata)
+ optional string name = 1;
+
+ // Size of that section as part of the file.
+ optional uint64 file_size = 2;
+
+ // Size of that section when loaded in memory.
+ optional uint64 vm_size = 3;
+}
+
+message File {
+ // Relative path from $OUT_DIR.
+ optional string path = 1;
+
+ // File sections.
+ repeated SectionDescriptior sections = 2;
+}
+
+message FileSizeMetrics {
+ repeated File files = 1;
+}
diff --git a/scripts/Android.bp b/scripts/Android.bp
index b9163cc..310c959 100644
--- a/scripts/Android.bp
+++ b/scripts/Android.bp
@@ -209,13 +209,18 @@
test_suites: ["general-tests"],
}
+python_library_host {
+ name: "ninja_rsp",
+ srcs: ["ninja_rsp.py"],
+}
+
python_binary_host {
name: "lint-project-xml",
main: "lint-project-xml.py",
srcs: [
"lint-project-xml.py",
- "ninja_rsp.py",
],
+ libs: ["ninja_rsp"],
}
python_binary_host {
@@ -223,8 +228,8 @@
main: "gen-kotlin-build-file.py",
srcs: [
"gen-kotlin-build-file.py",
- "ninja_rsp.py",
],
+ libs: ["ninja_rsp"],
}
python_binary_host {