tools/perf/benchmarks

#!/usr/bin/env python3
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
if __name__ == "__main__":
    sys.dont_write_bytecode = True

import argparse
import dataclasses
import datetime
import json
import os
import pathlib
import random
import re
import shutil
import subprocess
import time
import uuid

import pretty
import utils


class FatalError(Exception):
    def __init__(self):
        pass


class OptionsError(Exception):
    def __init__(self, message):
        self.message = message


@dataclasses.dataclass(frozen=True)
class Lunch:
    "Lunch combination"

    target_product: str
    "TARGET_PRODUCT"

    target_release: str
    "TARGET_RELEASE"

    target_build_variant: str
    "TARGET_BUILD_VARIANT"

    def ToDict(self):
        return {
            "TARGET_PRODUCT": self.target_product,
            "TARGET_RELEASE": self.target_release,
            "TARGET_BUILD_VARIANT": self.target_build_variant,
        }

    def Combine(self):
        return f"{self.target_product}-{self.target_release}-{self.target_build_variant}"


@dataclasses.dataclass(frozen=True)
class Change:
    "A change that we make to the tree, and how to undo it"
    label: str
    "String to print in the log when the change is made"

    change: callable
    "Function to change the source tree"

    undo: callable
    "Function to revert the source tree to its previous condition in the most minimal way possible."


@dataclasses.dataclass(frozen=True)
class Benchmark:
    "Something we measure"

    id: str
    "Short ID for the benchmark, for the command line"

    title: str
    "Title for reports"

    change: Change
    "Source tree modification for the benchmark that will be measured"

    modules: list[str]
    "Build modules to build on soong command line"

    preroll: int
    "Number of times to run the build command to stabilize"

    postroll: int
    "Number of times to run the build command after reverting the action to stabilize"


@dataclasses.dataclass(frozen=True)
class FileSnapshot:
    "Snapshot of a file's contents."

    filename: str
    "The file that was snapshottened"

    contents: str
    "The contents of the file"

    def write(self):
        "Write the contents back to the file"
        with open(self.filename, "w") as f:
            f.write(self.contents)


def Snapshot(filename):
    """Return a FileSnapshot with the file's current contents."""
    with open(filename) as f:
        contents = f.read()
    return FileSnapshot(filename, contents)


def Clean():
    """Remove the out directory."""
    def remove_out():
        if os.path.exists("out"):
            shutil.rmtree("out")
    return Change(label="Remove out", change=remove_out, undo=lambda: None)


def NoChange():
    """No change to the source tree."""
    return Change(label="No change", change=lambda: None, undo=lambda: None)


def Create(filename):
    "Create an action to create `filename`. The parent directory must exist."
    def create():
        with open(filename, "w") as f:
            pass
    def delete():
        os.remove(filename)
    return Change(
                label=f"Create {filename}",
                change=create,
                undo=delete,
            )


def Modify(filename, contents, before=None):
    """Create an action to modify `filename` by appending the result of `contents`
    before the last instances of `before` in the file.

    Raises an error if `before` doesn't appear in the file.
    """
    orig = Snapshot(filename)
    if before:
        index = orig.contents.rfind(before)
        if index < 0:
            report_error(f"{filename}: Unable to find string '{before}' for modify operation.")
            raise FatalError()
    else:
        index = len(orig.contents)
    modified = FileSnapshot(filename, orig.contents[:index] + contents() + orig.contents[index:])
    if False:
        print(f"Modify: {filename}")
        x = orig.contents.replace("\n", "\n   ORIG")
        print(f"   ORIG {x}")
        x = modified.contents.replace("\n", "\n   MODIFIED")
        print(f"   MODIFIED {x}")

    return Change(
            label="Modify " + filename,
            change=lambda: modified.write(),
            undo=lambda: orig.write()
        )

def AddJavaField(filename, prefix):
    return Modify(filename,
                  lambda: f"{prefix} static final int BENCHMARK = {random.randint(0, 1000000)};\n",
                  before="}")


def Comment(prefix, suffix=""):
    return lambda: prefix + " " + str(uuid.uuid4()) + suffix


class BenchmarkReport():
    "Information about a run of the benchmark"

    lunch: Lunch
    "lunch combo"

    benchmark: Benchmark
    "The benchmark object."

    iteration: int
    "Which iteration of the benchmark"

    log_dir: str
    "Path the the log directory, relative to the root of the reports directory"

    preroll_duration_ns: [int]
    "Durations of the in nanoseconds."

    duration_ns: int
    "Duration of the measured portion of the benchmark in nanoseconds."

    postroll_duration_ns: [int]
    "Durations of the postrolls in nanoseconds."

    complete: bool
    "Whether the benchmark made it all the way through the postrolls."

    def __init__(self, lunch, benchmark, iteration, log_dir):
        self.lunch = lunch
        self.benchmark = benchmark
        self.iteration = iteration
        self.log_dir = log_dir
        self.preroll_duration_ns = []
        self.duration_ns = -1
        self.postroll_duration_ns = []
        self.complete = False

    def ToDict(self):
        return {
            "lunch": self.lunch.ToDict(),
            "id": self.benchmark.id,
            "title": self.benchmark.title,
            "modules": self.benchmark.modules,
            "change": self.benchmark.change.label,
            "iteration": self.iteration,
            "log_dir": self.log_dir,
            "preroll_duration_ns": self.preroll_duration_ns,
            "duration_ns": self.duration_ns,
            "postroll_duration_ns": self.postroll_duration_ns,
            "complete": self.complete,
        }

class Runner():
    """Runs the benchmarks."""

    def __init__(self, options):
        self._options = options
        self._reports = []
        self._complete = False

    def Run(self):
        """Run all of the user-selected benchmarks."""
        # Clean out the log dir or create it if necessary
        prepare_log_dir(self._options.LogDir())

        try:
            for lunch in self._options.Lunches():
                print(lunch)
                for benchmark in self._options.Benchmarks():
                    for iteration in range(self._options.Iterations()):
                        self._run_benchmark(lunch, benchmark, iteration)
            self._complete = True
        finally:
            self._write_summary()


    def _run_benchmark(self, lunch, benchmark, iteration):
        """Run a single benchmark."""
        benchmark_log_subdir = self._log_dir(lunch, benchmark, iteration)
        benchmark_log_dir = self._options.LogDir().joinpath(benchmark_log_subdir)

        sys.stderr.write(f"STARTING BENCHMARK: {benchmark.id}\n")
        sys.stderr.write(f"             lunch: {lunch.Combine()}\n")
        sys.stderr.write(f"         iteration: {iteration}\n")
        sys.stderr.write(f" benchmark_log_dir: {benchmark_log_dir}\n")

        report = BenchmarkReport(lunch, benchmark, iteration, benchmark_log_subdir)
        self._reports.append(report)

        # Preroll builds
        for i in range(benchmark.preroll):
            ns = self._run_build(lunch, benchmark_log_dir.joinpath(f"pre_{i}"), benchmark.modules)
            report.preroll_duration_ns.append(ns)

        sys.stderr.write(f"PERFORMING CHANGE: {benchmark.change.label}\n")
        if not self._options.DryRun():
            benchmark.change.change()
        try:

            # Measured build
            ns = self._run_build(lunch, benchmark_log_dir.joinpath("measured"), benchmark.modules)
            report.duration_ns = ns

            dist_one = self._options.DistOne()
            if dist_one:
                # If we're disting just one benchmark, save the logs and we can stop here.
                self._dist(dist_one)
            else:
                # Postroll builds
                for i in range(benchmark.preroll):
                    ns = self._run_build(lunch, benchmark_log_dir.joinpath(f"post_{i}"),
                                         benchmark.modules)
                    report.postroll_duration_ns.append(ns)

        finally:
            # Always undo, even if we crashed or the build failed and we stopped.
            sys.stderr.write(f"UNDOING CHANGE: {benchmark.change.label}\n")
            if not self._options.DryRun():
                benchmark.change.undo()

        self._write_summary()
        sys.stderr.write(f"FINISHED BENCHMARK: {benchmark.id}\n")

    def _log_dir(self, lunch, benchmark, iteration):
        """Construct the log directory fir a benchmark run."""
        path = f"{lunch.Combine()}/{benchmark.id}"
        # Zero pad to the correct length for correct alpha sorting
        path += ("/%0" + str(len(str(self._options.Iterations()))) + "d") % iteration
        return path

    def _run_build(self, lunch, build_log_dir, modules):
        """Builds the modules.  Saves interesting log files to log_dir.  Raises FatalError
        if the build fails.
        """
        sys.stderr.write(f"STARTING BUILD {modules}\n")

        before_ns = time.perf_counter_ns()
        if not self._options.DryRun():
            cmd = [
                "build/soong/soong_ui.bash",
                "--build-mode",
                "--all-modules",
                f"--dir={self._options.root}",
                "--skip-metrics-upload",
            ] + modules
            env = dict(os.environ)
            env["TARGET_PRODUCT"] = lunch.target_product
            env["TARGET_RELEASE"] = lunch.target_release
            env["TARGET_BUILD_VARIANT"] = lunch.target_build_variant
            returncode = subprocess.call(cmd, env=env)
            if returncode != 0:
                report_error(f"Build failed: {' '.join(cmd)}")
                raise FatalError()

        after_ns = time.perf_counter_ns()

        # TODO: Copy some log files.

        sys.stderr.write(f"FINISHED BUILD {modules}\n")

        return after_ns - before_ns

    def _dist(self, dist_dir):
        out_dir = pathlib.Path("out")
        dest_dir = pathlib.Path(dist_dir).joinpath("logs")
        os.makedirs(dest_dir, exist_ok=True)
        basenames = [
            "build.trace.gz",
            "soong.log",
            "soong_build_metrics.pb",
            "soong_metrics",
        ]
        for base in basenames:
            src = out_dir.joinpath(base)
            if src.exists():
                sys.stderr.write(f"DIST: copied {src} to {dest_dir}\n")
                shutil.copy(src, dest_dir)

    def _write_summary(self):
        # Write the results, even if the build failed or we crashed, including
        # whether we finished all of the benchmarks.
        data = {
            "start_time": self._options.Timestamp().isoformat(),
            "branch": self._options.Branch(),
            "tag": self._options.Tag(),
            "benchmarks": [report.ToDict() for report in self._reports],
            "complete": self._complete,
        }
        with open(self._options.LogDir().joinpath("summary.json"), "w", encoding="utf-8") as f:
            json.dump(data, f, indent=2, sort_keys=True)


def benchmark_table(benchmarks):
    rows = [("ID", "DESCRIPTION", "REBUILD"),]
    rows += [(benchmark.id, benchmark.title, " ".join(benchmark.modules)) for benchmark in
             benchmarks]
    return rows


def prepare_log_dir(directory):
    if os.path.exists(directory):
        # If it exists and isn't a directory, fail.
        if not os.path.isdir(directory):
            report_error(f"Log directory already exists but isn't a directory: {directory}")
            raise FatalError()
        # Make sure the directory is empty. Do this rather than deleting it to handle
        # symlinks cleanly.
        for filename in os.listdir(directory):
            entry = os.path.join(directory, filename)
            if os.path.isdir(entry):
                shutil.rmtree(entry)
            else:
                os.unlink(entry)
    else:
        # Create it
        os.makedirs(directory)


class Options():
    def __init__(self):
        self._had_error = False

        # Wall time clock when we started
        self._timestamp = datetime.datetime.now(datetime.timezone.utc)

        # Move to the root of the tree right away. Everything must happen from there.
        self.root = utils.get_root()
        if not self.root:
            report_error("Unable to find root of tree from cwd.")
            raise FatalError()
        os.chdir(self.root)

        # Initialize the Benchmarks. Note that this pre-loads all of the files, etc.
        # Doing all that here forces us to fail fast if one of them can't load a required
        # file, at the cost of a small startup speed. Don't make this do something slow
        # like scan the whole tree.
        self._init_benchmarks()

        # Argument parsing
        epilog = f"""
benchmarks:
{pretty.FormatTable(benchmark_table(self._benchmarks), prefix="  ")}
"""

        parser = argparse.ArgumentParser(
                prog="benchmarks",
                allow_abbrev=False, # Don't let people write unsupportable scripts.
                formatter_class=argparse.RawDescriptionHelpFormatter,
                epilog=epilog,
                description="Run build system performance benchmarks.")
        self.parser = parser

        parser.add_argument("--log-dir",
                            help="Directory for logs. Default is $TOP/../benchmarks/.")
        parser.add_argument("--dated-logs", action="store_true",
                            help="Append timestamp to log dir.")
        parser.add_argument("-n", action="store_true", dest="dry_run",
                            help="Dry run. Don't run the build commands but do everything else.")
        parser.add_argument("--tag",
                            help="Variant of the run, for when there are multiple perf runs.")
        parser.add_argument("--lunch", nargs="*",
                            help="Lunch combos to test")
        parser.add_argument("--iterations", type=int, default=1,
                            help="Number of iterations of each test to run.")
        parser.add_argument("--branch", type=str,
                            help="Specify branch. Otherwise a guess will be made based on repo.")
        parser.add_argument("--benchmark", nargs="*", default=[b.id for b in self._benchmarks],
                            metavar="BENCHMARKS",
                            help="Benchmarks to run.  Default suite will be run if omitted.")
        parser.add_argument("--dist-one", action="store_true",
                            help="Copy logs and metrics to the given dist dir. Requires that only"
                                + " one benchmark be supplied. Postroll steps will be skipped.")

        self._args = parser.parse_args()

        self._branch = self._branch()
        self._log_dir = self._log_dir()
        self._lunches = self._lunches()

        # Validate the benchmark ids
        all_ids = [benchmark.id for benchmark in self._benchmarks]
        bad_ids = [id for id in self._args.benchmark if id not in all_ids]
        if bad_ids:
            for id in bad_ids:
                self._error(f"Invalid benchmark: {id}")

        # --dist-one requires that only one benchmark be supplied
        if self._args.dist_one and len(self.Benchmarks()) != 1:
            self._error("--dist-one requires that exactly one --benchmark.")

        if self._had_error:
            raise FatalError()

    def Timestamp(self):
        return self._timestamp

    def _branch(self):
        """Return the branch, either from the command line or by guessing from repo."""
        if self._args.branch:
            return self._args.branch
        try:
            branch = subprocess.check_output(f"cd {self.root}/.repo/manifests"
                        + " && git rev-parse --abbrev-ref --symbolic-full-name @{u}",
                    shell=True, encoding="utf-8")
            return branch.strip().split("/")[-1]
        except subprocess.CalledProcessError as ex:
            report_error("Can't get branch from .repo dir. Specify --branch argument")
            report_error(str(ex))
            raise FatalError()

    def Branch(self):
        return self._branch

    def _log_dir(self):
        "The log directory to use, based on the current options"
        if self._args.log_dir:
            d = pathlib.Path(self._args.log_dir).resolve().absolute()
        else:
            d = self.root.joinpath("..", utils.DEFAULT_REPORT_DIR)
        if self._args.dated_logs:
            d = d.joinpath(self._timestamp.strftime('%Y-%m-%d'))
        d = d.joinpath(self._branch)
        if self._args.tag:
            d = d.joinpath(self._args.tag)
        return d.resolve().absolute()

    def LogDir(self):
        return self._log_dir

    def Benchmarks(self):
        return [b for b in self._benchmarks if b.id in self._args.benchmark]

    def Tag(self):
        return self._args.tag

    def DryRun(self):
        return self._args.dry_run

    def _lunches(self):
        def parse_lunch(lunch):
            parts = lunch.split("-")
            if len(parts) != 3:
                raise OptionsError(f"Invalid lunch combo: {lunch}")
            return Lunch(parts[0], parts[1], parts[2])
        # If they gave lunch targets on the command line use that
        if self._args.lunch:
            result = []
            # Split into Lunch objects
            for lunch in self._args.lunch:
                try:
                    result.append(parse_lunch(lunch))
                except OptionsError as ex:
                    self._error(ex.message)
            return result
        # Use whats in the environment
        product = os.getenv("TARGET_PRODUCT")
        release = os.getenv("TARGET_RELEASE")
        variant = os.getenv("TARGET_BUILD_VARIANT")
        if (not product) or (not release) or (not variant):
            # If they didn't give us anything, fail rather than guessing. There's no good
            # default for AOSP.
            self._error("No lunch combo specified. Either pass --lunch argument or run lunch.")
            return []
        return [Lunch(product, release, variant),]

    def Lunches(self):
        return self._lunches

    def Iterations(self):
        return self._args.iterations

    def DistOne(self):
        return self._args.dist_one

    def _init_benchmarks(self):
        """Initialize the list of benchmarks."""
        # Assumes that we've already chdired to the root of the tree.
        self._benchmarks = [
            Benchmark(id="full",
                      title="Full build",
                      change=Clean(),
                      modules=["droid"],
                      preroll=0,
                      postroll=3,
                      ),
            Benchmark(id="nochange",
                      title="No change",
                      change=NoChange(),
                      modules=["droid"],
                      preroll=2,
                      postroll=3,
                      ),
            Benchmark(id="unreferenced",
                      title="Create unreferenced file",
                      change=Create("bionic/unreferenced.txt"),
                      modules=["droid"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="modify_bp",
                      title="Modify Android.bp",
                      change=Modify("bionic/libc/Android.bp", Comment("//")),
                      modules=["droid"],
                      preroll=1,
                      postroll=3,
                      ),
            Benchmark(id="modify_stdio",
                      title="Modify stdio.cpp",
                      change=Modify("bionic/libc/stdio/stdio.cpp", Comment("//")),
                      modules=["libc"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="modify_adbd",
                      title="Modify adbd",
                      change=Modify("packages/modules/adb/daemon/main.cpp", Comment("//")),
                      modules=["adbd"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="services_private_field",
                      title="Add private field to ActivityManagerService.java",
                      change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java",
                                          "private"),
                      modules=["services"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="services_public_field",
                      title="Add public field to ActivityManagerService.java",
                      change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java",
                                          "/** @hide */ public"),
                      modules=["services"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="services_api",
                      title="Add API to ActivityManagerService.javaa",
                      change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java",
                                          "@android.annotation.SuppressLint(\"UnflaggedApi\") public"),
                      modules=["services"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="framework_private_field",
                      title="Add private field to Settings.java",
                      change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java",
                                          "private"),
                      modules=["framework-minus-apex"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="framework_public_field",
                      title="Add public field to Settings.java",
                      change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java",
                                          "/** @hide */ public"),
                      modules=["framework-minus-apex"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="framework_api",
                      title="Add API to Settings.java",
                      change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java",
                                          "@android.annotation.SuppressLint(\"UnflaggedApi\") public"),
                      modules=["framework-minus-apex"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="modify_framework_resource",
                      title="Modify framework resource",
                      change=Modify("frameworks/base/core/res/res/values/config.xml",
                                    lambda: str(uuid.uuid4()),
                                    before="</string>"),
                      modules=["framework-minus-apex"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="add_framework_resource",
                      title="Add framework resource",
                      change=Modify("frameworks/base/core/res/res/values/config.xml",
                                    lambda: f"<string name=\"BENCHMARK\">{uuid.uuid4()}</string>",
                                    before="</resources>"),
                      modules=["framework-minus-apex"],
                      preroll=1,
                      postroll=2,
                      ),
            Benchmark(id="add_systemui_field",
                      title="Add SystemUI field",
                      change=AddJavaField("frameworks/base/packages/SystemUI/src/com/android/systemui/wmshell/WMShell.java",
                                    "public"),
                      modules=["SystemUI"],
                      preroll=1,
                      postroll=2,
                      ),
        ]

    def _error(self, message):
        report_error(message)
        self._had_error = True


def report_error(message):
    sys.stderr.write(f"error: {message}\n")


def main(argv):
    try:
        options = Options()
        runner = Runner(options)
        runner.Run()
    except FatalError:
        sys.stderr.write(f"FAILED\n")
        sys.exit(1)


if __name__ == "__main__":
    main(sys.argv)