Add MetricsProcessor to calculate metrics stats
This CL adds MetricsProcessor class to calculate metrics stats
for both device tests and host tests.
Test: atest MicrodroidBenchmarks
Change-Id: I4f742d6800f14c857a5d0e0718e09870e8ca6547
diff --git a/tests/benchmark/Android.bp b/tests/benchmark/Android.bp
index 0d1cd91..bf78202 100644
--- a/tests/benchmark/Android.bp
+++ b/tests/benchmark/Android.bp
@@ -10,6 +10,7 @@
srcs: ["src/java/**/*.java"],
static_libs: [
"MicrodroidDeviceTestHelper",
+ "MicrodroidTestHelper",
"androidx.test.runner",
"androidx.test.ext.junit",
"com.android.microdroid.testservice-java",
diff --git a/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java b/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java
index 200b7bf..8e65132 100644
--- a/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java
+++ b/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java
@@ -32,6 +32,7 @@
import android.util.Log;
import com.android.microdroid.test.MicrodroidDeviceTestBase;
+import com.android.microdroid.test.common.MetricsProcessor;
import com.android.microdroid.testservice.IBenchmarkService;
import org.junit.Before;
@@ -46,6 +47,7 @@
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@RunWith(Parameterized.class)
@@ -68,6 +70,8 @@
@Parameterized.Parameter public boolean mProtectedVm;
+ private final MetricsProcessor mMetricsProcessor = new MetricsProcessor(METRIC_NAME_PREFIX);
+
private Instrumentation mInstrumentation;
@Before
@@ -153,11 +157,11 @@
userspaceBootTimeMetrics.add(result.getUserspaceElapsedNanoTime() / nanoToMilli);
}
- reportMetrics(vmStartingTimeMetrics, "vm_starting_time_", "_ms");
- reportMetrics(bootTimeMetrics, "boot_time_", "_ms");
- reportMetrics(bootloaderTimeMetrics, "bootloader_time_", "_ms");
- reportMetrics(kernelBootTimeMetrics, "kernel_boot_time_", "_ms");
- reportMetrics(userspaceBootTimeMetrics, "userspace_boot_time_", "_ms");
+ reportMetrics(vmStartingTimeMetrics, "vm_starting_time", "ms");
+ reportMetrics(bootTimeMetrics, "boot_time", "ms");
+ reportMetrics(bootloaderTimeMetrics, "bootloader_time", "ms");
+ reportMetrics(kernelBootTimeMetrics, "kernel_boot_time", "ms");
+ reportMetrics(userspaceBootTimeMetrics, "userspace_boot_time", "ms");
}
@Test
@@ -187,7 +191,7 @@
mInner.newVmConfigBuilder("assets/vm_config_io.json")
.debugLevel(DebugLevel.FULL)
.build();
- List<Double> transferRates = new ArrayList<>();
+ List<Double> transferRates = new ArrayList<>(IO_TEST_TRIAL_COUNT);
for (int i = 0; i < IO_TEST_TRIAL_COUNT; ++i) {
int port = (mProtectedVm ? 5666 : 6666) + i;
@@ -196,7 +200,7 @@
VirtualMachine vm = mInner.getVirtualMachineManager().get(vmName);
BenchmarkVmListener.create(new VsockListener(transferRates, port)).runToFinish(TAG, vm);
}
- reportMetrics(transferRates, "vsock/transfer_host_to_vm_", "_mb_per_sec");
+ reportMetrics(transferRates, "vsock/transfer_host_to_vm", "mb_per_sec");
}
@Test
@@ -214,7 +218,7 @@
mInner.newVmConfigBuilder("assets/vm_config_io.json")
.debugLevel(DebugLevel.FULL)
.build();
- List<Double> readRates = new ArrayList<>();
+ List<Double> readRates = new ArrayList<>(IO_TEST_TRIAL_COUNT);
for (int i = 0; i < IO_TEST_TRIAL_COUNT + 1; ++i) {
if (i == 1) {
@@ -230,33 +234,15 @@
.runToFinish(TAG, vm);
}
reportMetrics(
- readRates,
- isRand ? "virtio-blk/rand_read_" : "virtio-blk/seq_read_",
- "_mb_per_sec");
+ readRates, isRand ? "virtio-blk/rand_read" : "virtio-blk/seq_read", "mb_per_sec");
}
- private void reportMetrics(List<Double> data, String base, String suffix) {
- double sum = 0;
- double min = Double.MAX_VALUE;
- double max = Double.MIN_VALUE;
- for (double d : data) {
- sum += d;
- if (min > d) min = d;
- if (max < d) max = d;
- }
- double avg = sum / data.size();
- double sqSum = 0;
- for (double d : data) {
- sqSum += (d - avg) * (d - avg);
- }
- double stdDev = Math.sqrt(sqSum / (data.size() - 1));
-
+ private void reportMetrics(List<Double> metrics, String name, String unit) {
+ Map<String, Double> stats = mMetricsProcessor.computeStats(metrics, name, unit);
Bundle bundle = new Bundle();
- String prefix = METRIC_NAME_PREFIX + base;
- bundle.putDouble(prefix + "min" + suffix, min);
- bundle.putDouble(prefix + "max" + suffix, max);
- bundle.putDouble(prefix + "average" + suffix, avg);
- bundle.putDouble(prefix + "stdev" + suffix, stdDev);
+ for (Map.Entry<String, Double> entry : stats.entrySet()) {
+ bundle.putDouble(entry.getKey(), entry.getValue());
+ }
mInstrumentation.sendStatus(0, bundle);
}
diff --git a/tests/benchmark_hostside/Android.bp b/tests/benchmark_hostside/Android.bp
index 354a260..c5c7571 100644
--- a/tests/benchmark_hostside/Android.bp
+++ b/tests/benchmark_hostside/Android.bp
@@ -12,6 +12,7 @@
],
static_libs: [
"MicrodroidHostTestHelper",
+ "MicrodroidTestHelper",
],
test_suites: [
"general-tests",
diff --git a/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java b/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java
index 8e9da55..143a35a 100644
--- a/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java
+++ b/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java
@@ -16,8 +16,6 @@
package android.avf.test;
-import android.platform.test.annotations.RootPermissionTest;
-
import static com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestMetrics;
import static com.google.common.truth.Truth.assertWithMessage;
@@ -25,8 +23,11 @@
import static org.junit.Assume.assumeTrue;
+import android.platform.test.annotations.RootPermissionTest;
+
import com.android.microdroid.test.CommandRunner;
import com.android.microdroid.test.MicrodroidHostTestCaseBase;
+import com.android.microdroid.test.common.MetricsProcessor;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
import com.android.tradefed.util.CommandResult;
@@ -36,6 +37,9 @@
import org.junit.Test;
import org.junit.runner.RunWith;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -61,6 +65,8 @@
private static final int ROUND_COUNT = 5;
private static final String METRIC_PREFIX = "avf_perf/hostside/";
+ private final MetricsProcessor mMetricsProcessor = new MetricsProcessor(METRIC_PREFIX);
+
@Before
public void setUp() throws Exception {
testIfDeviceIsCapable(getDevice());
@@ -80,11 +86,10 @@
@Test
public void testBootEnableAndDisablePKVM() throws Exception {
-
testPKVMStatusSwitchSupported();
- double[] bootWithPKVMEnableTime = new double[ROUND_COUNT];
- double[] bootWithoutPKVMEnableTime = new double[ROUND_COUNT];
+ List<Double> bootWithPKVMEnableTime = new ArrayList<>(ROUND_COUNT);
+ List<Double> bootWithoutPKVMEnableTime = new ArrayList<>(ROUND_COUNT);
for (int round = 0; round < ROUND_COUNT; ++round) {
@@ -93,7 +98,7 @@
rebootFromBootloaderAndWaitBootCompleted();
long elapsedWithPKVMEnable = System.nanoTime() - start;
double elapsedSec = elapsedWithPKVMEnable / NANOS_IN_SEC;
- bootWithPKVMEnableTime[round] = elapsedSec;
+ bootWithPKVMEnableTime.add(elapsedSec);
CLog.i("Boot time with PKVM enable took " + elapsedSec + "s");
setPKVMStatusWithRebootToBootloader(false);
@@ -101,20 +106,20 @@
rebootFromBootloaderAndWaitBootCompleted();
long elapsedWithoutPKVMEnable = System.nanoTime() - start;
elapsedSec = elapsedWithoutPKVMEnable / NANOS_IN_SEC;
- bootWithoutPKVMEnableTime[round] = elapsedSec;
+ bootWithoutPKVMEnableTime.add(elapsedSec);
CLog.i("Boot time with PKVM disable took " + elapsedSec + "s");
}
- reportMetric("boot_time_with_pkvm_enable", "s", bootWithPKVMEnableTime);
- reportMetric("boot_time_with_pkvm_disable", "s", bootWithoutPKVMEnableTime);
+ reportMetric(bootWithPKVMEnableTime, "boot_time_with_pkvm_enable", "s");
+ reportMetric(bootWithoutPKVMEnableTime, "boot_time_with_pkvm_disable", "s");
}
@Test
public void testBootWithAndWithoutCompOS() throws Exception {
assume().withMessage("Skip on CF; too slow").that(isCuttlefish()).isFalse();
- double[] bootWithCompOsTime = new double[ROUND_COUNT];
- double[] bootWithoutCompOsTime = new double[ROUND_COUNT];
+ List<Double> bootWithCompOsTime = new ArrayList<>(ROUND_COUNT);
+ List<Double> bootWithoutCompOsTime = new ArrayList<>(ROUND_COUNT);
for (int round = 0; round < ROUND_COUNT; ++round) {
@@ -125,7 +130,7 @@
rebootAndWaitBootCompleted();
long elapsedWithCompOS = System.nanoTime() - start;
double elapsedSec = elapsedWithCompOS / NANOS_IN_SEC;
- bootWithCompOsTime[round] = elapsedSec;
+ bootWithCompOsTime.add(elapsedSec);
CLog.i("Boot time with compilation OS took " + elapsedSec + "s");
// Boot time without compilation OS test.
@@ -134,12 +139,12 @@
rebootAndWaitBootCompleted();
long elapsedWithoutCompOS = System.nanoTime() - start;
elapsedSec = elapsedWithoutCompOS / NANOS_IN_SEC;
- bootWithoutCompOsTime[round] = elapsedSec;
+ bootWithoutCompOsTime.add(elapsedSec);
CLog.i("Boot time without compilation OS took " + elapsedSec + "s");
}
- reportMetric("boot_time_with_compos", "s", bootWithCompOsTime);
- reportMetric("boot_time_without_compos", "s", bootWithoutCompOsTime);
+ reportMetric(bootWithCompOsTime, "boot_time_with_compos", "s");
+ reportMetric(bootWithoutCompOsTime, "boot_time_without_compos", "s");
}
private void testPKVMStatusSwitchSupported() throws Exception {
@@ -154,31 +159,12 @@
assumeTrue(!result.getStderr().contains("Invalid oem command"));
}
- private void reportMetric(String name, String unit, double[] values) {
- double sum = 0;
- double min = Double.MAX_VALUE;
- double max = Double.MIN_VALUE;
-
- for (double val : values) {
- sum += val;
- min = val < min ? val : min;
- max = val > max ? val : max;
- }
-
- double average = sum / values.length;
-
- double variance = 0;
- for (double val : values) {
- final double tmp = val - average;
- variance += tmp * tmp;
- }
- double stdev = Math.sqrt(variance / (double) (values.length - 1));
-
+ private void reportMetric(List<Double> data, String name, String unit) {
+ Map<String, Double> stats = mMetricsProcessor.computeStats(data, name, unit);
TestMetrics metrics = new TestMetrics();
- metrics.addTestMetric(METRIC_PREFIX + name + "_average_" + unit, Double.toString(average));
- metrics.addTestMetric(METRIC_PREFIX + name + "_min_" + unit, Double.toString(min));
- metrics.addTestMetric(METRIC_PREFIX + name + "_max_" + unit, Double.toString(max));
- metrics.addTestMetric(METRIC_PREFIX + name + "_stdev_" + unit, Double.toString(stdev));
+ for (Map.Entry<String, Double> entry : stats.entrySet()) {
+ metrics.addTestMetric(entry.getKey(), Double.toString(entry.getValue()));
+ }
}
private void setPKVMStatusWithRebootToBootloader(boolean isEnable) throws Exception {
diff --git a/tests/helper/Android.bp b/tests/helper/Android.bp
index f8dcef7..6d769e6 100644
--- a/tests/helper/Android.bp
+++ b/tests/helper/Android.bp
@@ -9,8 +9,14 @@
}
java_library_static {
+ name: "MicrodroidTestHelper",
+ srcs: ["src/java/com/android/microdroid/test/common/*.java"],
+ host_supported: true,
+}
+
+java_library_static {
name: "MicrodroidDeviceTestHelper",
- srcs: ["src/java/com/android/microdroid/**/*.java"],
+ srcs: ["src/java/com/android/microdroid/test/*.java"],
static_libs: [
"androidx.test.runner",
"androidx.test.ext.junit",
diff --git a/tests/helper/src/java/com/android/microdroid/test/common/MetricsProcessor.java b/tests/helper/src/java/com/android/microdroid/test/common/MetricsProcessor.java
new file mode 100644
index 0000000..c12b07d
--- /dev/null
+++ b/tests/helper/src/java/com/android/microdroid/test/common/MetricsProcessor.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.microdroid.test.common;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** This class processes the metrics for both device tests and host tests. */
+public final class MetricsProcessor {
+ private final String mPrefix;
+
+ public MetricsProcessor(String prefix) {
+ mPrefix = prefix;
+ }
+
+ /**
+ * Computes the min, max, average and standard deviation of the given metrics and saves them in
+ * a {@link Map} with the corresponding keys equal to [mPrefix + name +
+ * _[min|max|average|stdev]_ + unit].
+ */
+ public Map<String, Double> computeStats(List<Double> metrics, String name, String unit) {
+ double sum = 0;
+ double min = Double.MAX_VALUE;
+ double max = Double.MIN_VALUE;
+ for (double d : metrics) {
+ sum += d;
+ if (min > d) min = d;
+ if (max < d) max = d;
+ }
+ double avg = sum / metrics.size();
+ double sqSum = 0;
+ for (double d : metrics) {
+ sqSum += (d - avg) * (d - avg);
+ }
+ double stdDev = Math.sqrt(sqSum / (metrics.size() - 1));
+
+ Map<String, Double> stats = new HashMap<String, Double>();
+ String prefix = mPrefix + name;
+ stats.put(prefix + "_min_" + unit, min);
+ stats.put(prefix + "_max_" + unit, max);
+ stats.put(prefix + "_average_" + unit, avg);
+ stats.put(prefix + "_stdev_" + unit, stdDev);
+ return stats;
+ }
+}