blob: b4ae9a2639c6df74b4624a13346b86ca78a5e8e0 [file] [log] [blame]
Colin Crossd0be2102019-11-26 16:16:57 -08001// Copyright 2019 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// This executable runs a series of build commands to test and benchmark some critical user journeys.
16package main
17
18import (
19 "context"
20 "fmt"
21 "os"
22 "path/filepath"
23 "strconv"
24 "strings"
25 "time"
26
27 "android/soong/ui/build"
28 "android/soong/ui/logger"
29 "android/soong/ui/metrics"
30 "android/soong/ui/status"
31 "android/soong/ui/terminal"
32 "android/soong/ui/tracer"
33)
34
35type Test struct {
Patrice Arrudaf3261392020-03-23 08:18:36 -070036 name string
37 args []string
38 before func() error
Colin Crossd0be2102019-11-26 16:16:57 -080039
40 results TestResults
41}
42
43type TestResults struct {
44 metrics *metrics.Metrics
45 err error
46}
47
48// Run runs a single build command. It emulates the "m" command line by calling into Soong UI directly.
49func (t *Test) Run(logsDir string) {
50 output := terminal.NewStatusOutput(os.Stdout, "", false, false)
51
52 log := logger.New(output)
53 defer log.Cleanup()
54
55 ctx, cancel := context.WithCancel(context.Background())
56 defer cancel()
57
58 trace := tracer.New(log)
59 defer trace.Close()
60
61 met := metrics.New()
62
63 stat := &status.Status{}
64 defer stat.Finish()
65 stat.AddOutput(output)
66 stat.AddOutput(trace.StatusTracer())
67
68 build.SetupSignals(log, cancel, func() {
69 trace.Close()
70 log.Cleanup()
71 stat.Finish()
72 })
73
74 buildCtx := build.Context{ContextImpl: &build.ContextImpl{
75 Context: ctx,
76 Logger: log,
77 Metrics: met,
78 Tracer: trace,
79 Writer: output,
80 Status: stat,
81 }}
82
83 defer logger.Recover(func(err error) {
84 t.results.err = err
85 })
86
87 config := build.NewConfig(buildCtx, t.args...)
88 build.SetupOutDir(buildCtx, config)
89
90 os.MkdirAll(logsDir, 0777)
91 log.SetOutput(filepath.Join(logsDir, "soong.log"))
92 trace.SetOutput(filepath.Join(logsDir, "build.trace"))
93 stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log")))
94 stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log")))
95 stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error")))
96 stat.AddOutput(status.NewCriticalPath(log))
97
98 defer met.Dump(filepath.Join(logsDir, "soong_metrics"))
99
100 if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
101 if !strings.HasSuffix(start, "N") {
102 if start_time, err := strconv.ParseUint(start, 10, 64); err == nil {
103 log.Verbosef("Took %dms to start up.",
104 time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds())
105 buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano()))
106 }
107 }
108
109 if executable, err := os.Executable(); err == nil {
110 trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
111 }
112 }
113
114 f := build.NewSourceFinder(buildCtx, config)
115 defer f.Shutdown()
116 build.FindSources(buildCtx, config, f)
117
Anton Hansson5a7861a2021-06-04 10:09:01 +0100118 build.Build(buildCtx, config)
Colin Crossd0be2102019-11-26 16:16:57 -0800119
120 t.results.metrics = met
121}
122
Patrice Arrudaf3261392020-03-23 08:18:36 -0700123// Touch the Intent.java file to cause a rebuild of the frameworks to monitor the
124// incremental build speed as mentioned b/152046247. Intent.java file was chosen
125// as it is a key component of the framework and is often modified.
126func touchIntentFile() error {
127 const intentFileName = "frameworks/base/core/java/android/content/Intent.java"
128 currentTime := time.Now().Local()
129 return os.Chtimes(intentFileName, currentTime, currentTime)
130}
131
Colin Crossd0be2102019-11-26 16:16:57 -0800132func main() {
133 outDir := os.Getenv("OUT_DIR")
134 if outDir == "" {
135 outDir = "out"
136 }
137
138 cujDir := filepath.Join(outDir, "cuj_tests")
139
140 // Use a subdirectory for the out directory for the tests to keep them isolated.
141 os.Setenv("OUT_DIR", filepath.Join(cujDir, "out"))
142
143 // Each of these tests is run in sequence without resetting the output tree. The state of the output tree will
144 // affect each successive test. To maintain the validity of the benchmarks across changes, care must be taken
145 // to avoid changing the state of the tree when a test is run. This is most easily accomplished by adding tests
146 // at the end.
147 tests := []Test{
148 {
149 // Reset the out directory to get reproducible results.
150 name: "clean",
151 args: []string{"clean"},
152 },
153 {
154 // Parse the build files.
155 name: "nothing",
156 args: []string{"nothing"},
157 },
158 {
159 // Parse the build files again to monitor issues like globs rerunning.
160 name: "nothing_rebuild",
161 args: []string{"nothing"},
162 },
163 {
164 // Parse the build files again, this should always be very short.
165 name: "nothing_rebuild_twice",
166 args: []string{"nothing"},
167 },
168 {
169 // Build the framework as a common developer task and one that keeps getting longer.
170 name: "framework",
171 args: []string{"framework"},
172 },
173 {
174 // Build the framework again to make sure it doesn't rebuild anything.
175 name: "framework_rebuild",
176 args: []string{"framework"},
177 },
178 {
179 // Build the framework again to make sure it doesn't rebuild anything even if it did the second time.
180 name: "framework_rebuild_twice",
181 args: []string{"framework"},
182 },
Patrice Arrudaf3261392020-03-23 08:18:36 -0700183 {
184 // Scenario major_inc_build (b/152046247): tracking build speed of major incremental build.
185 name: "major_inc_build_droid",
186 args: []string{"droid"},
187 },
188 {
189 name: "major_inc_build_framework_minus_apex_after_droid_build",
190 args: []string{"framework-minus-apex"},
191 before: touchIntentFile,
192 },
193 {
194 name: "major_inc_build_framework_after_droid_build",
195 args: []string{"framework"},
196 before: touchIntentFile,
197 },
198 {
199 name: "major_inc_build_sync_after_droid_build",
200 args: []string{"sync"},
201 before: touchIntentFile,
202 },
203 {
204 name: "major_inc_build_droid_rebuild",
205 args: []string{"droid"},
206 before: touchIntentFile,
207 },
208 {
209 name: "major_inc_build_update_api_after_droid_rebuild",
210 args: []string{"update-api"},
211 before: touchIntentFile,
212 },
Colin Crossd0be2102019-11-26 16:16:57 -0800213 }
214
215 cujMetrics := metrics.NewCriticalUserJourneysMetrics()
216 defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb"))
217
218 for i, t := range tests {
219 logsSubDir := fmt.Sprintf("%02d_%s", i, t.name)
220 logsDir := filepath.Join(cujDir, "logs", logsSubDir)
Patrice Arrudaf3261392020-03-23 08:18:36 -0700221 if t.before != nil {
222 if err := t.before(); err != nil {
223 fmt.Printf("error running before function on test %q: %v\n", t.name, err)
224 break
225 }
226 }
Colin Crossd0be2102019-11-26 16:16:57 -0800227 t.Run(logsDir)
228 if t.results.err != nil {
229 fmt.Printf("error running test %q: %s\n", t.name, t.results.err)
230 break
231 }
232 if t.results.metrics != nil {
233 cujMetrics.Add(t.name, t.results.metrics)
234 }
235 }
236}