blob: 1cd5fea329a7f6662ff82ef326865b0e3eb575d6 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18 "crypto/md5"
19 "fmt"
20 "io/ioutil"
Dan Willemsen71edc8b2019-01-02 12:21:18 -080021 "os"
22 "os/user"
Dan Willemsen1e704462016-08-21 15:17:17 -070023 "path/filepath"
Dan Willemsen1e704462016-08-21 15:17:17 -070024 "strings"
Dan Willemsenb82471a2018-05-17 16:37:09 -070025
Nan Zhang17f27672018-12-12 16:01:49 -080026 "android/soong/ui/metrics"
Dan Willemsenb82471a2018-05-17 16:37:09 -070027 "android/soong/ui/status"
Dan Willemsen1e704462016-08-21 15:17:17 -070028)
29
30var spaceSlashReplacer = strings.NewReplacer("/", "_", " ", "_")
31
Dan Willemsen29971232018-09-26 14:58:30 -070032const katiBuildSuffix = ""
33const katiCleanspecSuffix = "-cleanspec"
Dan Willemsenfb1271a2018-09-26 15:00:42 -070034const katiPackageSuffix = "-package"
Dan Willemsen29971232018-09-26 14:58:30 -070035
Dan Willemsen1e704462016-08-21 15:17:17 -070036// genKatiSuffix creates a suffix for kati-generated files so that we can cache
37// them based on their inputs. So this should encode all common changes to Kati
38// inputs. Currently that includes the TARGET_PRODUCT, kati-processed command
39// line arguments, and the directories specified by mm/mmm.
40func genKatiSuffix(ctx Context, config Config) {
41 katiSuffix := "-" + config.TargetProduct()
42 if args := config.KatiArgs(); len(args) > 0 {
43 katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_"))
44 }
Dan Willemsen1e704462016-08-21 15:17:17 -070045
46 // If the suffix is too long, replace it with a md5 hash and write a
47 // file that contains the original suffix.
48 if len(katiSuffix) > 64 {
49 shortSuffix := "-" + fmt.Sprintf("%x", md5.Sum([]byte(katiSuffix)))
50 config.SetKatiSuffix(shortSuffix)
51
52 ctx.Verbosef("Kati ninja suffix too long: %q", katiSuffix)
53 ctx.Verbosef("Replacing with: %q", shortSuffix)
54
Dan Willemsen29971232018-09-26 14:58:30 -070055 if err := ioutil.WriteFile(strings.TrimSuffix(config.KatiBuildNinjaFile(), "ninja")+"suf", []byte(katiSuffix), 0777); err != nil {
Dan Willemsen1e704462016-08-21 15:17:17 -070056 ctx.Println("Error writing suffix file:", err)
57 }
58 } else {
59 config.SetKatiSuffix(katiSuffix)
60 }
61}
62
Dan Willemsenfb1271a2018-09-26 15:00:42 -070063func runKati(ctx Context, config Config, extraSuffix string, args []string, envFunc func(*Environment)) {
Dan Willemsenf173d592017-04-27 14:28:00 -070064 executable := config.PrebuiltBuildTool("ckati")
Dan Willemsen29971232018-09-26 14:58:30 -070065 args = append([]string{
Dan Willemsen1e704462016-08-21 15:17:17 -070066 "--ninja",
67 "--ninja_dir=" + config.OutDir(),
Dan Willemsen29971232018-09-26 14:58:30 -070068 "--ninja_suffix=" + config.KatiSuffix() + extraSuffix,
69 "--no_ninja_prelude",
Dan Willemsen6587bed2020-04-18 20:25:59 -070070 "--use_ninja_phony_output",
Dan Willemsen1e704462016-08-21 15:17:17 -070071 "--regen",
72 "--ignore_optional_include=" + filepath.Join(config.OutDir(), "%.P"),
73 "--detect_android_echo",
Dan Willemsenc38d3662017-02-24 10:53:23 -080074 "--color_warnings",
75 "--gen_all_targets",
Dan Willemsen29971232018-09-26 14:58:30 -070076 "--use_find_emulator",
Dan Willemsen418420e2017-05-30 14:07:45 -070077 "--werror_find_emulator",
Dan Willemsend368d6f2018-06-15 21:53:18 -070078 "--no_builtin_rules",
79 "--werror_suffix_rules",
Dan Willemsenf880b582018-07-23 23:09:05 -070080 "--warn_real_to_phony",
81 "--warn_phony_looks_real",
Dan Willemsen60977462019-04-18 09:40:15 -070082 "--werror_real_to_phony",
83 "--werror_phony_looks_real",
84 "--werror_writable",
Dan Willemsencc628902019-01-24 15:53:06 -080085 "--top_level_phony",
Dan Willemsen75d2c172017-10-12 20:46:34 -070086 "--kati_stats",
Dan Willemsen29971232018-09-26 14:58:30 -070087 }, args...)
88
Dan Willemsenf99915f2018-10-25 22:04:42 -070089 if config.Environment().IsEnvTrue("EMPTY_NINJA_FILE") {
90 args = append(args, "--empty_ninja_file")
91 }
92
Colin Cross9016b912019-11-11 14:57:42 -080093 if config.UseRemoteBuild() {
94 args = append(args, "--default_pool=local_pool")
95 }
96
Dan Willemsen29971232018-09-26 14:58:30 -070097 cmd := Command(ctx, config, "ckati", executable, args...)
98 cmd.Sandbox = katiSandbox
99 pipe, err := cmd.StdoutPipe()
100 if err != nil {
101 ctx.Fatalln("Error getting output pipe for ckati:", err)
102 }
103 cmd.Stderr = cmd.Stdout
104
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700105 envFunc(cmd.Environment)
106
Dan Willemsen71edc8b2019-01-02 12:21:18 -0800107 if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok {
Dan Willemsena17ac962020-03-10 15:38:57 -0700108 username := "unknown"
109 if u, err := user.Current(); err == nil {
110 username = u.Username
111 } else {
112 ctx.Println("Failed to get current user:", err)
Dan Willemsen71edc8b2019-01-02 12:21:18 -0800113 }
Dan Willemsena17ac962020-03-10 15:38:57 -0700114 cmd.Environment.Set("BUILD_USERNAME", username)
Dan Willemsen71edc8b2019-01-02 12:21:18 -0800115 }
116
117 if _, ok := cmd.Environment.Get("BUILD_HOSTNAME"); !ok {
118 hostname, err := os.Hostname()
119 if err != nil {
Dan Willemsena17ac962020-03-10 15:38:57 -0700120 ctx.Println("Failed to read hostname:", err)
121 hostname = "unknown"
Dan Willemsen71edc8b2019-01-02 12:21:18 -0800122 }
123 cmd.Environment.Set("BUILD_HOSTNAME", hostname)
124 }
125
Dan Willemsen29971232018-09-26 14:58:30 -0700126 cmd.StartOrFatal()
127 status.KatiReader(ctx.Status.StartTool(), pipe)
128 cmd.WaitOrFatal()
129}
130
131func runKatiBuild(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800132 ctx.BeginTrace(metrics.RunKati, "kati build")
Dan Willemsen29971232018-09-26 14:58:30 -0700133 defer ctx.EndTrace()
134
135 args := []string{
Dan Willemsen25a56182018-08-31 20:25:32 -0700136 "--writable", config.OutDir() + "/",
Dan Willemsenb2e6c2e2017-07-13 17:24:44 -0700137 "-f", "build/make/core/main.mk",
Dan Willemsen1e704462016-08-21 15:17:17 -0700138 }
139
Dan Willemsenfa42f3c2018-06-15 21:54:47 -0700140 // PDK builds still uses a few implicit rules
141 if !config.IsPdkBuild() {
Dan Willemsen2bc456e2018-06-21 21:41:35 -0700142 args = append(args, "--werror_implicit_rules")
Dan Willemsenfa42f3c2018-06-15 21:54:47 -0700143 }
144
Dan Willemsen3d60b112018-04-04 22:25:56 -0700145 if !config.BuildBrokenDupRules() {
146 args = append(args, "--werror_overriding_commands")
147 }
148
Dan Willemsen1e704462016-08-21 15:17:17 -0700149 args = append(args, config.KatiArgs()...)
150
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700151 args = append(args,
152 "SOONG_MAKEVARS_MK="+config.SoongMakeVarsMk(),
153 "SOONG_ANDROID_MK="+config.SoongAndroidMk(),
154 "TARGET_DEVICE_DIR="+config.TargetDeviceDir(),
155 "KATI_PACKAGE_MK_DIR="+config.KatiPackageMkDir())
Dan Willemsen1e704462016-08-21 15:17:17 -0700156
Dan Willemsen2d31a442018-10-20 21:33:41 -0700157 runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {})
Dan Willemsen78428262019-12-13 18:50:24 -0800158
Colin Cross8ba7d472020-06-25 11:27:52 -0700159 distGzipFile(ctx, config, config.KatiBuildNinjaFile())
160
Dan Willemsen78428262019-12-13 18:50:24 -0800161 cleanCopyHeaders(ctx, config)
Dan Willemsen1e775d72020-01-03 13:40:45 -0800162 cleanOldInstalledFiles(ctx, config)
Dan Willemsen78428262019-12-13 18:50:24 -0800163}
164
165func cleanCopyHeaders(ctx Context, config Config) {
166 ctx.BeginTrace("clean", "clean copy headers")
167 defer ctx.EndTrace()
168
169 data, err := ioutil.ReadFile(filepath.Join(config.ProductOut(), ".copied_headers_list"))
170 if err != nil {
171 if os.IsNotExist(err) {
172 return
173 }
174 ctx.Fatalf("Failed to read copied headers list: %v", err)
175 }
176
177 headers := strings.Fields(string(data))
178 if len(headers) < 1 {
179 ctx.Fatal("Failed to parse copied headers list: %q", string(data))
180 }
181 headerDir := headers[0]
182 headers = headers[1:]
183
184 filepath.Walk(headerDir,
185 func(path string, info os.FileInfo, err error) error {
186 if err != nil {
187 return nil
188 }
189 if info.IsDir() {
190 return nil
191 }
192 if !inList(path, headers) {
193 ctx.Printf("Removing obsolete header %q", path)
194 if err := os.Remove(path); err != nil {
195 ctx.Fatalf("Failed to remove obsolete header %q: %v", path, err)
196 }
197 }
198 return nil
199 })
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700200}
201
Dan Willemsen1e775d72020-01-03 13:40:45 -0800202func cleanOldInstalledFiles(ctx Context, config Config) {
203 ctx.BeginTrace("clean", "clean old installed files")
204 defer ctx.EndTrace()
205
206 // We shouldn't be removing files from one side of the two-step asan builds
207 var suffix string
208 if v, ok := config.Environment().Get("SANITIZE_TARGET"); ok {
209 if sanitize := strings.Fields(v); inList("address", sanitize) {
210 suffix = "_asan"
211 }
212 }
213
214 cleanOldFiles(ctx, config.ProductOut(), ".installable_files"+suffix)
215
216 cleanOldFiles(ctx, config.HostOut(), ".installable_test_files")
217}
218
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700219func runKatiPackage(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800220 ctx.BeginTrace(metrics.RunKati, "kati package")
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700221 defer ctx.EndTrace()
222
223 args := []string{
224 "--writable", config.DistDir() + "/",
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700225 "--werror_implicit_rules",
226 "--werror_overriding_commands",
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700227 "-f", "build/make/packaging/main.mk",
228 "KATI_PACKAGE_MK_DIR=" + config.KatiPackageMkDir(),
229 }
230
231 runKati(ctx, config, katiPackageSuffix, args, func(env *Environment) {
232 env.Allow([]string{
233 // Some generic basics
234 "LANG",
235 "LC_MESSAGES",
236 "PATH",
237 "PWD",
238 "TMPDIR",
239
240 // Tool configs
Dan Willemsen70c1ff82019-08-21 14:56:13 -0700241 "ASAN_SYMBOLIZER_PATH",
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700242 "JAVA_HOME",
243 "PYTHONDONTWRITEBYTECODE",
244
245 // Build configuration
246 "ANDROID_BUILD_SHELL",
247 "DIST_DIR",
248 "OUT_DIR",
249 }...)
250
251 if config.Dist() {
252 env.Set("DIST", "true")
Dan Willemsen2d31a442018-10-20 21:33:41 -0700253 env.Set("DIST_DIR", config.DistDir())
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700254 }
255 })
Colin Cross8ba7d472020-06-25 11:27:52 -0700256
257 distGzipFile(ctx, config, config.KatiPackageNinjaFile())
Dan Willemsen1e704462016-08-21 15:17:17 -0700258}
Dan Willemsen29f88272017-02-18 18:12:41 -0800259
Dan Willemsen59fdf962017-07-24 22:26:54 -0700260func runKatiCleanSpec(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800261 ctx.BeginTrace(metrics.RunKati, "kati cleanspec")
Dan Willemsen59fdf962017-07-24 22:26:54 -0700262 defer ctx.EndTrace()
263
Dan Willemsen29971232018-09-26 14:58:30 -0700264 runKati(ctx, config, katiCleanspecSuffix, []string{
265 "--werror_implicit_rules",
Dan Willemsen3d60b112018-04-04 22:25:56 -0700266 "--werror_overriding_commands",
Dan Willemsen59fdf962017-07-24 22:26:54 -0700267 "-f", "build/make/core/cleanbuild.mk",
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700268 "SOONG_MAKEVARS_MK=" + config.SoongMakeVarsMk(),
269 "TARGET_DEVICE_DIR=" + config.TargetDeviceDir(),
Dan Willemsen2d31a442018-10-20 21:33:41 -0700270 }, func(env *Environment) {})
Dan Willemsen59fdf962017-07-24 22:26:54 -0700271}