blob: e59146be19e8ba90ed93b46e091fec366fe69e4d [file] [log] [blame]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Convert makefile containing device configuration to Starlark file
16// The conversion can handle the following constructs in a makefile:
17// * comments
18// * simple variable assignments
19// * $(call init-product,<file>)
20// * $(call inherit-product-if-exists
21// * if directives
22// All other constructs are carried over to the output starlark file as comments.
23//
24package mk2rbc
25
26import (
27 "bytes"
28 "fmt"
29 "io"
Sasha Smundak6609ba72021-07-22 18:32:56 -070030 "io/fs"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080031 "io/ioutil"
32 "os"
33 "path/filepath"
34 "regexp"
Cole Faust62e05112022-04-05 17:56:11 -070035 "sort"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080036 "strconv"
37 "strings"
38 "text/scanner"
39
40 mkparser "android/soong/androidmk/parser"
41)
42
43const (
Sasha Smundak6d852dd2021-09-27 20:34:39 -070044 annotationCommentPrefix = "RBC#"
45 baseUri = "//build/make/core:product_config.rbc"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080046 // The name of the struct exported by the product_config.rbc
47 // that contains the functions and variables available to
48 // product configuration Starlark files.
49 baseName = "rblf"
50
Sasha Smundak65b547e2021-09-17 15:35:41 -070051 soongNsPrefix = "SOONG_CONFIG_"
52
Sasha Smundakb051c4e2020-11-05 20:45:07 -080053 // And here are the functions and variables:
Cole Fauste2a37982022-03-09 16:00:17 -080054 cfnGetCfg = baseName + ".cfg"
55 cfnMain = baseName + ".product_configuration"
56 cfnBoardMain = baseName + ".board_configuration"
57 cfnPrintVars = baseName + ".printvars"
58 cfnInherit = baseName + ".inherit"
59 cfnSetListDefault = baseName + ".setdefault"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080060)
61
62const (
Cole Faust9ebf6e42021-12-13 14:08:34 -080063 soongConfigAppend = "soong_config_append"
64 soongConfigAssign = "soong_config_set"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080065)
66
Cole Faust9ebf6e42021-12-13 14:08:34 -080067var knownFunctions = map[string]interface {
68 parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr
Sasha Smundakb051c4e2020-11-05 20:45:07 -080069}{
Cole Faust1cc08852022-02-28 11:12:08 -080070 "abspath": &simpleCallParser{name: baseName + ".abspath", returnType: starlarkTypeString},
71 "add-product-dex-preopt-module-config": &simpleCallParser{name: baseName + ".add_product_dex_preopt_module_config", returnType: starlarkTypeString, addHandle: true},
72 "add_soong_config_namespace": &simpleCallParser{name: baseName + ".soong_config_namespace", returnType: starlarkTypeVoid, addGlobals: true},
73 "add_soong_config_var_value": &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true},
74 soongConfigAssign: &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true},
75 soongConfigAppend: &simpleCallParser{name: baseName + ".soong_config_append", returnType: starlarkTypeVoid, addGlobals: true},
76 "soong_config_get": &simpleCallParser{name: baseName + ".soong_config_get", returnType: starlarkTypeString, addGlobals: true},
77 "add-to-product-copy-files-if-exists": &simpleCallParser{name: baseName + ".copy_if_exists", returnType: starlarkTypeList},
78 "addprefix": &simpleCallParser{name: baseName + ".addprefix", returnType: starlarkTypeList},
79 "addsuffix": &simpleCallParser{name: baseName + ".addsuffix", returnType: starlarkTypeList},
80 "copy-files": &simpleCallParser{name: baseName + ".copy_files", returnType: starlarkTypeList},
Cole Faust0e2b2562022-04-01 11:46:50 -070081 "dir": &simpleCallParser{name: baseName + ".dir", returnType: starlarkTypeString},
Cole Faust1cc08852022-02-28 11:12:08 -080082 "dist-for-goals": &simpleCallParser{name: baseName + ".mkdist_for_goals", returnType: starlarkTypeVoid, addGlobals: true},
Cole Faust6c41b8a2022-04-13 13:53:48 -070083 "enforce-product-packages-exist": &simpleCallParser{name: baseName + ".enforce_product_packages_exist", returnType: starlarkTypeVoid, addHandle: true},
Cole Faust1cc08852022-02-28 11:12:08 -080084 "error": &makeControlFuncParser{name: baseName + ".mkerror"},
85 "findstring": &simpleCallParser{name: baseName + ".findstring", returnType: starlarkTypeInt},
86 "find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList},
87 "filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList},
88 "filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList},
Cole Faust5a13aaf2022-04-27 17:49:35 -070089 "firstword": &simpleCallParser{name: baseName + ".first_word", returnType: starlarkTypeString},
Cole Faustf035d402022-03-28 14:02:50 -070090 "foreach": &foreachCallParser{},
Cole Faust1cc08852022-02-28 11:12:08 -080091 "if": &ifCallParser{},
92 "info": &makeControlFuncParser{name: baseName + ".mkinfo"},
93 "is-board-platform": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
94 "is-board-platform2": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
95 "is-board-platform-in-list": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true},
96 "is-board-platform-in-list2": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true},
97 "is-product-in-list": &isProductInListCallParser{},
98 "is-vendor-board-platform": &isVendorBoardPlatformCallParser{},
99 "is-vendor-board-qcom": &isVendorBoardQcomCallParser{},
Cole Faust5a13aaf2022-04-27 17:49:35 -0700100 "lastword": &simpleCallParser{name: baseName + ".last_word", returnType: starlarkTypeString},
Cole Faust1cc08852022-02-28 11:12:08 -0800101 "notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString},
102 "math_max": &mathMaxOrMinCallParser{function: "max"},
103 "math_min": &mathMaxOrMinCallParser{function: "min"},
104 "math_gt_or_eq": &mathComparisonCallParser{op: ">="},
105 "math_gt": &mathComparisonCallParser{op: ">"},
106 "math_lt": &mathComparisonCallParser{op: "<"},
107 "my-dir": &myDirCallParser{},
108 "patsubst": &substCallParser{fname: "patsubst"},
109 "product-copy-files-by-pattern": &simpleCallParser{name: baseName + ".product_copy_files_by_pattern", returnType: starlarkTypeList},
Cole Faustea9db582022-03-21 17:50:05 -0700110 "require-artifacts-in-path": &simpleCallParser{name: baseName + ".require_artifacts_in_path", returnType: starlarkTypeVoid, addHandle: true},
111 "require-artifacts-in-path-relaxed": &simpleCallParser{name: baseName + ".require_artifacts_in_path_relaxed", returnType: starlarkTypeVoid, addHandle: true},
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800112 // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
Cole Faust9ebf6e42021-12-13 14:08:34 -0800113 "shell": &shellCallParser{},
Cole Faust95b95cb2022-04-05 16:37:39 -0700114 "sort": &simpleCallParser{name: baseName + ".mksort", returnType: starlarkTypeList},
Cole Faust1cc08852022-02-28 11:12:08 -0800115 "strip": &simpleCallParser{name: baseName + ".mkstrip", returnType: starlarkTypeString},
Cole Faust9ebf6e42021-12-13 14:08:34 -0800116 "subst": &substCallParser{fname: "subst"},
117 "warning": &makeControlFuncParser{name: baseName + ".mkwarning"},
118 "word": &wordCallParser{},
Cole Faust94c4a9a2022-04-22 17:43:52 -0700119 "words": &wordsCallParser{},
Cole Faust1cc08852022-02-28 11:12:08 -0800120 "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList},
Cole Faust9ebf6e42021-12-13 14:08:34 -0800121}
122
Cole Faustf035d402022-03-28 14:02:50 -0700123// The same as knownFunctions, but returns a []starlarkNode instead of a starlarkExpr
124var knownNodeFunctions = map[string]interface {
125 parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode
126}{
127 "eval": &evalNodeParser{},
128 "if": &ifCallNodeParser{},
129 "inherit-product": &inheritProductCallParser{loadAlways: true},
130 "inherit-product-if-exists": &inheritProductCallParser{loadAlways: false},
131 "foreach": &foreachCallNodeParser{},
132}
133
Cole Faust1e275862022-04-26 14:28:04 -0700134// These look like variables, but are actually functions, and would give
135// undefined variable errors if we converted them as variables. Instead,
136// emit an error instead of converting them.
137var unsupportedFunctions = map[string]bool{
138 "local-generated-sources-dir": true,
139 "local-intermediates-dir": true,
140}
141
Cole Faust9ebf6e42021-12-13 14:08:34 -0800142// These are functions that we don't implement conversions for, but
143// we allow seeing their definitions in the product config files.
144var ignoredDefines = map[string]bool{
145 "find-word-in-list": true, // internal macro
146 "get-vendor-board-platforms": true, // internal macro, used by is-board-platform, etc.
147 "is-android-codename": true, // unused by product config
148 "is-android-codename-in-list": true, // unused by product config
149 "is-chipset-in-board-platform": true, // unused by product config
150 "is-chipset-prefix-in-board-platform": true, // unused by product config
151 "is-not-board-platform": true, // defined but never used
152 "is-platform-sdk-version-at-least": true, // unused by product config
153 "match-prefix": true, // internal macro
154 "match-word": true, // internal macro
155 "match-word-in-list": true, // internal macro
156 "tb-modules": true, // defined in hardware/amlogic/tb_modules/tb_detect.mk, unused
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800157}
158
Cole Faustb0d32ab2021-12-09 14:00:59 -0800159var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800160
161// Conversion request parameters
162type Request struct {
Sasha Smundak422b6142021-11-11 18:31:59 -0800163 MkFile string // file to convert
164 Reader io.Reader // if set, read input from this stream instead
Sasha Smundak422b6142021-11-11 18:31:59 -0800165 OutputSuffix string // generated Starlark files suffix
166 OutputDir string // if set, root of the output hierarchy
167 ErrorLogger ErrorLogger
168 TracedVariables []string // trace assignment to these variables
169 TraceCalls bool
170 SourceFS fs.FS
171 MakefileFinder MakefileFinder
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800172}
173
Sasha Smundak7d934b92021-11-10 12:20:01 -0800174// ErrorLogger prints errors and gathers error statistics.
175// Its NewError function is called on every error encountered during the conversion.
176type ErrorLogger interface {
Sasha Smundak422b6142021-11-11 18:31:59 -0800177 NewError(el ErrorLocation, node mkparser.Node, text string, args ...interface{})
178}
179
180type ErrorLocation struct {
181 MkFile string
182 MkLine int
183}
184
185func (el ErrorLocation) String() string {
186 return fmt.Sprintf("%s:%d", el.MkFile, el.MkLine)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800187}
188
189// Derives module name for a given file. It is base name
190// (file name without suffix), with some characters replaced to make it a Starlark identifier
191func moduleNameForFile(mkFile string) string {
192 base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
193 // TODO(asmundak): what else can be in the product file names?
Sasha Smundak6609ba72021-07-22 18:32:56 -0700194 return strings.NewReplacer("-", "_", ".", "_").Replace(base)
195
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800196}
197
198func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
199 r := &mkparser.MakeString{StringPos: mkString.StringPos}
200 r.Strings = append(r.Strings, mkString.Strings...)
201 r.Variables = append(r.Variables, mkString.Variables...)
202 return r
203}
204
205func isMakeControlFunc(s string) bool {
206 return s == "error" || s == "warning" || s == "info"
207}
208
Cole Faustf0632662022-04-07 13:59:24 -0700209// varAssignmentScope points to the last assignment for each variable
210// in the current block. It is used during the parsing to chain
211// the assignments to a variable together.
212type varAssignmentScope struct {
213 outer *varAssignmentScope
214 vars map[string]bool
215}
216
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800217// Starlark output generation context
218type generationContext struct {
Cole Faustf0632662022-04-07 13:59:24 -0700219 buf strings.Builder
220 starScript *StarlarkScript
221 indentLevel int
222 inAssignment bool
223 tracedCount int
224 varAssignments *varAssignmentScope
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800225}
226
227func NewGenerateContext(ss *StarlarkScript) *generationContext {
Cole Faustf0632662022-04-07 13:59:24 -0700228 return &generationContext{
229 starScript: ss,
230 varAssignments: &varAssignmentScope{
231 outer: nil,
232 vars: make(map[string]bool),
233 },
234 }
235}
236
237func (gctx *generationContext) pushVariableAssignments() {
238 va := &varAssignmentScope{
239 outer: gctx.varAssignments,
240 vars: make(map[string]bool),
241 }
242 gctx.varAssignments = va
243}
244
245func (gctx *generationContext) popVariableAssignments() {
246 gctx.varAssignments = gctx.varAssignments.outer
247}
248
249func (gctx *generationContext) hasBeenAssigned(v variable) bool {
250 for va := gctx.varAssignments; va != nil; va = va.outer {
251 if _, ok := va.vars[v.name()]; ok {
252 return true
253 }
254 }
255 return false
256}
257
258func (gctx *generationContext) setHasBeenAssigned(v variable) {
259 gctx.varAssignments.vars[v.name()] = true
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800260}
261
262// emit returns generated script
263func (gctx *generationContext) emit() string {
264 ss := gctx.starScript
265
266 // The emitted code has the following layout:
267 // <initial comments>
268 // preamble, i.e.,
269 // load statement for the runtime support
270 // load statement for each unique submodule pulled in by this one
271 // def init(g, handle):
272 // cfg = rblf.cfg(handle)
273 // <statements>
274 // <warning if conversion was not clean>
275
276 iNode := len(ss.nodes)
277 for i, node := range ss.nodes {
278 if _, ok := node.(*commentNode); !ok {
279 iNode = i
280 break
281 }
282 node.emit(gctx)
283 }
284
285 gctx.emitPreamble()
286
287 gctx.newLine()
288 // The arguments passed to the init function are the global dictionary
289 // ('g') and the product configuration dictionary ('cfg')
290 gctx.write("def init(g, handle):")
291 gctx.indentLevel++
292 if gctx.starScript.traceCalls {
293 gctx.newLine()
294 gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
295 }
296 gctx.newLine()
297 gctx.writef("cfg = %s(handle)", cfnGetCfg)
298 for _, node := range ss.nodes[iNode:] {
299 node.emit(gctx)
300 }
301
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800302 if gctx.starScript.traceCalls {
303 gctx.newLine()
304 gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
305 }
306 gctx.indentLevel--
307 gctx.write("\n")
308 return gctx.buf.String()
309}
310
311func (gctx *generationContext) emitPreamble() {
312 gctx.newLine()
313 gctx.writef("load(%q, %q)", baseUri, baseName)
314 // Emit exactly one load statement for each URI.
315 loadedSubConfigs := make(map[string]string)
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800316 for _, mi := range gctx.starScript.inherited {
317 uri := mi.path
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800318 if m, ok := loadedSubConfigs[uri]; ok {
319 // No need to emit load statement, but fix module name.
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800320 mi.moduleLocalName = m
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800321 continue
322 }
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800323 if mi.optional || mi.missing {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800324 uri += "|init"
325 }
326 gctx.newLine()
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800327 gctx.writef("load(%q, %s = \"init\")", uri, mi.entryName())
328 loadedSubConfigs[uri] = mi.moduleLocalName
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800329 }
330 gctx.write("\n")
331}
332
333func (gctx *generationContext) emitPass() {
334 gctx.newLine()
335 gctx.write("pass")
336}
337
338func (gctx *generationContext) write(ss ...string) {
339 for _, s := range ss {
340 gctx.buf.WriteString(s)
341 }
342}
343
344func (gctx *generationContext) writef(format string, args ...interface{}) {
345 gctx.write(fmt.Sprintf(format, args...))
346}
347
348func (gctx *generationContext) newLine() {
349 if gctx.buf.Len() == 0 {
350 return
351 }
352 gctx.write("\n")
353 gctx.writef("%*s", 2*gctx.indentLevel, "")
354}
355
Sasha Smundak422b6142021-11-11 18:31:59 -0800356func (gctx *generationContext) emitConversionError(el ErrorLocation, message string) {
357 gctx.writef(`rblf.mk2rbc_error("%s", %q)`, el, message)
358}
359
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800360func (gctx *generationContext) emitLoadCheck(im inheritedModule) {
361 if !im.needsLoadCheck() {
362 return
363 }
364 gctx.newLine()
365 gctx.writef("if not %s:", im.entryName())
366 gctx.indentLevel++
367 gctx.newLine()
368 gctx.write(`rblf.mkerror("`, gctx.starScript.mkFile, `", "Cannot find %s" % (`)
369 im.pathExpr().emit(gctx)
370 gctx.write("))")
371 gctx.indentLevel--
372}
373
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800374type knownVariable struct {
375 name string
376 class varClass
377 valueType starlarkType
378}
379
380type knownVariables map[string]knownVariable
381
382func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
383 v, exists := pcv[name]
384 if !exists {
385 pcv[name] = knownVariable{name, varClass, valueType}
386 return
387 }
388 // Conflict resolution:
389 // * config class trumps everything
390 // * any type trumps unknown type
391 match := varClass == v.class
392 if !match {
393 if varClass == VarClassConfig {
394 v.class = VarClassConfig
395 match = true
396 } else if v.class == VarClassConfig {
397 match = true
398 }
399 }
400 if valueType != v.valueType {
401 if valueType != starlarkTypeUnknown {
402 if v.valueType == starlarkTypeUnknown {
403 v.valueType = valueType
404 } else {
405 match = false
406 }
407 }
408 }
409 if !match {
410 fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
411 name, varClass, valueType, v.class, v.valueType)
412 }
413}
414
415// All known product variables.
416var KnownVariables = make(knownVariables)
417
418func init() {
419 for _, kv := range []string{
420 // Kernel-related variables that we know are lists.
421 "BOARD_VENDOR_KERNEL_MODULES",
422 "BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
423 "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
424 "BOARD_RECOVERY_KERNEL_MODULES",
425 // Other variables we knwo are lists
426 "ART_APEX_JARS",
427 } {
428 KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
429 }
430}
431
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800432// Information about the generated Starlark script.
433type StarlarkScript struct {
Sasha Smundak422b6142021-11-11 18:31:59 -0800434 mkFile string
435 moduleName string
436 mkPos scanner.Position
437 nodes []starlarkNode
438 inherited []*moduleInfo
439 hasErrors bool
Sasha Smundak422b6142021-11-11 18:31:59 -0800440 traceCalls bool // print enter/exit each init function
441 sourceFS fs.FS
442 makefileFinder MakefileFinder
443 nodeLocator func(pos mkparser.Pos) int
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800444}
445
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800446// parseContext holds the script we are generating and all the ephemeral data
447// needed during the parsing.
448type parseContext struct {
449 script *StarlarkScript
450 nodes []mkparser.Node // Makefile as parsed by mkparser
451 currentNodeIndex int // Node in it we are processing
452 ifNestLevel int
453 moduleNameCount map[string]int // count of imported modules with given basename
454 fatalError error
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800455 outputSuffix string
Sasha Smundak7d934b92021-11-10 12:20:01 -0800456 errorLogger ErrorLogger
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800457 tracedVariables map[string]bool // variables to be traced in the generated script
458 variables map[string]variable
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800459 outputDir string
Sasha Smundak6609ba72021-07-22 18:32:56 -0700460 dependentModules map[string]*moduleInfo
Sasha Smundak3deb9682021-07-26 18:42:25 -0700461 soongNamespaces map[string]map[string]bool
Sasha Smundak6d852dd2021-09-27 20:34:39 -0700462 includeTops []string
Cole Faustf92c9f22022-03-14 14:35:50 -0700463 typeHints map[string]starlarkType
464 atTopOfMakefile bool
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800465}
466
467func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
468 predefined := []struct{ name, value string }{
469 {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
470 {"LOCAL_PATH", filepath.Dir(ss.mkFile)},
Cole Faust5a13aaf2022-04-27 17:49:35 -0700471 {"MAKEFILE_LIST", ss.mkFile},
Cole Faust9b6111a2022-02-02 15:38:33 -0800472 {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800473 // TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
474 {"TARGET_COPY_OUT_SYSTEM", "system"},
475 {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
476 {"TARGET_COPY_OUT_DATA", "data"},
477 {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
478 {"TARGET_COPY_OUT_OEM", "oem"},
479 {"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
480 {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
481 {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
482 {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
483 {"TARGET_COPY_OUT_ROOT", "root"},
484 {"TARGET_COPY_OUT_RECOVERY", "recovery"},
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800485 {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800486 // TODO(asmundak): to process internal config files, we need the following variables:
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800487 // TARGET_VENDOR
488 // target_base_product
489 //
490
491 // the following utility variables are set in build/make/common/core.mk:
492 {"empty", ""},
493 {"space", " "},
494 {"comma", ","},
495 {"newline", "\n"},
496 {"pound", "#"},
497 {"backslash", "\\"},
498 }
499 ctx := &parseContext{
500 script: ss,
501 nodes: nodes,
502 currentNodeIndex: 0,
503 ifNestLevel: 0,
504 moduleNameCount: make(map[string]int),
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800505 variables: make(map[string]variable),
Sasha Smundak6609ba72021-07-22 18:32:56 -0700506 dependentModules: make(map[string]*moduleInfo),
Sasha Smundak3deb9682021-07-26 18:42:25 -0700507 soongNamespaces: make(map[string]map[string]bool),
Cole Faust6c934f62022-01-06 15:51:12 -0800508 includeTops: []string{},
Cole Faustf92c9f22022-03-14 14:35:50 -0700509 typeHints: make(map[string]starlarkType),
510 atTopOfMakefile: true,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800511 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800512 for _, item := range predefined {
513 ctx.variables[item.name] = &predefinedVariable{
514 baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
515 value: &stringLiteralExpr{item.value},
516 }
517 }
518
519 return ctx
520}
521
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800522func (ctx *parseContext) hasNodes() bool {
523 return ctx.currentNodeIndex < len(ctx.nodes)
524}
525
526func (ctx *parseContext) getNode() mkparser.Node {
527 if !ctx.hasNodes() {
528 return nil
529 }
530 node := ctx.nodes[ctx.currentNodeIndex]
531 ctx.currentNodeIndex++
532 return node
533}
534
535func (ctx *parseContext) backNode() {
536 if ctx.currentNodeIndex <= 0 {
537 panic("Cannot back off")
538 }
539 ctx.currentNodeIndex--
540}
541
Cole Faustdd569ae2022-01-31 15:48:29 -0800542func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800543 // Handle only simple variables
Cole Faust00afd4f2022-04-26 14:01:56 -0700544 if !a.Name.Const() || a.Target != nil {
Cole Faustdd569ae2022-01-31 15:48:29 -0800545 return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")}
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800546 }
547 name := a.Name.Strings[0]
Sasha Smundakea3bc3a2021-11-10 13:06:42 -0800548 // The `override` directive
549 // override FOO :=
550 // is parsed as an assignment to a variable named `override FOO`.
551 // There are very few places where `override` is used, just flag it.
552 if strings.HasPrefix(name, "override ") {
Cole Faustdd569ae2022-01-31 15:48:29 -0800553 return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")}
Sasha Smundakea3bc3a2021-11-10 13:06:42 -0800554 }
Cole Faust5d5fcc32022-04-26 18:02:05 -0700555 if name == ".KATI_READONLY" {
556 // Skip assignments to .KATI_READONLY. If it was in the output file, it
557 // would be an error because it would be sorted before the definition of
558 // the variable it's trying to make readonly.
559 return []starlarkNode{}
560 }
Sasha Smundakea3bc3a2021-11-10 13:06:42 -0800561
Cole Faustc00184e2021-11-08 12:08:57 -0800562 // Soong configuration
Sasha Smundak3deb9682021-07-26 18:42:25 -0700563 if strings.HasPrefix(name, soongNsPrefix) {
Cole Faustdd569ae2022-01-31 15:48:29 -0800564 return ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a)
Sasha Smundak3deb9682021-07-26 18:42:25 -0700565 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800566 lhs := ctx.addVariable(name)
567 if lhs == nil {
Cole Faustdd569ae2022-01-31 15:48:29 -0800568 return []starlarkNode{ctx.newBadNode(a, "unknown variable %s", name)}
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800569 }
Cole Faust3c4fc992022-02-28 16:05:01 -0800570 _, isTraced := ctx.tracedVariables[lhs.name()]
Sasha Smundak422b6142021-11-11 18:31:59 -0800571 asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced, location: ctx.errorLocation(a)}
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800572 if lhs.valueType() == starlarkTypeUnknown {
573 // Try to divine variable type from the RHS
574 asgn.value = ctx.parseMakeString(a, a.Value)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800575 inferred_type := asgn.value.typ()
576 if inferred_type != starlarkTypeUnknown {
Sasha Smundak9d011ab2021-07-09 16:00:57 -0700577 lhs.setValueType(inferred_type)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800578 }
579 }
580 if lhs.valueType() == starlarkTypeList {
Cole Faustdd569ae2022-01-31 15:48:29 -0800581 xConcat, xBad := ctx.buildConcatExpr(a)
582 if xBad != nil {
Cole Faust1e275862022-04-26 14:28:04 -0700583 asgn.value = xBad
584 } else {
585 switch len(xConcat.items) {
586 case 0:
587 asgn.value = &listExpr{}
588 case 1:
589 asgn.value = xConcat.items[0]
590 default:
591 asgn.value = xConcat
592 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800593 }
594 } else {
595 asgn.value = ctx.parseMakeString(a, a.Value)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800596 }
597
Cole Faust421a1922022-03-16 14:35:45 -0700598 if asgn.lhs.valueType() == starlarkTypeString &&
599 asgn.value.typ() != starlarkTypeUnknown &&
600 asgn.value.typ() != starlarkTypeString {
601 asgn.value = &toStringExpr{expr: asgn.value}
602 }
603
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800604 switch a.Type {
605 case "=", ":=":
606 asgn.flavor = asgnSet
607 case "+=":
Cole Fauste2a37982022-03-09 16:00:17 -0800608 asgn.flavor = asgnAppend
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800609 case "?=":
610 asgn.flavor = asgnMaybeSet
611 default:
612 panic(fmt.Errorf("unexpected assignment type %s", a.Type))
613 }
614
Cole Faustdd569ae2022-01-31 15:48:29 -0800615 return []starlarkNode{asgn}
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800616}
617
Cole Faustdd569ae2022-01-31 15:48:29 -0800618func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Assignment) []starlarkNode {
Sasha Smundak3deb9682021-07-26 18:42:25 -0700619 val := ctx.parseMakeString(asgn, asgn.Value)
620 if xBad, ok := val.(*badExpr); ok {
Cole Faustdd569ae2022-01-31 15:48:29 -0800621 return []starlarkNode{&exprNode{expr: xBad}}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700622 }
Sasha Smundak3deb9682021-07-26 18:42:25 -0700623
624 // Unfortunately, Soong namespaces can be set up by directly setting corresponding Make
625 // variables instead of via add_soong_config_namespace + add_soong_config_var_value.
626 // Try to divine the call from the assignment as follows:
627 if name == "NAMESPACES" {
628 // Upon seeng
629 // SOONG_CONFIG_NAMESPACES += foo
630 // remember that there is a namespace `foo` and act as we saw
631 // $(call add_soong_config_namespace,foo)
632 s, ok := maybeString(val)
633 if !ok {
Cole Faustdd569ae2022-01-31 15:48:29 -0800634 return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_NAMESPACES assignment, please use add_soong_config_namespace instead")}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700635 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800636 result := make([]starlarkNode, 0)
Sasha Smundak3deb9682021-07-26 18:42:25 -0700637 for _, ns := range strings.Fields(s) {
638 ctx.addSoongNamespace(ns)
Cole Faustdd569ae2022-01-31 15:48:29 -0800639 result = append(result, &exprNode{&callExpr{
Cole Faust9ebf6e42021-12-13 14:08:34 -0800640 name: baseName + ".soong_config_namespace",
641 args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{ns}},
Sasha Smundak3deb9682021-07-26 18:42:25 -0700642 returnType: starlarkTypeVoid,
643 }})
644 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800645 return result
Sasha Smundak3deb9682021-07-26 18:42:25 -0700646 } else {
647 // Upon seeing
648 // SOONG_CONFIG_x_y = v
649 // find a namespace called `x` and act as if we encountered
Cole Faustc00184e2021-11-08 12:08:57 -0800650 // $(call soong_config_set,x,y,v)
Sasha Smundak3deb9682021-07-26 18:42:25 -0700651 // or check that `x_y` is a namespace, and then add the RHS of this assignment as variables in
652 // it.
653 // Emit an error in the ambiguous situation (namespaces `foo_bar` with a variable `baz`
654 // and `foo` with a variable `bar_baz`.
655 namespaceName := ""
656 if ctx.hasSoongNamespace(name) {
657 namespaceName = name
658 }
659 var varName string
660 for pos, ch := range name {
661 if !(ch == '_' && ctx.hasSoongNamespace(name[0:pos])) {
662 continue
663 }
664 if namespaceName != "" {
Cole Faustdd569ae2022-01-31 15:48:29 -0800665 return []starlarkNode{ctx.newBadNode(asgn, "ambiguous soong namespace (may be either `%s` or `%s`)", namespaceName, name[0:pos])}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700666 }
667 namespaceName = name[0:pos]
668 varName = name[pos+1:]
669 }
670 if namespaceName == "" {
Cole Faustdd569ae2022-01-31 15:48:29 -0800671 return []starlarkNode{ctx.newBadNode(asgn, "cannot figure out Soong namespace, please use add_soong_config_var_value macro instead")}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700672 }
673 if varName == "" {
674 // Remember variables in this namespace
675 s, ok := maybeString(val)
676 if !ok {
Cole Faustdd569ae2022-01-31 15:48:29 -0800677 return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_ assignment, please use add_soong_config_var_value instead")}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700678 }
679 ctx.updateSoongNamespace(asgn.Type != "+=", namespaceName, strings.Fields(s))
Cole Faustdd569ae2022-01-31 15:48:29 -0800680 return []starlarkNode{}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700681 }
682
683 // Finally, handle assignment to a namespace variable
684 if !ctx.hasNamespaceVar(namespaceName, varName) {
Cole Faustdd569ae2022-01-31 15:48:29 -0800685 return []starlarkNode{ctx.newBadNode(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName)}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700686 }
Cole Faust9ebf6e42021-12-13 14:08:34 -0800687 fname := baseName + "." + soongConfigAssign
Sasha Smundak65b547e2021-09-17 15:35:41 -0700688 if asgn.Type == "+=" {
Cole Faust9ebf6e42021-12-13 14:08:34 -0800689 fname = baseName + "." + soongConfigAppend
Sasha Smundak65b547e2021-09-17 15:35:41 -0700690 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800691 return []starlarkNode{&exprNode{&callExpr{
Sasha Smundak65b547e2021-09-17 15:35:41 -0700692 name: fname,
Cole Faust9ebf6e42021-12-13 14:08:34 -0800693 args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{namespaceName}, &stringLiteralExpr{varName}, val},
Sasha Smundak3deb9682021-07-26 18:42:25 -0700694 returnType: starlarkTypeVoid,
Cole Faustdd569ae2022-01-31 15:48:29 -0800695 }}}
Sasha Smundak3deb9682021-07-26 18:42:25 -0700696 }
697}
698
Cole Faustdd569ae2022-01-31 15:48:29 -0800699func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) (*concatExpr, *badExpr) {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800700 xConcat := &concatExpr{}
701 var xItemList *listExpr
702 addToItemList := func(x ...starlarkExpr) {
703 if xItemList == nil {
704 xItemList = &listExpr{[]starlarkExpr{}}
705 }
706 xItemList.items = append(xItemList.items, x...)
707 }
708 finishItemList := func() {
709 if xItemList != nil {
710 xConcat.items = append(xConcat.items, xItemList)
711 xItemList = nil
712 }
713 }
714
715 items := a.Value.Words()
716 for _, item := range items {
717 // A function call in RHS is supposed to return a list, all other item
718 // expressions return individual elements.
719 switch x := ctx.parseMakeString(a, item).(type) {
720 case *badExpr:
Cole Faustdd569ae2022-01-31 15:48:29 -0800721 return nil, x
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800722 case *stringLiteralExpr:
723 addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
724 default:
725 switch x.typ() {
726 case starlarkTypeList:
727 finishItemList()
728 xConcat.items = append(xConcat.items, x)
729 case starlarkTypeString:
730 finishItemList()
731 xConcat.items = append(xConcat.items, &callExpr{
732 object: x,
733 name: "split",
734 args: nil,
735 returnType: starlarkTypeList,
736 })
737 default:
738 addToItemList(x)
739 }
740 }
741 }
742 if xItemList != nil {
743 xConcat.items = append(xConcat.items, xItemList)
744 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800745 return xConcat, nil
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800746}
747
Sasha Smundak6609ba72021-07-22 18:32:56 -0700748func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
749 modulePath := ctx.loadedModulePath(path)
750 if mi, ok := ctx.dependentModules[modulePath]; ok {
Sasha Smundak868c5e32021-09-23 16:20:58 -0700751 mi.optional = mi.optional && optional
Sasha Smundak6609ba72021-07-22 18:32:56 -0700752 return mi
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800753 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800754 moduleName := moduleNameForFile(path)
755 moduleLocalName := "_" + moduleName
756 n, found := ctx.moduleNameCount[moduleName]
757 if found {
758 moduleLocalName += fmt.Sprintf("%d", n)
759 }
760 ctx.moduleNameCount[moduleName] = n + 1
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800761 _, err := fs.Stat(ctx.script.sourceFS, path)
Sasha Smundak6609ba72021-07-22 18:32:56 -0700762 mi := &moduleInfo{
763 path: modulePath,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800764 originalPath: path,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800765 moduleLocalName: moduleLocalName,
Sasha Smundak6609ba72021-07-22 18:32:56 -0700766 optional: optional,
Sasha Smundak6bc132a2022-01-10 17:02:16 -0800767 missing: err != nil,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800768 }
Sasha Smundak6609ba72021-07-22 18:32:56 -0700769 ctx.dependentModules[modulePath] = mi
770 ctx.script.inherited = append(ctx.script.inherited, mi)
771 return mi
772}
773
774func (ctx *parseContext) handleSubConfig(
Cole Faustdd569ae2022-01-31 15:48:29 -0800775 v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule) starlarkNode) []starlarkNode {
Sasha Smundak6609ba72021-07-22 18:32:56 -0700776
Cole Faust62e05112022-04-05 17:56:11 -0700777 // Allow seeing $(sort $(wildcard realPathExpr)) or $(wildcard realPathExpr)
778 // because those are functionally the same as not having the sort/wildcard calls.
779 if ce, ok := pathExpr.(*callExpr); ok && ce.name == "rblf.mksort" && len(ce.args) == 1 {
780 if ce2, ok2 := ce.args[0].(*callExpr); ok2 && ce2.name == "rblf.expand_wildcard" && len(ce2.args) == 1 {
781 pathExpr = ce2.args[0]
782 }
783 } else if ce2, ok2 := pathExpr.(*callExpr); ok2 && ce2.name == "rblf.expand_wildcard" && len(ce2.args) == 1 {
784 pathExpr = ce2.args[0]
785 }
786
Sasha Smundak6609ba72021-07-22 18:32:56 -0700787 // In a simple case, the name of a module to inherit/include is known statically.
788 if path, ok := maybeString(pathExpr); ok {
Sasha Smundak868c5e32021-09-23 16:20:58 -0700789 // Note that even if this directive loads a module unconditionally, a module may be
790 // absent without causing any harm if this directive is inside an if/else block.
791 moduleShouldExist := loadAlways && ctx.ifNestLevel == 0
Sasha Smundak6609ba72021-07-22 18:32:56 -0700792 if strings.Contains(path, "*") {
793 if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
Cole Faust62e05112022-04-05 17:56:11 -0700794 sort.Strings(paths)
Cole Faustdd569ae2022-01-31 15:48:29 -0800795 result := make([]starlarkNode, 0)
Sasha Smundak6609ba72021-07-22 18:32:56 -0700796 for _, p := range paths {
Sasha Smundak868c5e32021-09-23 16:20:58 -0700797 mi := ctx.newDependentModule(p, !moduleShouldExist)
Cole Faustdd569ae2022-01-31 15:48:29 -0800798 result = append(result, processModule(inheritedStaticModule{mi, loadAlways}))
Sasha Smundak6609ba72021-07-22 18:32:56 -0700799 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800800 return result
Sasha Smundak6609ba72021-07-22 18:32:56 -0700801 } else {
Cole Faustdd569ae2022-01-31 15:48:29 -0800802 return []starlarkNode{ctx.newBadNode(v, "cannot glob wildcard argument")}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700803 }
804 } else {
Sasha Smundak868c5e32021-09-23 16:20:58 -0700805 mi := ctx.newDependentModule(path, !moduleShouldExist)
Cole Faustdd569ae2022-01-31 15:48:29 -0800806 return []starlarkNode{processModule(inheritedStaticModule{mi, loadAlways})}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700807 }
Sasha Smundak6609ba72021-07-22 18:32:56 -0700808 }
809
810 // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
811 // source tree that may be a match and the corresponding variable values. For instance, if the source tree
812 // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
813 // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
814 // We then emit the code that loads all of them, e.g.:
815 // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
816 // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
817 // And then inherit it as follows:
818 // _e = {
819 // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
820 // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
821 // if _e:
822 // rblf.inherit(handle, _e[0], _e[1])
823 //
824 var matchingPaths []string
Cole Faust9df1d732022-04-26 16:27:22 -0700825 var needsWarning = false
826 if interpolate, ok := pathExpr.(*interpolateExpr); ok {
827 pathPattern := []string{interpolate.chunks[0]}
828 for _, chunk := range interpolate.chunks[1:] {
829 if chunk != "" {
830 pathPattern = append(pathPattern, chunk)
831 }
832 }
833 if pathPattern[0] == "" && len(ctx.includeTops) > 0 {
834 // If pattern starts from the top. restrict it to the directories where
835 // we know inherit-product uses dynamically calculated path.
836 for _, p := range ctx.includeTops {
837 pathPattern[0] = p
838 matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
839 }
840 } else {
841 matchingPaths = ctx.findMatchingPaths(pathPattern)
842 }
843 needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0
844 } else if len(ctx.includeTops) > 0 {
845 for _, p := range ctx.includeTops {
846 matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{p, ""})...)
847 }
848 } else {
Cole Faustdd569ae2022-01-31 15:48:29 -0800849 return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700850 }
851
Sasha Smundak6609ba72021-07-22 18:32:56 -0700852 // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
Sasha Smundak90be8c52021-08-03 11:06:10 -0700853 const maxMatchingFiles = 150
Sasha Smundak6609ba72021-07-22 18:32:56 -0700854 if len(matchingPaths) > maxMatchingFiles {
Cole Faustdd569ae2022-01-31 15:48:29 -0800855 return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700856 }
Cole Faust93f8d392022-03-02 13:31:30 -0800857
Cole Faust9df1d732022-04-26 16:27:22 -0700858 res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning}
Cole Faust93f8d392022-03-02 13:31:30 -0800859 for _, p := range matchingPaths {
860 // A product configuration files discovered dynamically may attempt to inherit
861 // from another one which does not exist in this source tree. Prevent load errors
862 // by always loading the dynamic files as optional.
863 res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
Sasha Smundak6609ba72021-07-22 18:32:56 -0700864 }
Cole Faust93f8d392022-03-02 13:31:30 -0800865 return []starlarkNode{processModule(res)}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700866}
867
868func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
Cole Faust9b6111a2022-02-02 15:38:33 -0800869 files := ctx.script.makefileFinder.Find(".")
Sasha Smundak6609ba72021-07-22 18:32:56 -0700870 if len(pattern) == 0 {
871 return files
872 }
873
874 // Create regular expression from the pattern
875 s_regexp := "^" + regexp.QuoteMeta(pattern[0])
876 for _, s := range pattern[1:] {
877 s_regexp += ".*" + regexp.QuoteMeta(s)
878 }
879 s_regexp += "$"
880 rex := regexp.MustCompile(s_regexp)
881
882 // Now match
883 var res []string
884 for _, p := range files {
885 if rex.MatchString(p) {
886 res = append(res, p)
887 }
888 }
889 return res
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800890}
891
Cole Faustf035d402022-03-28 14:02:50 -0700892type inheritProductCallParser struct {
893 loadAlways bool
894}
895
896func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, args *mkparser.MakeString) []starlarkNode {
Cole Faust9ebf6e42021-12-13 14:08:34 -0800897 args.TrimLeftSpaces()
898 args.TrimRightSpaces()
899 pathExpr := ctx.parseMakeString(v, args)
900 if _, ok := pathExpr.(*badExpr); ok {
Cole Faustdd569ae2022-01-31 15:48:29 -0800901 return []starlarkNode{ctx.newBadNode(v, "Unable to parse argument to inherit")}
Cole Faust9ebf6e42021-12-13 14:08:34 -0800902 }
Cole Faustf035d402022-03-28 14:02:50 -0700903 return ctx.handleSubConfig(v, pathExpr, p.loadAlways, func(im inheritedModule) starlarkNode {
904 return &inheritNode{im, p.loadAlways}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700905 })
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800906}
907
Cole Faust20052982022-04-22 14:43:55 -0700908func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode {
909 loadAlways := v.Name[0] != '-'
910 return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode {
Cole Faustdd569ae2022-01-31 15:48:29 -0800911 return &includeNode{im, loadAlways}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700912 })
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800913}
914
Cole Faustdd569ae2022-01-31 15:48:29 -0800915func (ctx *parseContext) handleVariable(v *mkparser.Variable) []starlarkNode {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800916 // Handle:
917 // $(call inherit-product,...)
918 // $(call inherit-product-if-exists,...)
919 // $(info xxx)
920 // $(warning xxx)
921 // $(error xxx)
Cole Faust9ebf6e42021-12-13 14:08:34 -0800922 // $(call other-custom-functions,...)
923
Cole Faustf035d402022-03-28 14:02:50 -0700924 if name, args, ok := ctx.maybeParseFunctionCall(v, v.Name); ok {
925 if kf, ok := knownNodeFunctions[name]; ok {
926 return kf.parse(ctx, v, args)
927 }
Cole Faust9ebf6e42021-12-13 14:08:34 -0800928 }
Cole Faustf035d402022-03-28 14:02:50 -0700929
Cole Faustdd569ae2022-01-31 15:48:29 -0800930 return []starlarkNode{&exprNode{expr: ctx.parseReference(v, v.Name)}}
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800931}
932
Cole Faustdd569ae2022-01-31 15:48:29 -0800933func (ctx *parseContext) maybeHandleDefine(directive *mkparser.Directive) starlarkNode {
Sasha Smundakf3e072a2021-07-14 12:50:28 -0700934 macro_name := strings.Fields(directive.Args.Strings[0])[0]
935 // Ignore the macros that we handle
Cole Faust9ebf6e42021-12-13 14:08:34 -0800936 _, ignored := ignoredDefines[macro_name]
937 _, known := knownFunctions[macro_name]
938 if !ignored && !known {
Cole Faustdd569ae2022-01-31 15:48:29 -0800939 return ctx.newBadNode(directive, "define is not supported: %s", macro_name)
Sasha Smundakf3e072a2021-07-14 12:50:28 -0700940 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800941 return nil
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800942}
943
Cole Faustdd569ae2022-01-31 15:48:29 -0800944func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) starlarkNode {
945 ssSwitch := &switchNode{
946 ssCases: []*switchCase{ctx.processBranch(ifDirective)},
947 }
948 for ctx.hasNodes() && ctx.fatalError == nil {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800949 node := ctx.getNode()
950 switch x := node.(type) {
951 case *mkparser.Directive:
952 switch x.Name {
953 case "else", "elifdef", "elifndef", "elifeq", "elifneq":
Cole Faustdd569ae2022-01-31 15:48:29 -0800954 ssSwitch.ssCases = append(ssSwitch.ssCases, ctx.processBranch(x))
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800955 case "endif":
Cole Faustdd569ae2022-01-31 15:48:29 -0800956 return ssSwitch
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800957 default:
Cole Faustdd569ae2022-01-31 15:48:29 -0800958 return ctx.newBadNode(node, "unexpected directive %s", x.Name)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800959 }
960 default:
Cole Faustdd569ae2022-01-31 15:48:29 -0800961 return ctx.newBadNode(ifDirective, "unexpected statement")
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800962 }
963 }
964 if ctx.fatalError == nil {
965 ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
966 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800967 return ctx.newBadNode(ifDirective, "no matching endif for %s", ifDirective.Dump())
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800968}
969
970// processBranch processes a single branch (if/elseif/else) until the next directive
971// on the same level.
Cole Faustdd569ae2022-01-31 15:48:29 -0800972func (ctx *parseContext) processBranch(check *mkparser.Directive) *switchCase {
973 block := &switchCase{gate: ctx.parseCondition(check)}
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800974 defer func() {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800975 ctx.ifNestLevel--
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800976 }()
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800977 ctx.ifNestLevel++
978
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800979 for ctx.hasNodes() {
980 node := ctx.getNode()
Cole Faust591a1fe2021-11-08 15:37:57 -0800981 if d, ok := node.(*mkparser.Directive); ok {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800982 switch d.Name {
983 case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800984 ctx.backNode()
Cole Faustdd569ae2022-01-31 15:48:29 -0800985 return block
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800986 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800987 }
Cole Faustdd569ae2022-01-31 15:48:29 -0800988 block.nodes = append(block.nodes, ctx.handleSimpleStatement(node)...)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800989 }
990 ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
Cole Faustdd569ae2022-01-31 15:48:29 -0800991 return block
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800992}
993
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800994func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
995 switch check.Name {
996 case "ifdef", "ifndef", "elifdef", "elifndef":
Cole Faust71514c02022-01-27 17:21:41 -0800997 if !check.Args.Const() {
Cole Faustdd569ae2022-01-31 15:48:29 -0800998 return ctx.newBadNode(check, "ifdef variable ref too complex: %s", check.Args.Dump())
Cole Faust71514c02022-01-27 17:21:41 -0800999 }
Cole Faustf0632662022-04-07 13:59:24 -07001000 v := NewVariableRefExpr(ctx.addVariable(check.Args.Strings[0]))
Cole Faust71514c02022-01-27 17:21:41 -08001001 if strings.HasSuffix(check.Name, "ndef") {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001002 v = &notExpr{v}
1003 }
1004 return &ifNode{
1005 isElif: strings.HasPrefix(check.Name, "elif"),
1006 expr: v,
1007 }
1008 case "ifeq", "ifneq", "elifeq", "elifneq":
1009 return &ifNode{
1010 isElif: strings.HasPrefix(check.Name, "elif"),
1011 expr: ctx.parseCompare(check),
1012 }
1013 case "else":
1014 return &elseNode{}
1015 default:
1016 panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
1017 }
1018}
1019
1020func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001021 if ctx.errorLogger != nil {
Sasha Smundak422b6142021-11-11 18:31:59 -08001022 ctx.errorLogger.NewError(ctx.errorLocation(node), node, text, args...)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001023 }
1024 ctx.script.hasErrors = true
Cole Faustdd569ae2022-01-31 15:48:29 -08001025 return &badExpr{errorLocation: ctx.errorLocation(node), message: fmt.Sprintf(text, args...)}
1026}
1027
1028// records that the given node failed to be converted and includes an explanatory message
1029func (ctx *parseContext) newBadNode(failedNode mkparser.Node, message string, args ...interface{}) starlarkNode {
1030 return &exprNode{ctx.newBadExpr(failedNode, message, args...)}
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001031}
1032
1033func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
1034 // Strip outer parentheses
1035 mkArg := cloneMakeString(cond.Args)
1036 mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
1037 n := len(mkArg.Strings)
1038 mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
1039 args := mkArg.Split(",")
1040 // TODO(asmundak): handle the case where the arguments are in quotes and space-separated
1041 if len(args) != 2 {
1042 return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
1043 }
1044 args[0].TrimRightSpaces()
1045 args[1].TrimLeftSpaces()
1046
1047 isEq := !strings.HasSuffix(cond.Name, "neq")
Cole Faustf8320212021-11-10 15:05:07 -08001048 xLeft := ctx.parseMakeString(cond, args[0])
1049 xRight := ctx.parseMakeString(cond, args[1])
1050 if bad, ok := xLeft.(*badExpr); ok {
1051 return bad
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001052 }
Cole Faustf8320212021-11-10 15:05:07 -08001053 if bad, ok := xRight.(*badExpr); ok {
1054 return bad
1055 }
1056
1057 if expr, ok := ctx.parseCompareSpecialCases(cond, xLeft, xRight); ok {
1058 return expr
1059 }
1060
Cole Faust9ebf6e42021-12-13 14:08:34 -08001061 var stringOperand string
1062 var otherOperand starlarkExpr
1063 if s, ok := maybeString(xLeft); ok {
1064 stringOperand = s
1065 otherOperand = xRight
1066 } else if s, ok := maybeString(xRight); ok {
1067 stringOperand = s
1068 otherOperand = xLeft
1069 }
1070
Cole Faust9ebf6e42021-12-13 14:08:34 -08001071 // If we've identified one of the operands as being a string literal, check
1072 // for some special cases we can do to simplify the resulting expression.
1073 if otherOperand != nil {
1074 if stringOperand == "" {
1075 if isEq {
Cole Faustf035d402022-03-28 14:02:50 -07001076 return negateExpr(otherOperand)
Cole Faust9ebf6e42021-12-13 14:08:34 -08001077 } else {
1078 return otherOperand
1079 }
1080 }
1081 if stringOperand == "true" && otherOperand.typ() == starlarkTypeBool {
1082 if !isEq {
Cole Faustf035d402022-03-28 14:02:50 -07001083 return negateExpr(otherOperand)
Cole Faust9ebf6e42021-12-13 14:08:34 -08001084 } else {
1085 return otherOperand
1086 }
1087 }
Cole Fausta99afdf2022-04-26 12:06:49 -07001088 if otherOperand.typ() == starlarkTypeList {
1089 fields := strings.Fields(stringOperand)
1090 elements := make([]starlarkExpr, len(fields))
1091 for i, s := range fields {
1092 elements[i] = &stringLiteralExpr{literal: s}
1093 }
1094 return &eqExpr{
1095 left: otherOperand,
1096 right: &listExpr{elements},
1097 isEq: isEq,
1098 }
1099 }
Cole Faustb1103e22022-01-06 15:22:05 -08001100 if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt {
1101 return &eqExpr{
1102 left: otherOperand,
1103 right: &intLiteralExpr{literal: intOperand},
1104 isEq: isEq,
1105 }
1106 }
Cole Faust9ebf6e42021-12-13 14:08:34 -08001107 }
1108
Cole Faustf8320212021-11-10 15:05:07 -08001109 return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001110}
1111
Cole Faustf8320212021-11-10 15:05:07 -08001112// Given an if statement's directive and the left/right starlarkExprs,
1113// check if the starlarkExprs are one of a few hardcoded special cases
Cole Faust9932f752022-02-08 11:56:25 -08001114// that can be converted to a simpler equality expression than simply comparing
Cole Faustf8320212021-11-10 15:05:07 -08001115// the two.
1116func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, left starlarkExpr,
1117 right starlarkExpr) (starlarkExpr, bool) {
1118 isEq := !strings.HasSuffix(directive.Name, "neq")
1119
1120 // All the special cases require a call on one side and a
1121 // string literal/variable on the other. Turn the left/right variables into
1122 // call/value variables, and return false if that's not possible.
1123 var value starlarkExpr = nil
1124 call, ok := left.(*callExpr)
1125 if ok {
1126 switch right.(type) {
1127 case *stringLiteralExpr, *variableRefExpr:
1128 value = right
1129 }
1130 } else {
1131 call, _ = right.(*callExpr)
1132 switch left.(type) {
1133 case *stringLiteralExpr, *variableRefExpr:
1134 value = left
1135 }
1136 }
1137
1138 if call == nil || value == nil {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001139 return nil, false
1140 }
Cole Faustf8320212021-11-10 15:05:07 -08001141
Cole Faustf8320212021-11-10 15:05:07 -08001142 switch call.name {
Cole Faust9932f752022-02-08 11:56:25 -08001143 case baseName + ".filter":
1144 return ctx.parseCompareFilterFuncResult(directive, call, value, isEq)
Cole Faust9ebf6e42021-12-13 14:08:34 -08001145 case baseName + ".findstring":
Cole Faustf8320212021-11-10 15:05:07 -08001146 return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true
Cole Faust9ebf6e42021-12-13 14:08:34 -08001147 case baseName + ".strip":
Cole Faustf8320212021-11-10 15:05:07 -08001148 return ctx.parseCompareStripFuncResult(directive, call, value, !isEq), true
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001149 }
Cole Faustf8320212021-11-10 15:05:07 -08001150 return nil, false
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001151}
1152
1153func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
Cole Faust9932f752022-02-08 11:56:25 -08001154 filterFuncCall *callExpr, xValue starlarkExpr, negate bool) (starlarkExpr, bool) {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001155 // We handle:
Sasha Smundak0554d762021-07-08 18:26:12 -07001156 // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...]
1157 // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...]
Cole Faust9932f752022-02-08 11:56:25 -08001158 if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
1159 return nil, false
1160 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001161 xPattern := filterFuncCall.args[0]
1162 xText := filterFuncCall.args[1]
1163 var xInList *stringLiteralExpr
Sasha Smundak0554d762021-07-08 18:26:12 -07001164 var expr starlarkExpr
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001165 var ok bool
Cole Faust9932f752022-02-08 11:56:25 -08001166 if xInList, ok = xPattern.(*stringLiteralExpr); ok && !strings.ContainsRune(xInList.literal, '%') && xText.typ() == starlarkTypeList {
1167 expr = xText
1168 } else if xInList, ok = xText.(*stringLiteralExpr); ok {
1169 expr = xPattern
1170 } else {
1171 return nil, false
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001172 }
Cole Faust9932f752022-02-08 11:56:25 -08001173 slExpr := newStringListExpr(strings.Fields(xInList.literal))
1174 // Generate simpler code for the common cases:
1175 if expr.typ() == starlarkTypeList {
1176 if len(slExpr.items) == 1 {
1177 // Checking that a string belongs to list
1178 return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}, true
Sasha Smundak0554d762021-07-08 18:26:12 -07001179 } else {
Cole Faust9932f752022-02-08 11:56:25 -08001180 return nil, false
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001181 }
Cole Faust9932f752022-02-08 11:56:25 -08001182 } else if len(slExpr.items) == 1 {
1183 return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}, true
1184 } else {
1185 return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}, true
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001186 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001187}
1188
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001189func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
1190 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
Sasha Smundak0554d762021-07-08 18:26:12 -07001191 if isEmptyString(xValue) {
1192 return &eqExpr{
1193 left: &callExpr{
1194 object: xCall.args[1],
1195 name: "find",
1196 args: []starlarkExpr{xCall.args[0]},
1197 returnType: starlarkTypeInt,
1198 },
1199 right: &intLiteralExpr{-1},
1200 isEq: !negate,
1201 }
Cole Faust0e9418c2021-12-13 16:33:25 -08001202 } else if s, ok := maybeString(xValue); ok {
1203 if s2, ok := maybeString(xCall.args[0]); ok && s == s2 {
1204 return &eqExpr{
1205 left: &callExpr{
1206 object: xCall.args[1],
1207 name: "find",
1208 args: []starlarkExpr{xCall.args[0]},
1209 returnType: starlarkTypeInt,
1210 },
1211 right: &intLiteralExpr{-1},
1212 isEq: negate,
1213 }
1214 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001215 }
Cole Faust0e9418c2021-12-13 16:33:25 -08001216 return ctx.newBadExpr(directive, "$(findstring) can only be compared to nothing or its first argument")
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001217}
1218
1219func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
1220 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
1221 if _, ok := xValue.(*stringLiteralExpr); !ok {
1222 return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
1223 }
1224 return &eqExpr{
1225 left: &callExpr{
1226 name: "strip",
1227 args: xCall.args,
1228 returnType: starlarkTypeString,
1229 },
1230 right: xValue, isEq: !negate}
1231}
1232
Cole Faustf035d402022-03-28 14:02:50 -07001233func (ctx *parseContext) maybeParseFunctionCall(node mkparser.Node, ref *mkparser.MakeString) (name string, args *mkparser.MakeString, ok bool) {
1234 ref.TrimLeftSpaces()
1235 ref.TrimRightSpaces()
1236
1237 words := ref.SplitN(" ", 2)
1238 if !words[0].Const() {
1239 return "", nil, false
1240 }
1241
1242 name = words[0].Dump()
1243 args = mkparser.SimpleMakeString("", words[0].Pos())
1244 if len(words) >= 2 {
1245 args = words[1]
1246 }
1247 args.TrimLeftSpaces()
1248 if name == "call" {
1249 words = args.SplitN(",", 2)
1250 if words[0].Empty() || !words[0].Const() {
1251 return "", nil, false
1252 }
1253 name = words[0].Dump()
1254 if len(words) < 2 {
Cole Faust6c41b8a2022-04-13 13:53:48 -07001255 args = mkparser.SimpleMakeString("", words[0].Pos())
Cole Faustf035d402022-03-28 14:02:50 -07001256 } else {
1257 args = words[1]
1258 }
1259 }
1260 ok = true
1261 return
1262}
1263
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001264// parses $(...), returning an expression
1265func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
1266 ref.TrimLeftSpaces()
1267 ref.TrimRightSpaces()
1268 refDump := ref.Dump()
1269
1270 // Handle only the case where the first (or only) word is constant
1271 words := ref.SplitN(" ", 2)
1272 if !words[0].Const() {
Cole Faust13238772022-04-28 14:29:57 -07001273 if len(words) == 1 {
1274 expr := ctx.parseMakeString(node, ref)
1275 return &callExpr{
1276 object: &identifierExpr{"cfg"},
1277 name: "get",
1278 args: []starlarkExpr{
1279 expr,
1280 &callExpr{
1281 object: &identifierExpr{"g"},
1282 name: "get",
1283 args: []starlarkExpr{
1284 expr,
1285 &stringLiteralExpr{literal: ""},
1286 },
1287 returnType: starlarkTypeUnknown,
1288 },
1289 },
1290 returnType: starlarkTypeUnknown,
1291 }
1292 } else {
1293 return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
1294 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001295 }
1296
Cole Faust1e275862022-04-26 14:28:04 -07001297 if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok {
1298 if _, unsupported := unsupportedFunctions[name]; unsupported {
1299 return ctx.newBadExpr(node, "%s is not supported", refDump)
1300 }
1301 }
1302
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001303 // If it is a single word, it can be a simple variable
1304 // reference or a function call
Cole Faustf035d402022-03-28 14:02:50 -07001305 if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" {
Sasha Smundak65b547e2021-09-17 15:35:41 -07001306 if strings.HasPrefix(refDump, soongNsPrefix) {
1307 // TODO (asmundak): if we find many, maybe handle them.
Cole Faustc00184e2021-11-08 12:08:57 -08001308 return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: %s", refDump)
Sasha Smundak65b547e2021-09-17 15:35:41 -07001309 }
Cole Faustc36c9622021-12-07 15:20:45 -08001310 // Handle substitution references: https://www.gnu.org/software/make/manual/html_node/Substitution-Refs.html
1311 if strings.Contains(refDump, ":") {
1312 parts := strings.SplitN(refDump, ":", 2)
1313 substParts := strings.SplitN(parts[1], "=", 2)
1314 if len(substParts) < 2 || strings.Count(substParts[0], "%") > 1 {
1315 return ctx.newBadExpr(node, "Invalid substitution reference")
1316 }
1317 if !strings.Contains(substParts[0], "%") {
1318 if strings.Contains(substParts[1], "%") {
1319 return ctx.newBadExpr(node, "A substitution reference must have a %% in the \"before\" part of the substitution if it has one in the \"after\" part.")
1320 }
1321 substParts[0] = "%" + substParts[0]
1322 substParts[1] = "%" + substParts[1]
1323 }
1324 v := ctx.addVariable(parts[0])
1325 if v == nil {
1326 return ctx.newBadExpr(node, "unknown variable %s", refDump)
1327 }
1328 return &callExpr{
Cole Faust9ebf6e42021-12-13 14:08:34 -08001329 name: baseName + ".mkpatsubst",
1330 returnType: starlarkTypeString,
Cole Faustc36c9622021-12-07 15:20:45 -08001331 args: []starlarkExpr{
1332 &stringLiteralExpr{literal: substParts[0]},
1333 &stringLiteralExpr{literal: substParts[1]},
Cole Faustf0632662022-04-07 13:59:24 -07001334 NewVariableRefExpr(v),
Cole Faustc36c9622021-12-07 15:20:45 -08001335 },
1336 }
1337 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001338 if v := ctx.addVariable(refDump); v != nil {
Cole Faustf0632662022-04-07 13:59:24 -07001339 return NewVariableRefExpr(v)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001340 }
1341 return ctx.newBadExpr(node, "unknown variable %s", refDump)
1342 }
1343
Cole Faustf035d402022-03-28 14:02:50 -07001344 if name, args, ok := ctx.maybeParseFunctionCall(node, ref); ok {
1345 if kf, found := knownFunctions[name]; found {
1346 return kf.parse(ctx, node, args)
Sasha Smundak6609ba72021-07-22 18:32:56 -07001347 } else {
Cole Faustf035d402022-03-28 14:02:50 -07001348 return ctx.newBadExpr(node, "cannot handle invoking %s", name)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001349 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001350 }
Cole Faust1e275862022-04-26 14:28:04 -07001351 return ctx.newBadExpr(node, "cannot handle %s", refDump)
Cole Faust9ebf6e42021-12-13 14:08:34 -08001352}
1353
1354type simpleCallParser struct {
1355 name string
1356 returnType starlarkType
1357 addGlobals bool
Cole Faust1cc08852022-02-28 11:12:08 -08001358 addHandle bool
Cole Faust9ebf6e42021-12-13 14:08:34 -08001359}
1360
1361func (p *simpleCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1362 expr := &callExpr{name: p.name, returnType: p.returnType}
1363 if p.addGlobals {
1364 expr.args = append(expr.args, &globalsExpr{})
1365 }
Cole Faust1cc08852022-02-28 11:12:08 -08001366 if p.addHandle {
1367 expr.args = append(expr.args, &identifierExpr{name: "handle"})
1368 }
Cole Faust9ebf6e42021-12-13 14:08:34 -08001369 for _, arg := range args.Split(",") {
1370 arg.TrimLeftSpaces()
1371 arg.TrimRightSpaces()
1372 x := ctx.parseMakeString(node, arg)
1373 if xBad, ok := x.(*badExpr); ok {
1374 return xBad
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001375 }
Cole Faust9ebf6e42021-12-13 14:08:34 -08001376 expr.args = append(expr.args, x)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001377 }
1378 return expr
1379}
1380
Cole Faust9ebf6e42021-12-13 14:08:34 -08001381type makeControlFuncParser struct {
1382 name string
1383}
1384
1385func (p *makeControlFuncParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1386 // Make control functions need special treatment as everything
1387 // after the name is a single text argument
1388 x := ctx.parseMakeString(node, args)
1389 if xBad, ok := x.(*badExpr); ok {
1390 return xBad
1391 }
1392 return &callExpr{
1393 name: p.name,
1394 args: []starlarkExpr{
1395 &stringLiteralExpr{ctx.script.mkFile},
1396 x,
1397 },
1398 returnType: starlarkTypeUnknown,
1399 }
1400}
1401
1402type shellCallParser struct{}
1403
1404func (p *shellCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1405 // Shell functions need special treatment as everything
1406 // after the name is a single text argument
1407 x := ctx.parseMakeString(node, args)
1408 if xBad, ok := x.(*badExpr); ok {
1409 return xBad
1410 }
1411 return &callExpr{
1412 name: baseName + ".shell",
1413 args: []starlarkExpr{x},
1414 returnType: starlarkTypeUnknown,
1415 }
1416}
1417
1418type myDirCallParser struct{}
1419
1420func (p *myDirCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1421 if !args.Empty() {
1422 return ctx.newBadExpr(node, "my-dir function cannot have any arguments passed to it.")
1423 }
Cole Faustf5adedc2022-03-18 14:05:06 -07001424 return &stringLiteralExpr{literal: filepath.Dir(ctx.script.mkFile)}
Cole Faust9ebf6e42021-12-13 14:08:34 -08001425}
1426
Cole Faust9ebf6e42021-12-13 14:08:34 -08001427type isProductInListCallParser struct{}
1428
1429func (p *isProductInListCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1430 if args.Empty() {
1431 return ctx.newBadExpr(node, "is-product-in-list requires an argument")
1432 }
1433 return &inExpr{
Cole Faustf0632662022-04-07 13:59:24 -07001434 expr: NewVariableRefExpr(ctx.addVariable("TARGET_PRODUCT")),
Cole Faust9ebf6e42021-12-13 14:08:34 -08001435 list: maybeConvertToStringList(ctx.parseMakeString(node, args)),
1436 isNot: false,
1437 }
1438}
1439
1440type isVendorBoardPlatformCallParser struct{}
1441
1442func (p *isVendorBoardPlatformCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1443 if args.Empty() || !identifierFullMatchRegex.MatchString(args.Dump()) {
1444 return ctx.newBadExpr(node, "cannot handle non-constant argument to is-vendor-board-platform")
1445 }
1446 return &inExpr{
Cole Faustf0632662022-04-07 13:59:24 -07001447 expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")),
1448 list: NewVariableRefExpr(ctx.addVariable(args.Dump() + "_BOARD_PLATFORMS")),
Cole Faust9ebf6e42021-12-13 14:08:34 -08001449 isNot: false,
1450 }
1451}
1452
1453type isVendorBoardQcomCallParser struct{}
1454
1455func (p *isVendorBoardQcomCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1456 if !args.Empty() {
1457 return ctx.newBadExpr(node, "is-vendor-board-qcom does not accept any arguments")
1458 }
1459 return &inExpr{
Cole Faustf0632662022-04-07 13:59:24 -07001460 expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")),
1461 list: NewVariableRefExpr(ctx.addVariable("QCOM_BOARD_PLATFORMS")),
Cole Faust9ebf6e42021-12-13 14:08:34 -08001462 isNot: false,
1463 }
1464}
1465
1466type substCallParser struct {
1467 fname string
1468}
1469
1470func (p *substCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001471 words := args.Split(",")
1472 if len(words) != 3 {
Cole Faust9ebf6e42021-12-13 14:08:34 -08001473 return ctx.newBadExpr(node, "%s function should have 3 arguments", p.fname)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001474 }
Sasha Smundak35434ed2021-11-05 16:29:56 -07001475 from := ctx.parseMakeString(node, words[0])
1476 if xBad, ok := from.(*badExpr); ok {
1477 return xBad
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001478 }
Sasha Smundak35434ed2021-11-05 16:29:56 -07001479 to := ctx.parseMakeString(node, words[1])
1480 if xBad, ok := to.(*badExpr); ok {
1481 return xBad
1482 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001483 words[2].TrimLeftSpaces()
1484 words[2].TrimRightSpaces()
1485 obj := ctx.parseMakeString(node, words[2])
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001486 typ := obj.typ()
Cole Faust9ebf6e42021-12-13 14:08:34 -08001487 if typ == starlarkTypeString && p.fname == "subst" {
Sasha Smundak94b41c72021-07-12 18:30:42 -07001488 // Optimization: if it's $(subst from, to, string), emit string.replace(from, to)
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001489 return &callExpr{
1490 object: obj,
1491 name: "replace",
Sasha Smundak35434ed2021-11-05 16:29:56 -07001492 args: []starlarkExpr{from, to},
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001493 returnType: typ,
1494 }
1495 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001496 return &callExpr{
Cole Faust9ebf6e42021-12-13 14:08:34 -08001497 name: baseName + ".mk" + p.fname,
Sasha Smundak35434ed2021-11-05 16:29:56 -07001498 args: []starlarkExpr{from, to, obj},
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001499 returnType: obj.typ(),
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001500 }
1501}
1502
Cole Faust9ebf6e42021-12-13 14:08:34 -08001503type ifCallParser struct{}
1504
1505func (p *ifCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
Cole Faust4eadba72021-12-07 11:54:52 -08001506 words := args.Split(",")
1507 if len(words) != 2 && len(words) != 3 {
1508 return ctx.newBadExpr(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))
1509 }
1510 condition := ctx.parseMakeString(node, words[0])
1511 ifTrue := ctx.parseMakeString(node, words[1])
1512 var ifFalse starlarkExpr
1513 if len(words) == 3 {
1514 ifFalse = ctx.parseMakeString(node, words[2])
1515 } else {
1516 switch ifTrue.typ() {
1517 case starlarkTypeList:
1518 ifFalse = &listExpr{items: []starlarkExpr{}}
1519 case starlarkTypeInt:
1520 ifFalse = &intLiteralExpr{literal: 0}
1521 case starlarkTypeBool:
1522 ifFalse = &boolLiteralExpr{literal: false}
1523 default:
1524 ifFalse = &stringLiteralExpr{literal: ""}
1525 }
1526 }
1527 return &ifExpr{
1528 condition,
1529 ifTrue,
1530 ifFalse,
1531 }
1532}
1533
Cole Faustf035d402022-03-28 14:02:50 -07001534type ifCallNodeParser struct{}
Cole Faust9ebf6e42021-12-13 14:08:34 -08001535
Cole Faustf035d402022-03-28 14:02:50 -07001536func (p *ifCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
1537 words := args.Split(",")
1538 if len(words) != 2 && len(words) != 3 {
1539 return []starlarkNode{ctx.newBadNode(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))}
1540 }
1541
1542 ifn := &ifNode{expr: ctx.parseMakeString(node, words[0])}
1543 cases := []*switchCase{
1544 {
1545 gate: ifn,
1546 nodes: ctx.parseNodeMakeString(node, words[1]),
1547 },
1548 }
1549 if len(words) == 3 {
1550 cases = append(cases, &switchCase{
1551 gate: &elseNode{},
1552 nodes: ctx.parseNodeMakeString(node, words[2]),
1553 })
1554 }
1555 if len(cases) == 2 {
1556 if len(cases[1].nodes) == 0 {
1557 // Remove else branch if it has no contents
1558 cases = cases[:1]
1559 } else if len(cases[0].nodes) == 0 {
1560 // If the if branch has no contents but the else does,
1561 // move them to the if and negate its condition
1562 ifn.expr = negateExpr(ifn.expr)
1563 cases[0].nodes = cases[1].nodes
1564 cases = cases[:1]
1565 }
1566 }
1567
1568 return []starlarkNode{&switchNode{ssCases: cases}}
1569}
1570
1571type foreachCallParser struct{}
1572
1573func (p *foreachCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
Cole Faustb0d32ab2021-12-09 14:00:59 -08001574 words := args.Split(",")
1575 if len(words) != 3 {
1576 return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
1577 }
1578 if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
1579 return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier")
1580 }
1581 loopVarName := words[0].Strings[0]
1582 list := ctx.parseMakeString(node, words[1])
1583 action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr {
1584 if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
1585 return &identifierExpr{loopVarName}
1586 }
1587 return nil
1588 })
1589
1590 if list.typ() != starlarkTypeList {
1591 list = &callExpr{
Cole Faust9ebf6e42021-12-13 14:08:34 -08001592 name: baseName + ".words",
1593 returnType: starlarkTypeList,
Cole Faustb0d32ab2021-12-09 14:00:59 -08001594 args: []starlarkExpr{list},
1595 }
1596 }
1597
Cole Faust72374fc2022-05-05 11:45:04 -07001598 var result starlarkExpr = &foreachExpr{
Cole Faustb0d32ab2021-12-09 14:00:59 -08001599 varName: loopVarName,
1600 list: list,
1601 action: action,
1602 }
Cole Faust72374fc2022-05-05 11:45:04 -07001603
1604 if action.typ() == starlarkTypeList {
1605 result = &callExpr{
1606 name: baseName + ".flatten_2d_list",
1607 args: []starlarkExpr{result},
1608 returnType: starlarkTypeList,
1609 }
1610 }
1611
1612 return result
Cole Faustb0d32ab2021-12-09 14:00:59 -08001613}
1614
Cole Faustf035d402022-03-28 14:02:50 -07001615func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starlarkExpr) {
1616 switch a := node.(type) {
1617 case *ifNode:
1618 a.expr = a.expr.transform(transformer)
1619 case *switchCase:
1620 transformNode(a.gate, transformer)
1621 for _, n := range a.nodes {
1622 transformNode(n, transformer)
1623 }
1624 case *switchNode:
1625 for _, n := range a.ssCases {
1626 transformNode(n, transformer)
1627 }
1628 case *exprNode:
1629 a.expr = a.expr.transform(transformer)
1630 case *assignmentNode:
1631 a.value = a.value.transform(transformer)
1632 case *foreachNode:
1633 a.list = a.list.transform(transformer)
1634 for _, n := range a.actions {
1635 transformNode(n, transformer)
1636 }
Cole Faust9df1d732022-04-26 16:27:22 -07001637 case *inheritNode:
1638 if b, ok := a.module.(inheritedDynamicModule); ok {
1639 b.path = b.path.transform(transformer)
1640 a.module = b
1641 }
1642 case *includeNode:
1643 if b, ok := a.module.(inheritedDynamicModule); ok {
1644 b.path = b.path.transform(transformer)
1645 a.module = b
1646 }
Cole Faustf035d402022-03-28 14:02:50 -07001647 }
1648}
1649
1650type foreachCallNodeParser struct{}
1651
1652func (p *foreachCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
1653 words := args.Split(",")
1654 if len(words) != 3 {
1655 return []starlarkNode{ctx.newBadNode(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))}
1656 }
1657 if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
1658 return []starlarkNode{ctx.newBadNode(node, "first argument to foreach function must be a simple string identifier")}
1659 }
1660
1661 loopVarName := words[0].Strings[0]
1662
1663 list := ctx.parseMakeString(node, words[1])
1664 if list.typ() != starlarkTypeList {
1665 list = &callExpr{
1666 name: baseName + ".words",
1667 returnType: starlarkTypeList,
1668 args: []starlarkExpr{list},
1669 }
1670 }
1671
1672 actions := ctx.parseNodeMakeString(node, words[2])
1673 // TODO(colefaust): Replace transforming code with something more elegant
1674 for _, action := range actions {
1675 transformNode(action, func(expr starlarkExpr) starlarkExpr {
1676 if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
1677 return &identifierExpr{loopVarName}
1678 }
1679 return nil
1680 })
1681 }
1682
1683 return []starlarkNode{&foreachNode{
1684 varName: loopVarName,
1685 list: list,
1686 actions: actions,
1687 }}
1688}
1689
Cole Faust9ebf6e42021-12-13 14:08:34 -08001690type wordCallParser struct{}
1691
1692func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001693 words := args.Split(",")
1694 if len(words) != 2 {
1695 return ctx.newBadExpr(node, "word function should have 2 arguments")
1696 }
Cole Faust94c4a9a2022-04-22 17:43:52 -07001697 var index = 0
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001698 if words[0].Const() {
Cole Faust94c4a9a2022-04-22 17:43:52 -07001699 if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil {
1700 index = i
1701 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001702 }
1703 if index < 1 {
1704 return ctx.newBadExpr(node, "word index should be constant positive integer")
1705 }
1706 words[1].TrimLeftSpaces()
1707 words[1].TrimRightSpaces()
1708 array := ctx.parseMakeString(node, words[1])
Cole Faust94c4a9a2022-04-22 17:43:52 -07001709 if bad, ok := array.(*badExpr); ok {
1710 return bad
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001711 }
1712 if array.typ() != starlarkTypeList {
Cole Faust94c4a9a2022-04-22 17:43:52 -07001713 array = &callExpr{
1714 name: baseName + ".words",
1715 args: []starlarkExpr{array},
1716 returnType: starlarkTypeList,
1717 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001718 }
Cole Faust94c4a9a2022-04-22 17:43:52 -07001719 return &indexExpr{array, &intLiteralExpr{index - 1}}
1720}
1721
1722type wordsCallParser struct{}
1723
1724func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1725 args.TrimLeftSpaces()
1726 args.TrimRightSpaces()
1727 array := ctx.parseMakeString(node, args)
1728 if bad, ok := array.(*badExpr); ok {
1729 return bad
1730 }
1731 if array.typ() != starlarkTypeList {
1732 array = &callExpr{
1733 name: baseName + ".words",
1734 args: []starlarkExpr{array},
1735 returnType: starlarkTypeList,
1736 }
1737 }
1738 return &callExpr{
1739 name: "len",
1740 args: []starlarkExpr{array},
1741 returnType: starlarkTypeInt,
1742 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001743}
1744
Cole Faustb1103e22022-01-06 15:22:05 -08001745func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) {
1746 parsedArgs := make([]starlarkExpr, 0)
1747 for _, arg := range args.Split(",") {
1748 expr := ctx.parseMakeString(node, arg)
1749 if expr.typ() == starlarkTypeList {
1750 return nil, fmt.Errorf("argument to math argument has type list, which cannot be converted to int")
1751 }
1752 if s, ok := maybeString(expr); ok {
1753 intVal, err := strconv.Atoi(strings.TrimSpace(s))
1754 if err != nil {
1755 return nil, err
1756 }
1757 expr = &intLiteralExpr{literal: intVal}
1758 } else if expr.typ() != starlarkTypeInt {
1759 expr = &callExpr{
1760 name: "int",
1761 args: []starlarkExpr{expr},
1762 returnType: starlarkTypeInt,
1763 }
1764 }
1765 parsedArgs = append(parsedArgs, expr)
1766 }
1767 if len(parsedArgs) != expectedArgs {
1768 return nil, fmt.Errorf("function should have %d arguments", expectedArgs)
1769 }
1770 return parsedArgs, nil
1771}
1772
1773type mathComparisonCallParser struct {
1774 op string
1775}
1776
1777func (p *mathComparisonCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1778 parsedArgs, err := parseIntegerArguments(ctx, node, args, 2)
1779 if err != nil {
1780 return ctx.newBadExpr(node, err.Error())
1781 }
1782 return &binaryOpExpr{
1783 left: parsedArgs[0],
1784 right: parsedArgs[1],
1785 op: p.op,
1786 returnType: starlarkTypeBool,
1787 }
1788}
1789
1790type mathMaxOrMinCallParser struct {
1791 function string
1792}
1793
1794func (p *mathMaxOrMinCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1795 parsedArgs, err := parseIntegerArguments(ctx, node, args, 2)
1796 if err != nil {
1797 return ctx.newBadExpr(node, err.Error())
1798 }
1799 return &callExpr{
1800 object: nil,
1801 name: p.function,
1802 args: parsedArgs,
1803 returnType: starlarkTypeInt,
1804 }
1805}
1806
Cole Faustf035d402022-03-28 14:02:50 -07001807type evalNodeParser struct{}
1808
1809func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
1810 parser := mkparser.NewParser("Eval expression", strings.NewReader(args.Dump()))
1811 nodes, errs := parser.Parse()
1812 if errs != nil {
1813 return []starlarkNode{ctx.newBadNode(node, "Unable to parse eval statement")}
1814 }
1815
1816 if len(nodes) == 0 {
1817 return []starlarkNode{}
1818 } else if len(nodes) == 1 {
1819 switch n := nodes[0].(type) {
1820 case *mkparser.Assignment:
1821 if n.Name.Const() {
1822 return ctx.handleAssignment(n)
1823 }
1824 case *mkparser.Comment:
1825 return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}}
Cole Faust20052982022-04-22 14:43:55 -07001826 case *mkparser.Directive:
1827 if n.Name == "include" || n.Name == "-include" {
1828 return ctx.handleInclude(n)
1829 }
1830 case *mkparser.Variable:
1831 // Technically inherit-product(-if-exists) don't need to be put inside
1832 // an eval, but some makefiles do it, presumably because they copy+pasted
1833 // from a $(eval include ...)
1834 if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok {
1835 if name == "inherit-product" || name == "inherit-product-if-exists" {
1836 return ctx.handleVariable(n)
1837 }
1838 }
Cole Faustf035d402022-03-28 14:02:50 -07001839 }
1840 }
1841
Cole Faust20052982022-04-22 14:43:55 -07001842 return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")}
Cole Faustf035d402022-03-28 14:02:50 -07001843}
1844
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001845func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
1846 if mk.Const() {
1847 return &stringLiteralExpr{mk.Dump()}
1848 }
1849 if mkRef, ok := mk.SingleVariable(); ok {
1850 return ctx.parseReference(node, mkRef)
1851 }
1852 // If we reached here, it's neither string literal nor a simple variable,
1853 // we need a full-blown interpolation node that will generate
1854 // "a%b%c" % (X, Y) for a$(X)b$(Y)c
Cole Faustfc438682021-12-14 12:46:32 -08001855 parts := make([]starlarkExpr, len(mk.Variables)+len(mk.Strings))
1856 for i := 0; i < len(parts); i++ {
1857 if i%2 == 0 {
1858 parts[i] = &stringLiteralExpr{literal: mk.Strings[i/2]}
1859 } else {
1860 parts[i] = ctx.parseReference(node, mk.Variables[i/2].Name)
1861 if x, ok := parts[i].(*badExpr); ok {
1862 return x
1863 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001864 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001865 }
Cole Faustfc438682021-12-14 12:46:32 -08001866 return NewInterpolateExpr(parts)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001867}
1868
Cole Faustf035d402022-03-28 14:02:50 -07001869func (ctx *parseContext) parseNodeMakeString(node mkparser.Node, mk *mkparser.MakeString) []starlarkNode {
1870 // Discard any constant values in the make string, as they would be top level
1871 // string literals and do nothing.
1872 result := make([]starlarkNode, 0, len(mk.Variables))
1873 for i := range mk.Variables {
1874 result = append(result, ctx.handleVariable(&mk.Variables[i])...)
1875 }
1876 return result
1877}
1878
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001879// Handles the statements whose treatment is the same in all contexts: comment,
1880// assignment, variable (which is a macro call in reality) and all constructs that
1881// do not handle in any context ('define directive and any unrecognized stuff).
Cole Faustdd569ae2022-01-31 15:48:29 -08001882func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNode {
1883 var result []starlarkNode
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001884 switch x := node.(type) {
1885 case *mkparser.Comment:
Cole Faustdd569ae2022-01-31 15:48:29 -08001886 if n, handled := ctx.maybeHandleAnnotation(x); handled && n != nil {
1887 result = []starlarkNode{n}
1888 } else if !handled {
1889 result = []starlarkNode{&commentNode{strings.TrimSpace("#" + x.Comment)}}
Cole Faust7940c6a2022-01-31 15:54:05 -08001890 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001891 case *mkparser.Assignment:
Cole Faustdd569ae2022-01-31 15:48:29 -08001892 result = ctx.handleAssignment(x)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001893 case *mkparser.Variable:
Cole Faustdd569ae2022-01-31 15:48:29 -08001894 result = ctx.handleVariable(x)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001895 case *mkparser.Directive:
1896 switch x.Name {
1897 case "define":
Cole Faustdd569ae2022-01-31 15:48:29 -08001898 if res := ctx.maybeHandleDefine(x); res != nil {
1899 result = []starlarkNode{res}
1900 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001901 case "include", "-include":
Cole Faust20052982022-04-22 14:43:55 -07001902 result = ctx.handleInclude(x)
Cole Faust591a1fe2021-11-08 15:37:57 -08001903 case "ifeq", "ifneq", "ifdef", "ifndef":
Cole Faustdd569ae2022-01-31 15:48:29 -08001904 result = []starlarkNode{ctx.handleIfBlock(x)}
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001905 default:
Cole Faustdd569ae2022-01-31 15:48:29 -08001906 result = []starlarkNode{ctx.newBadNode(x, "unexpected directive %s", x.Name)}
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001907 }
1908 default:
Cole Faustdd569ae2022-01-31 15:48:29 -08001909 result = []starlarkNode{ctx.newBadNode(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#"))}
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001910 }
Cole Faust6c934f62022-01-06 15:51:12 -08001911
1912 // Clear the includeTops after each non-comment statement
1913 // so that include annotations placed on certain statements don't apply
1914 // globally for the rest of the makefile was well.
Cole Faustf92c9f22022-03-14 14:35:50 -07001915 if _, wasComment := node.(*mkparser.Comment); !wasComment {
1916 ctx.atTopOfMakefile = false
Cole Faust6c934f62022-01-06 15:51:12 -08001917 ctx.includeTops = []string{}
1918 }
Cole Faustdd569ae2022-01-31 15:48:29 -08001919
1920 if result == nil {
1921 result = []starlarkNode{}
1922 }
Cole Faustf035d402022-03-28 14:02:50 -07001923
Cole Faustdd569ae2022-01-31 15:48:29 -08001924 return result
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001925}
1926
Cole Faustf92c9f22022-03-14 14:35:50 -07001927// The types allowed in a type_hint
1928var typeHintMap = map[string]starlarkType{
1929 "string": starlarkTypeString,
1930 "list": starlarkTypeList,
1931}
1932
Sasha Smundak6d852dd2021-09-27 20:34:39 -07001933// Processes annotation. An annotation is a comment that starts with #RBC# and provides
1934// a conversion hint -- say, where to look for the dynamically calculated inherit/include
Cole Faust7940c6a2022-01-31 15:54:05 -08001935// paths. Returns true if the comment was a successfully-handled annotation.
Cole Faustdd569ae2022-01-31 15:48:29 -08001936func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) (starlarkNode, bool) {
Sasha Smundak6d852dd2021-09-27 20:34:39 -07001937 maybeTrim := func(s, prefix string) (string, bool) {
1938 if strings.HasPrefix(s, prefix) {
1939 return strings.TrimSpace(strings.TrimPrefix(s, prefix)), true
1940 }
1941 return s, false
1942 }
1943 annotation, ok := maybeTrim(cnode.Comment, annotationCommentPrefix)
1944 if !ok {
Cole Faustdd569ae2022-01-31 15:48:29 -08001945 return nil, false
Sasha Smundak6d852dd2021-09-27 20:34:39 -07001946 }
1947 if p, ok := maybeTrim(annotation, "include_top"); ok {
Cole Faustf7ed5342021-12-21 14:15:12 -08001948 // Don't allow duplicate include tops, because then we will generate
1949 // invalid starlark code. (duplicate keys in the _entry dictionary)
1950 for _, top := range ctx.includeTops {
1951 if top == p {
Cole Faustdd569ae2022-01-31 15:48:29 -08001952 return nil, true
Cole Faustf7ed5342021-12-21 14:15:12 -08001953 }
1954 }
Sasha Smundak6d852dd2021-09-27 20:34:39 -07001955 ctx.includeTops = append(ctx.includeTops, p)
Cole Faustdd569ae2022-01-31 15:48:29 -08001956 return nil, true
Cole Faustf92c9f22022-03-14 14:35:50 -07001957 } else if p, ok := maybeTrim(annotation, "type_hint"); ok {
1958 // Type hints must come at the beginning the file, to avoid confusion
1959 // if a type hint was specified later and thus only takes effect for half
1960 // of the file.
1961 if !ctx.atTopOfMakefile {
1962 return ctx.newBadNode(cnode, "type_hint annotations must come before the first Makefile statement"), true
1963 }
1964
1965 parts := strings.Fields(p)
1966 if len(parts) <= 1 {
1967 return ctx.newBadNode(cnode, "Invalid type_hint annotation: %s. Must be a variable type followed by a list of variables of that type", p), true
1968 }
1969
1970 var varType starlarkType
1971 if varType, ok = typeHintMap[parts[0]]; !ok {
1972 varType = starlarkTypeUnknown
1973 }
1974 if varType == starlarkTypeUnknown {
1975 return ctx.newBadNode(cnode, "Invalid type_hint annotation. Only list/string types are accepted, found %s", parts[0]), true
1976 }
1977
1978 for _, name := range parts[1:] {
1979 // Don't allow duplicate type hints
1980 if _, ok := ctx.typeHints[name]; ok {
1981 return ctx.newBadNode(cnode, "Duplicate type hint for variable %s", name), true
1982 }
1983 ctx.typeHints[name] = varType
1984 }
1985 return nil, true
Sasha Smundak6d852dd2021-09-27 20:34:39 -07001986 }
Cole Faustdd569ae2022-01-31 15:48:29 -08001987 return ctx.newBadNode(cnode, "unsupported annotation %s", cnode.Comment), true
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001988}
1989
1990func (ctx *parseContext) loadedModulePath(path string) string {
1991 // During the transition to Roboleaf some of the product configuration files
1992 // will be converted and checked in while the others will be generated on the fly
1993 // and run. The runner (rbcrun application) accommodates this by allowing three
1994 // different ways to specify the loaded file location:
1995 // 1) load(":<file>",...) loads <file> from the same directory
1996 // 2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
1997 // 3) load("/absolute/path/to/<file> absolute path
1998 // If the file being generated and the file it wants to load are in the same directory,
1999 // generate option 1.
2000 // Otherwise, if output directory is not specified, generate 2)
2001 // Finally, if output directory has been specified and the file being generated and
2002 // the file it wants to load from are in the different directories, generate 2) or 3):
2003 // * if the file being loaded exists in the source tree, generate 2)
2004 // * otherwise, generate 3)
2005 // Finally, figure out the loaded module path and name and create a node for it
2006 loadedModuleDir := filepath.Dir(path)
2007 base := filepath.Base(path)
2008 loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
2009 if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
2010 return ":" + loadedModuleName
2011 }
2012 if ctx.outputDir == "" {
2013 return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
2014 }
2015 if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
2016 return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
2017 }
2018 return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
2019}
2020
Sasha Smundak3deb9682021-07-26 18:42:25 -07002021func (ctx *parseContext) addSoongNamespace(ns string) {
2022 if _, ok := ctx.soongNamespaces[ns]; ok {
2023 return
2024 }
2025 ctx.soongNamespaces[ns] = make(map[string]bool)
2026}
2027
2028func (ctx *parseContext) hasSoongNamespace(name string) bool {
2029 _, ok := ctx.soongNamespaces[name]
2030 return ok
2031}
2032
2033func (ctx *parseContext) updateSoongNamespace(replace bool, namespaceName string, varNames []string) {
2034 ctx.addSoongNamespace(namespaceName)
2035 vars := ctx.soongNamespaces[namespaceName]
2036 if replace {
2037 vars = make(map[string]bool)
2038 ctx.soongNamespaces[namespaceName] = vars
2039 }
2040 for _, v := range varNames {
2041 vars[v] = true
2042 }
2043}
2044
2045func (ctx *parseContext) hasNamespaceVar(namespaceName string, varName string) bool {
2046 vars, ok := ctx.soongNamespaces[namespaceName]
2047 if ok {
2048 _, ok = vars[varName]
2049 }
2050 return ok
2051}
2052
Sasha Smundak422b6142021-11-11 18:31:59 -08002053func (ctx *parseContext) errorLocation(node mkparser.Node) ErrorLocation {
2054 return ErrorLocation{ctx.script.mkFile, ctx.script.nodeLocator(node.Pos())}
2055}
2056
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002057func (ss *StarlarkScript) String() string {
2058 return NewGenerateContext(ss).emit()
2059}
2060
2061func (ss *StarlarkScript) SubConfigFiles() []string {
Sasha Smundak6609ba72021-07-22 18:32:56 -07002062
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002063 var subs []string
2064 for _, src := range ss.inherited {
2065 subs = append(subs, src.originalPath)
2066 }
2067 return subs
2068}
2069
2070func (ss *StarlarkScript) HasErrors() bool {
2071 return ss.hasErrors
2072}
2073
2074// Convert reads and parses a makefile. If successful, parsed tree
2075// is returned and then can be passed to String() to get the generated
2076// Starlark file.
2077func Convert(req Request) (*StarlarkScript, error) {
2078 reader := req.Reader
2079 if reader == nil {
2080 mkContents, err := ioutil.ReadFile(req.MkFile)
2081 if err != nil {
2082 return nil, err
2083 }
2084 reader = bytes.NewBuffer(mkContents)
2085 }
2086 parser := mkparser.NewParser(req.MkFile, reader)
2087 nodes, errs := parser.Parse()
2088 if len(errs) > 0 {
2089 for _, e := range errs {
2090 fmt.Fprintln(os.Stderr, "ERROR:", e)
2091 }
2092 return nil, fmt.Errorf("bad makefile %s", req.MkFile)
2093 }
2094 starScript := &StarlarkScript{
Sasha Smundak422b6142021-11-11 18:31:59 -08002095 moduleName: moduleNameForFile(req.MkFile),
2096 mkFile: req.MkFile,
Sasha Smundak422b6142021-11-11 18:31:59 -08002097 traceCalls: req.TraceCalls,
2098 sourceFS: req.SourceFS,
2099 makefileFinder: req.MakefileFinder,
2100 nodeLocator: func(pos mkparser.Pos) int { return parser.Unpack(pos).Line },
Cole Faustdd569ae2022-01-31 15:48:29 -08002101 nodes: make([]starlarkNode, 0),
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002102 }
2103 ctx := newParseContext(starScript, nodes)
2104 ctx.outputSuffix = req.OutputSuffix
2105 ctx.outputDir = req.OutputDir
2106 ctx.errorLogger = req.ErrorLogger
2107 if len(req.TracedVariables) > 0 {
2108 ctx.tracedVariables = make(map[string]bool)
2109 for _, v := range req.TracedVariables {
2110 ctx.tracedVariables[v] = true
2111 }
2112 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002113 for ctx.hasNodes() && ctx.fatalError == nil {
Cole Faustdd569ae2022-01-31 15:48:29 -08002114 starScript.nodes = append(starScript.nodes, ctx.handleSimpleStatement(ctx.getNode())...)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002115 }
2116 if ctx.fatalError != nil {
2117 return nil, ctx.fatalError
2118 }
2119 return starScript, nil
2120}
2121
Cole Faust864028a2021-12-01 13:43:17 -08002122func Launcher(mainModuleUri, inputVariablesUri, mainModuleName string) string {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002123 var buf bytes.Buffer
2124 fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
Cole Faust864028a2021-12-01 13:43:17 -08002125 fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri)
Sasha Smundakd7d07ad2021-09-10 15:42:34 -07002126 fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri)
Cole Faust864028a2021-12-01 13:43:17 -08002127 fmt.Fprintf(&buf, "%s(%s(%q, init, input_variables_init))\n", cfnPrintVars, cfnMain, mainModuleName)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002128 return buf.String()
2129}
2130
Cole Faust6ed7cb42021-10-07 17:08:46 -07002131func BoardLauncher(mainModuleUri string, inputVariablesUri string) string {
2132 var buf bytes.Buffer
2133 fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
2134 fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri)
2135 fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri)
Cole Fausta0604662022-02-28 11:53:58 -08002136 fmt.Fprintf(&buf, "%s(%s(init, input_variables_init))\n", cfnPrintVars, cfnBoardMain)
Cole Faust6ed7cb42021-10-07 17:08:46 -07002137 return buf.String()
2138}
2139
Sasha Smundakb051c4e2020-11-05 20:45:07 -08002140func MakePath2ModuleName(mkPath string) string {
2141 return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
2142}