blob: 1757965bac0d3414e0f300d39566c7bfcdebdf61 [file] [log] [blame]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Convert makefile containing device configuration to Starlark file
16// The conversion can handle the following constructs in a makefile:
17// * comments
18// * simple variable assignments
19// * $(call init-product,<file>)
20// * $(call inherit-product-if-exists
21// * if directives
22// All other constructs are carried over to the output starlark file as comments.
23//
24package mk2rbc
25
26import (
27 "bytes"
28 "fmt"
29 "io"
Sasha Smundak6609ba72021-07-22 18:32:56 -070030 "io/fs"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080031 "io/ioutil"
32 "os"
33 "path/filepath"
34 "regexp"
35 "strconv"
36 "strings"
37 "text/scanner"
38
39 mkparser "android/soong/androidmk/parser"
40)
41
42const (
43 baseUri = "//build/make/core:product_config.rbc"
44 // The name of the struct exported by the product_config.rbc
45 // that contains the functions and variables available to
46 // product configuration Starlark files.
47 baseName = "rblf"
48
49 // And here are the functions and variables:
50 cfnGetCfg = baseName + ".cfg"
51 cfnMain = baseName + ".product_configuration"
52 cfnPrintVars = baseName + ".printvars"
53 cfnWarning = baseName + ".warning"
54 cfnLocalAppend = baseName + ".local_append"
55 cfnLocalSetDefault = baseName + ".local_set_default"
56 cfnInherit = baseName + ".inherit"
57 cfnSetListDefault = baseName + ".setdefault"
58)
59
60const (
61 // Phony makefile functions, they are eventually rewritten
62 // according to knownFunctions map
63 fileExistsPhony = "$file_exists"
64 wildcardExistsPhony = "$wildcard_exists"
65)
66
67const (
68 callLoadAlways = "inherit-product"
69 callLoadIf = "inherit-product-if-exists"
70)
71
72var knownFunctions = map[string]struct {
73 // The name of the runtime function this function call in makefiles maps to.
74 // If it starts with !, then this makefile function call is rewritten to
75 // something else.
76 runtimeName string
77 returnType starlarkType
78}{
79 fileExistsPhony: {baseName + ".file_exists", starlarkTypeBool},
80 wildcardExistsPhony: {baseName + ".file_wildcard_exists", starlarkTypeBool},
81 "add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList},
82 "addprefix": {baseName + ".addprefix", starlarkTypeList},
83 "addsuffix": {baseName + ".addsuffix", starlarkTypeList},
84 "enforce-product-packages-exist": {baseName + ".enforce_product_packages_exist", starlarkTypeVoid},
85 "error": {baseName + ".mkerror", starlarkTypeVoid},
86 "findstring": {"!findstring", starlarkTypeInt},
87 "find-copy-subdir-files": {baseName + ".find_and_copy", starlarkTypeList},
Sasha Smundakf3e072a2021-07-14 12:50:28 -070088 "find-word-in-list": {"!find-word-in-list", starlarkTypeUnknown}, // internal macro
Sasha Smundakb051c4e2020-11-05 20:45:07 -080089 "filter": {baseName + ".filter", starlarkTypeList},
90 "filter-out": {baseName + ".filter_out", starlarkTypeList},
Sasha Smundakf3e072a2021-07-14 12:50:28 -070091 "get-vendor-board-platforms": {"!get-vendor-board-platforms", starlarkTypeList}, // internal macro, used by is-board-platform, etc.
Sasha Smundakb051c4e2020-11-05 20:45:07 -080092 "info": {baseName + ".mkinfo", starlarkTypeVoid},
Sasha Smundakf3e072a2021-07-14 12:50:28 -070093 "is-android-codename": {"!is-android-codename", starlarkTypeBool}, // unused by product config
94 "is-android-codename-in-list": {"!is-android-codename-in-list", starlarkTypeBool}, // unused by product config
Sasha Smundakb051c4e2020-11-05 20:45:07 -080095 "is-board-platform": {"!is-board-platform", starlarkTypeBool},
96 "is-board-platform-in-list": {"!is-board-platform-in-list", starlarkTypeBool},
Sasha Smundakf3e072a2021-07-14 12:50:28 -070097 "is-chipset-in-board-platform": {"!is-chipset-in-board-platform", starlarkTypeUnknown}, // unused by product config
98 "is-chipset-prefix-in-board-platform": {"!is-chipset-prefix-in-board-platform", starlarkTypeBool}, // unused by product config
99 "is-not-board-platform": {"!is-not-board-platform", starlarkTypeBool}, // defined but never used
100 "is-platform-sdk-version-at-least": {"!is-platform-sdk-version-at-least", starlarkTypeBool}, // unused by product config
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800101 "is-product-in-list": {"!is-product-in-list", starlarkTypeBool},
102 "is-vendor-board-platform": {"!is-vendor-board-platform", starlarkTypeBool},
103 callLoadAlways: {"!inherit-product", starlarkTypeVoid},
104 callLoadIf: {"!inherit-product-if-exists", starlarkTypeVoid},
Sasha Smundakf3e072a2021-07-14 12:50:28 -0700105 "match-prefix": {"!match-prefix", starlarkTypeUnknown}, // internal macro
106 "match-word": {"!match-word", starlarkTypeUnknown}, // internal macro
107 "match-word-in-list": {"!match-word-in-list", starlarkTypeUnknown}, // internal macro
Sasha Smundak6609ba72021-07-22 18:32:56 -0700108 "my-dir": {"!my-dir", starlarkTypeString},
Sasha Smundak94b41c72021-07-12 18:30:42 -0700109 "patsubst": {baseName + ".mkpatsubst", starlarkTypeString},
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800110 "produce_copy_files": {baseName + ".produce_copy_files", starlarkTypeList},
111 "require-artifacts-in-path": {baseName + ".require_artifacts_in_path", starlarkTypeVoid},
112 "require-artifacts-in-path-relaxed": {baseName + ".require_artifacts_in_path_relaxed", starlarkTypeVoid},
113 // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
Sasha Smundakf3e072a2021-07-14 12:50:28 -0700114 "shell": {baseName + ".shell", starlarkTypeString},
115 "strip": {baseName + ".mkstrip", starlarkTypeString},
116 "tb-modules": {"!tb-modules", starlarkTypeUnknown}, // defined in hardware/amlogic/tb_modules/tb_detect.mk, unused
117 "subst": {baseName + ".mksubst", starlarkTypeString},
118 "warning": {baseName + ".mkwarning", starlarkTypeVoid},
119 "word": {baseName + "!word", starlarkTypeString},
120 "wildcard": {baseName + ".expand_wildcard", starlarkTypeList},
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800121}
122
123var builtinFuncRex = regexp.MustCompile(
124 "^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" +
125 "|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" +
126 "|if|info|join|lastword|notdir|or|origin|patsubst|realpath" +
127 "|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" +
128 "|wildcard)")
129
130// Conversion request parameters
131type Request struct {
132 MkFile string // file to convert
133 Reader io.Reader // if set, read input from this stream instead
134 RootDir string // root directory path used to resolve included files
135 OutputSuffix string // generated Starlark files suffix
136 OutputDir string // if set, root of the output hierarchy
137 ErrorLogger ErrorMonitorCB
138 TracedVariables []string // trace assignment to these variables
139 TraceCalls bool
140 WarnPartialSuccess bool
Sasha Smundak6609ba72021-07-22 18:32:56 -0700141 SourceFS fs.FS
142 MakefileFinder MakefileFinder
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800143}
144
145// An error sink allowing to gather error statistics.
146// NewError is called on every error encountered during processing.
147type ErrorMonitorCB interface {
148 NewError(s string, node mkparser.Node, args ...interface{})
149}
150
151// Derives module name for a given file. It is base name
152// (file name without suffix), with some characters replaced to make it a Starlark identifier
153func moduleNameForFile(mkFile string) string {
154 base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
155 // TODO(asmundak): what else can be in the product file names?
Sasha Smundak6609ba72021-07-22 18:32:56 -0700156 return strings.NewReplacer("-", "_", ".", "_").Replace(base)
157
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800158}
159
160func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
161 r := &mkparser.MakeString{StringPos: mkString.StringPos}
162 r.Strings = append(r.Strings, mkString.Strings...)
163 r.Variables = append(r.Variables, mkString.Variables...)
164 return r
165}
166
167func isMakeControlFunc(s string) bool {
168 return s == "error" || s == "warning" || s == "info"
169}
170
171// Starlark output generation context
172type generationContext struct {
173 buf strings.Builder
174 starScript *StarlarkScript
175 indentLevel int
176 inAssignment bool
177 tracedCount int
178}
179
180func NewGenerateContext(ss *StarlarkScript) *generationContext {
181 return &generationContext{starScript: ss}
182}
183
184// emit returns generated script
185func (gctx *generationContext) emit() string {
186 ss := gctx.starScript
187
188 // The emitted code has the following layout:
189 // <initial comments>
190 // preamble, i.e.,
191 // load statement for the runtime support
192 // load statement for each unique submodule pulled in by this one
193 // def init(g, handle):
194 // cfg = rblf.cfg(handle)
195 // <statements>
196 // <warning if conversion was not clean>
197
198 iNode := len(ss.nodes)
199 for i, node := range ss.nodes {
200 if _, ok := node.(*commentNode); !ok {
201 iNode = i
202 break
203 }
204 node.emit(gctx)
205 }
206
207 gctx.emitPreamble()
208
209 gctx.newLine()
210 // The arguments passed to the init function are the global dictionary
211 // ('g') and the product configuration dictionary ('cfg')
212 gctx.write("def init(g, handle):")
213 gctx.indentLevel++
214 if gctx.starScript.traceCalls {
215 gctx.newLine()
216 gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
217 }
218 gctx.newLine()
219 gctx.writef("cfg = %s(handle)", cfnGetCfg)
220 for _, node := range ss.nodes[iNode:] {
221 node.emit(gctx)
222 }
223
224 if ss.hasErrors && ss.warnPartialSuccess {
225 gctx.newLine()
226 gctx.writef("%s(%q, %q)", cfnWarning, filepath.Base(ss.mkFile), "partially successful conversion")
227 }
228 if gctx.starScript.traceCalls {
229 gctx.newLine()
230 gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
231 }
232 gctx.indentLevel--
233 gctx.write("\n")
234 return gctx.buf.String()
235}
236
237func (gctx *generationContext) emitPreamble() {
238 gctx.newLine()
239 gctx.writef("load(%q, %q)", baseUri, baseName)
240 // Emit exactly one load statement for each URI.
241 loadedSubConfigs := make(map[string]string)
242 for _, sc := range gctx.starScript.inherited {
243 uri := sc.path
244 if m, ok := loadedSubConfigs[uri]; ok {
245 // No need to emit load statement, but fix module name.
246 sc.moduleLocalName = m
247 continue
248 }
Sasha Smundak6609ba72021-07-22 18:32:56 -0700249 if sc.optional {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800250 uri += "|init"
251 }
252 gctx.newLine()
253 gctx.writef("load(%q, %s = \"init\")", uri, sc.entryName())
254 loadedSubConfigs[uri] = sc.moduleLocalName
255 }
256 gctx.write("\n")
257}
258
259func (gctx *generationContext) emitPass() {
260 gctx.newLine()
261 gctx.write("pass")
262}
263
264func (gctx *generationContext) write(ss ...string) {
265 for _, s := range ss {
266 gctx.buf.WriteString(s)
267 }
268}
269
270func (gctx *generationContext) writef(format string, args ...interface{}) {
271 gctx.write(fmt.Sprintf(format, args...))
272}
273
274func (gctx *generationContext) newLine() {
275 if gctx.buf.Len() == 0 {
276 return
277 }
278 gctx.write("\n")
279 gctx.writef("%*s", 2*gctx.indentLevel, "")
280}
281
282type knownVariable struct {
283 name string
284 class varClass
285 valueType starlarkType
286}
287
288type knownVariables map[string]knownVariable
289
290func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
291 v, exists := pcv[name]
292 if !exists {
293 pcv[name] = knownVariable{name, varClass, valueType}
294 return
295 }
296 // Conflict resolution:
297 // * config class trumps everything
298 // * any type trumps unknown type
299 match := varClass == v.class
300 if !match {
301 if varClass == VarClassConfig {
302 v.class = VarClassConfig
303 match = true
304 } else if v.class == VarClassConfig {
305 match = true
306 }
307 }
308 if valueType != v.valueType {
309 if valueType != starlarkTypeUnknown {
310 if v.valueType == starlarkTypeUnknown {
311 v.valueType = valueType
312 } else {
313 match = false
314 }
315 }
316 }
317 if !match {
318 fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
319 name, varClass, valueType, v.class, v.valueType)
320 }
321}
322
323// All known product variables.
324var KnownVariables = make(knownVariables)
325
326func init() {
327 for _, kv := range []string{
328 // Kernel-related variables that we know are lists.
329 "BOARD_VENDOR_KERNEL_MODULES",
330 "BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
331 "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
332 "BOARD_RECOVERY_KERNEL_MODULES",
333 // Other variables we knwo are lists
334 "ART_APEX_JARS",
335 } {
336 KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
337 }
338}
339
340type nodeReceiver interface {
341 newNode(node starlarkNode)
342}
343
344// Information about the generated Starlark script.
345type StarlarkScript struct {
346 mkFile string
347 moduleName string
348 mkPos scanner.Position
349 nodes []starlarkNode
Sasha Smundak6609ba72021-07-22 18:32:56 -0700350 inherited []*moduleInfo
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800351 hasErrors bool
352 topDir string
353 traceCalls bool // print enter/exit each init function
354 warnPartialSuccess bool
Sasha Smundak6609ba72021-07-22 18:32:56 -0700355 sourceFS fs.FS
356 makefileFinder MakefileFinder
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800357}
358
359func (ss *StarlarkScript) newNode(node starlarkNode) {
360 ss.nodes = append(ss.nodes, node)
361}
362
363// varAssignmentScope points to the last assignment for each variable
364// in the current block. It is used during the parsing to chain
365// the assignments to a variable together.
366type varAssignmentScope struct {
367 outer *varAssignmentScope
368 vars map[string]*assignmentNode
369}
370
371// parseContext holds the script we are generating and all the ephemeral data
372// needed during the parsing.
373type parseContext struct {
374 script *StarlarkScript
375 nodes []mkparser.Node // Makefile as parsed by mkparser
376 currentNodeIndex int // Node in it we are processing
377 ifNestLevel int
378 moduleNameCount map[string]int // count of imported modules with given basename
379 fatalError error
380 builtinMakeVars map[string]starlarkExpr
381 outputSuffix string
382 errorLogger ErrorMonitorCB
383 tracedVariables map[string]bool // variables to be traced in the generated script
384 variables map[string]variable
385 varAssignments *varAssignmentScope
386 receiver nodeReceiver // receptacle for the generated starlarkNode's
387 receiverStack []nodeReceiver
388 outputDir string
Sasha Smundak6609ba72021-07-22 18:32:56 -0700389 dependentModules map[string]*moduleInfo
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800390}
391
392func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
Sasha Smundak6609ba72021-07-22 18:32:56 -0700393 topdir, _ := filepath.Split(filepath.Join(ss.topDir, "foo"))
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800394 predefined := []struct{ name, value string }{
395 {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
396 {"LOCAL_PATH", filepath.Dir(ss.mkFile)},
Sasha Smundak6609ba72021-07-22 18:32:56 -0700397 {"TOPDIR", topdir},
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800398 // TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
399 {"TARGET_COPY_OUT_SYSTEM", "system"},
400 {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
401 {"TARGET_COPY_OUT_DATA", "data"},
402 {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
403 {"TARGET_COPY_OUT_OEM", "oem"},
404 {"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
405 {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
406 {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
407 {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
408 {"TARGET_COPY_OUT_ROOT", "root"},
409 {"TARGET_COPY_OUT_RECOVERY", "recovery"},
410 {"TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"},
411 {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
412 {"TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"},
413 {"TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"},
414 {"TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"},
415 {"TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"},
416 {"TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"},
417 {"TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"},
418 // TODO(asmundak): to process internal config files, we need the following variables:
419 // BOARD_CONFIG_VENDOR_PATH
420 // TARGET_VENDOR
421 // target_base_product
422 //
423
424 // the following utility variables are set in build/make/common/core.mk:
425 {"empty", ""},
426 {"space", " "},
427 {"comma", ","},
428 {"newline", "\n"},
429 {"pound", "#"},
430 {"backslash", "\\"},
431 }
432 ctx := &parseContext{
433 script: ss,
434 nodes: nodes,
435 currentNodeIndex: 0,
436 ifNestLevel: 0,
437 moduleNameCount: make(map[string]int),
438 builtinMakeVars: map[string]starlarkExpr{},
439 variables: make(map[string]variable),
Sasha Smundak6609ba72021-07-22 18:32:56 -0700440 dependentModules: make(map[string]*moduleInfo),
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800441 }
442 ctx.pushVarAssignments()
443 for _, item := range predefined {
444 ctx.variables[item.name] = &predefinedVariable{
445 baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
446 value: &stringLiteralExpr{item.value},
447 }
448 }
449
450 return ctx
451}
452
453func (ctx *parseContext) lastAssignment(name string) *assignmentNode {
454 for va := ctx.varAssignments; va != nil; va = va.outer {
455 if v, ok := va.vars[name]; ok {
456 return v
457 }
458 }
459 return nil
460}
461
462func (ctx *parseContext) setLastAssignment(name string, asgn *assignmentNode) {
463 ctx.varAssignments.vars[name] = asgn
464}
465
466func (ctx *parseContext) pushVarAssignments() {
467 va := &varAssignmentScope{
468 outer: ctx.varAssignments,
469 vars: make(map[string]*assignmentNode),
470 }
471 ctx.varAssignments = va
472}
473
474func (ctx *parseContext) popVarAssignments() {
475 ctx.varAssignments = ctx.varAssignments.outer
476}
477
478func (ctx *parseContext) pushReceiver(rcv nodeReceiver) {
479 ctx.receiverStack = append(ctx.receiverStack, ctx.receiver)
480 ctx.receiver = rcv
481}
482
483func (ctx *parseContext) popReceiver() {
484 last := len(ctx.receiverStack) - 1
485 if last < 0 {
486 panic(fmt.Errorf("popReceiver: receiver stack empty"))
487 }
488 ctx.receiver = ctx.receiverStack[last]
489 ctx.receiverStack = ctx.receiverStack[0:last]
490}
491
492func (ctx *parseContext) hasNodes() bool {
493 return ctx.currentNodeIndex < len(ctx.nodes)
494}
495
496func (ctx *parseContext) getNode() mkparser.Node {
497 if !ctx.hasNodes() {
498 return nil
499 }
500 node := ctx.nodes[ctx.currentNodeIndex]
501 ctx.currentNodeIndex++
502 return node
503}
504
505func (ctx *parseContext) backNode() {
506 if ctx.currentNodeIndex <= 0 {
507 panic("Cannot back off")
508 }
509 ctx.currentNodeIndex--
510}
511
512func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) {
513 // Handle only simple variables
514 if !a.Name.Const() {
515 ctx.errorf(a, "Only simple variables are handled")
516 return
517 }
518 name := a.Name.Strings[0]
519 lhs := ctx.addVariable(name)
520 if lhs == nil {
521 ctx.errorf(a, "unknown variable %s", name)
522 return
523 }
524 _, isTraced := ctx.tracedVariables[name]
525 asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced}
526 if lhs.valueType() == starlarkTypeUnknown {
527 // Try to divine variable type from the RHS
528 asgn.value = ctx.parseMakeString(a, a.Value)
529 if xBad, ok := asgn.value.(*badExpr); ok {
530 ctx.wrapBadExpr(xBad)
531 return
532 }
533 inferred_type := asgn.value.typ()
534 if inferred_type != starlarkTypeUnknown {
Sasha Smundak9d011ab2021-07-09 16:00:57 -0700535 lhs.setValueType(inferred_type)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800536 }
537 }
538 if lhs.valueType() == starlarkTypeList {
539 xConcat := ctx.buildConcatExpr(a)
540 if xConcat == nil {
541 return
542 }
543 switch len(xConcat.items) {
544 case 0:
545 asgn.value = &listExpr{}
546 case 1:
547 asgn.value = xConcat.items[0]
548 default:
549 asgn.value = xConcat
550 }
551 } else {
552 asgn.value = ctx.parseMakeString(a, a.Value)
553 if xBad, ok := asgn.value.(*badExpr); ok {
554 ctx.wrapBadExpr(xBad)
555 return
556 }
557 }
558
559 // TODO(asmundak): move evaluation to a separate pass
560 asgn.value, _ = asgn.value.eval(ctx.builtinMakeVars)
561
562 asgn.previous = ctx.lastAssignment(name)
563 ctx.setLastAssignment(name, asgn)
564 switch a.Type {
565 case "=", ":=":
566 asgn.flavor = asgnSet
567 case "+=":
568 if asgn.previous == nil && !asgn.lhs.isPreset() {
569 asgn.flavor = asgnMaybeAppend
570 } else {
571 asgn.flavor = asgnAppend
572 }
573 case "?=":
574 asgn.flavor = asgnMaybeSet
575 default:
576 panic(fmt.Errorf("unexpected assignment type %s", a.Type))
577 }
578
579 ctx.receiver.newNode(asgn)
580}
581
582func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr {
583 xConcat := &concatExpr{}
584 var xItemList *listExpr
585 addToItemList := func(x ...starlarkExpr) {
586 if xItemList == nil {
587 xItemList = &listExpr{[]starlarkExpr{}}
588 }
589 xItemList.items = append(xItemList.items, x...)
590 }
591 finishItemList := func() {
592 if xItemList != nil {
593 xConcat.items = append(xConcat.items, xItemList)
594 xItemList = nil
595 }
596 }
597
598 items := a.Value.Words()
599 for _, item := range items {
600 // A function call in RHS is supposed to return a list, all other item
601 // expressions return individual elements.
602 switch x := ctx.parseMakeString(a, item).(type) {
603 case *badExpr:
604 ctx.wrapBadExpr(x)
605 return nil
606 case *stringLiteralExpr:
607 addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
608 default:
609 switch x.typ() {
610 case starlarkTypeList:
611 finishItemList()
612 xConcat.items = append(xConcat.items, x)
613 case starlarkTypeString:
614 finishItemList()
615 xConcat.items = append(xConcat.items, &callExpr{
616 object: x,
617 name: "split",
618 args: nil,
619 returnType: starlarkTypeList,
620 })
621 default:
622 addToItemList(x)
623 }
624 }
625 }
626 if xItemList != nil {
627 xConcat.items = append(xConcat.items, xItemList)
628 }
629 return xConcat
630}
631
Sasha Smundak6609ba72021-07-22 18:32:56 -0700632func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
633 modulePath := ctx.loadedModulePath(path)
634 if mi, ok := ctx.dependentModules[modulePath]; ok {
635 mi.optional = mi.optional || optional
636 return mi
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800637 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800638 moduleName := moduleNameForFile(path)
639 moduleLocalName := "_" + moduleName
640 n, found := ctx.moduleNameCount[moduleName]
641 if found {
642 moduleLocalName += fmt.Sprintf("%d", n)
643 }
644 ctx.moduleNameCount[moduleName] = n + 1
Sasha Smundak6609ba72021-07-22 18:32:56 -0700645 mi := &moduleInfo{
646 path: modulePath,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800647 originalPath: path,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800648 moduleLocalName: moduleLocalName,
Sasha Smundak6609ba72021-07-22 18:32:56 -0700649 optional: optional,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800650 }
Sasha Smundak6609ba72021-07-22 18:32:56 -0700651 ctx.dependentModules[modulePath] = mi
652 ctx.script.inherited = append(ctx.script.inherited, mi)
653 return mi
654}
655
656func (ctx *parseContext) handleSubConfig(
657 v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule)) {
658 pathExpr, _ = pathExpr.eval(ctx.builtinMakeVars)
659
660 // In a simple case, the name of a module to inherit/include is known statically.
661 if path, ok := maybeString(pathExpr); ok {
662 if strings.Contains(path, "*") {
663 if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
664 for _, p := range paths {
665 processModule(inheritedStaticModule{ctx.newDependentModule(p, !loadAlways), loadAlways})
666 }
667 } else {
668 ctx.errorf(v, "cannot glob wildcard argument")
669 }
670 } else {
671 processModule(inheritedStaticModule{ctx.newDependentModule(path, !loadAlways), loadAlways})
672 }
673 return
674 }
675
676 // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
677 // source tree that may be a match and the corresponding variable values. For instance, if the source tree
678 // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
679 // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
680 // We then emit the code that loads all of them, e.g.:
681 // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
682 // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
683 // And then inherit it as follows:
684 // _e = {
685 // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
686 // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
687 // if _e:
688 // rblf.inherit(handle, _e[0], _e[1])
689 //
690 var matchingPaths []string
691 varPath, ok := pathExpr.(*interpolateExpr)
692 if !ok {
693 ctx.errorf(v, "inherit-product/include argument is too complex")
694 return
695 }
696
697 pathPattern := []string{varPath.chunks[0]}
698 for _, chunk := range varPath.chunks[1:] {
699 if chunk != "" {
700 pathPattern = append(pathPattern, chunk)
701 }
702 }
703 if pathPattern[0] != "" {
704 matchingPaths = ctx.findMatchingPaths(pathPattern)
705 } else {
706 // Heuristics -- if pattern starts from top, restrict it to the directories where
707 // we know inherit-product uses dynamically calculated path.
708 for _, t := range []string{"vendor/qcom", "vendor/google_devices"} {
709 pathPattern[0] = t
710 matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
711 }
712 }
713 // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
714 const maxMatchingFiles = 100
715 if len(matchingPaths) > maxMatchingFiles {
716 ctx.errorf(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)
717 return
718 }
719 res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways}
720 for _, p := range matchingPaths {
721 // A product configuration files discovered dynamically may attempt to inherit
722 // from another one which does not exist in this source tree. Prevent load errors
723 // by always loading the dynamic files as optional.
724 res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
725 }
726 processModule(res)
727}
728
729func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
730 files := ctx.script.makefileFinder.Find(ctx.script.topDir)
731 if len(pattern) == 0 {
732 return files
733 }
734
735 // Create regular expression from the pattern
736 s_regexp := "^" + regexp.QuoteMeta(pattern[0])
737 for _, s := range pattern[1:] {
738 s_regexp += ".*" + regexp.QuoteMeta(s)
739 }
740 s_regexp += "$"
741 rex := regexp.MustCompile(s_regexp)
742
743 // Now match
744 var res []string
745 for _, p := range files {
746 if rex.MatchString(p) {
747 res = append(res, p)
748 }
749 }
750 return res
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800751}
752
753func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
Sasha Smundak6609ba72021-07-22 18:32:56 -0700754 ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800755 ctx.receiver.newNode(&inheritNode{im})
Sasha Smundak6609ba72021-07-22 18:32:56 -0700756 })
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800757}
758
759func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
Sasha Smundak6609ba72021-07-22 18:32:56 -0700760 ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
761 ctx.receiver.newNode(&includeNode{im})
762 })
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800763}
764
765func (ctx *parseContext) handleVariable(v *mkparser.Variable) {
766 // Handle:
767 // $(call inherit-product,...)
768 // $(call inherit-product-if-exists,...)
769 // $(info xxx)
770 // $(warning xxx)
771 // $(error xxx)
772 expr := ctx.parseReference(v, v.Name)
773 switch x := expr.(type) {
774 case *callExpr:
775 if x.name == callLoadAlways || x.name == callLoadIf {
776 ctx.handleInheritModule(v, x.args[0], x.name == callLoadAlways)
777 } else if isMakeControlFunc(x.name) {
778 // File name is the first argument
779 args := []starlarkExpr{
780 &stringLiteralExpr{ctx.script.mkFile},
781 x.args[0],
782 }
783 ctx.receiver.newNode(&exprNode{
784 &callExpr{name: x.name, args: args, returnType: starlarkTypeUnknown},
785 })
786 } else {
787 ctx.receiver.newNode(&exprNode{expr})
788 }
789 case *badExpr:
790 ctx.wrapBadExpr(x)
791 return
792 default:
793 ctx.errorf(v, "cannot handle %s", v.Dump())
794 return
795 }
796}
797
798func (ctx *parseContext) handleDefine(directive *mkparser.Directive) {
Sasha Smundakf3e072a2021-07-14 12:50:28 -0700799 macro_name := strings.Fields(directive.Args.Strings[0])[0]
800 // Ignore the macros that we handle
801 if _, ok := knownFunctions[macro_name]; !ok {
802 ctx.errorf(directive, "define is not supported: %s", macro_name)
803 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800804}
805
806func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) {
807 ssSwitch := &switchNode{}
808 ctx.pushReceiver(ssSwitch)
809 for ctx.processBranch(ifDirective); ctx.hasNodes() && ctx.fatalError == nil; {
810 node := ctx.getNode()
811 switch x := node.(type) {
812 case *mkparser.Directive:
813 switch x.Name {
814 case "else", "elifdef", "elifndef", "elifeq", "elifneq":
815 ctx.processBranch(x)
816 case "endif":
817 ctx.popReceiver()
818 ctx.receiver.newNode(ssSwitch)
819 return
820 default:
821 ctx.errorf(node, "unexpected directive %s", x.Name)
822 }
823 default:
824 ctx.errorf(ifDirective, "unexpected statement")
825 }
826 }
827 if ctx.fatalError == nil {
828 ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
829 }
830 ctx.popReceiver()
831}
832
833// processBranch processes a single branch (if/elseif/else) until the next directive
834// on the same level.
835func (ctx *parseContext) processBranch(check *mkparser.Directive) {
836 block := switchCase{gate: ctx.parseCondition(check)}
837 defer func() {
838 ctx.popVarAssignments()
839 ctx.ifNestLevel--
840
841 }()
842 ctx.pushVarAssignments()
843 ctx.ifNestLevel++
844
845 ctx.pushReceiver(&block)
846 for ctx.hasNodes() {
847 node := ctx.getNode()
848 if ctx.handleSimpleStatement(node) {
849 continue
850 }
851 switch d := node.(type) {
852 case *mkparser.Directive:
853 switch d.Name {
854 case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
855 ctx.popReceiver()
856 ctx.receiver.newNode(&block)
857 ctx.backNode()
858 return
859 case "ifdef", "ifndef", "ifeq", "ifneq":
860 ctx.handleIfBlock(d)
861 default:
862 ctx.errorf(d, "unexpected directive %s", d.Name)
863 }
864 default:
865 ctx.errorf(node, "unexpected statement")
866 }
867 }
868 ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
869 ctx.popReceiver()
870}
871
872func (ctx *parseContext) newIfDefinedNode(check *mkparser.Directive) (starlarkExpr, bool) {
873 if !check.Args.Const() {
874 return ctx.newBadExpr(check, "ifdef variable ref too complex: %s", check.Args.Dump()), false
875 }
876 v := ctx.addVariable(check.Args.Strings[0])
877 return &variableDefinedExpr{v}, true
878}
879
880func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
881 switch check.Name {
882 case "ifdef", "ifndef", "elifdef", "elifndef":
883 v, ok := ctx.newIfDefinedNode(check)
884 if ok && strings.HasSuffix(check.Name, "ndef") {
885 v = &notExpr{v}
886 }
887 return &ifNode{
888 isElif: strings.HasPrefix(check.Name, "elif"),
889 expr: v,
890 }
891 case "ifeq", "ifneq", "elifeq", "elifneq":
892 return &ifNode{
893 isElif: strings.HasPrefix(check.Name, "elif"),
894 expr: ctx.parseCompare(check),
895 }
896 case "else":
897 return &elseNode{}
898 default:
899 panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
900 }
901}
902
903func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
904 message := fmt.Sprintf(text, args...)
905 if ctx.errorLogger != nil {
906 ctx.errorLogger.NewError(text, node, args)
907 }
908 ctx.script.hasErrors = true
909 return &badExpr{node, message}
910}
911
912func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
913 // Strip outer parentheses
914 mkArg := cloneMakeString(cond.Args)
915 mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
916 n := len(mkArg.Strings)
917 mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
918 args := mkArg.Split(",")
919 // TODO(asmundak): handle the case where the arguments are in quotes and space-separated
920 if len(args) != 2 {
921 return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
922 }
923 args[0].TrimRightSpaces()
924 args[1].TrimLeftSpaces()
925
926 isEq := !strings.HasSuffix(cond.Name, "neq")
927 switch xLeft := ctx.parseMakeString(cond, args[0]).(type) {
928 case *stringLiteralExpr, *variableRefExpr:
929 switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
930 case *stringLiteralExpr, *variableRefExpr:
931 return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
932 case *badExpr:
933 return xRight
934 default:
935 expr, ok := ctx.parseCheckFunctionCallResult(cond, xLeft, args[1])
936 if ok {
937 return expr
938 }
939 return ctx.newBadExpr(cond, "right operand is too complex: %s", args[1].Dump())
940 }
941 case *badExpr:
942 return xLeft
943 default:
944 switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
945 case *stringLiteralExpr, *variableRefExpr:
946 expr, ok := ctx.parseCheckFunctionCallResult(cond, xRight, args[0])
947 if ok {
948 return expr
949 }
950 return ctx.newBadExpr(cond, "left operand is too complex: %s", args[0].Dump())
951 case *badExpr:
952 return xRight
953 default:
954 return ctx.newBadExpr(cond, "operands are too complex: (%s,%s)", args[0].Dump(), args[1].Dump())
955 }
956 }
957}
958
959func (ctx *parseContext) parseCheckFunctionCallResult(directive *mkparser.Directive, xValue starlarkExpr,
960 varArg *mkparser.MakeString) (starlarkExpr, bool) {
961 mkSingleVar, ok := varArg.SingleVariable()
962 if !ok {
963 return nil, false
964 }
965 expr := ctx.parseReference(directive, mkSingleVar)
966 negate := strings.HasSuffix(directive.Name, "neq")
967 checkIsSomethingFunction := func(xCall *callExpr) starlarkExpr {
968 s, ok := maybeString(xValue)
969 if !ok || s != "true" {
970 return ctx.newBadExpr(directive,
971 fmt.Sprintf("the result of %s can be compared only to 'true'", xCall.name))
972 }
973 if len(xCall.args) < 1 {
974 return ctx.newBadExpr(directive, "%s requires an argument", xCall.name)
975 }
976 return nil
977 }
978 switch x := expr.(type) {
979 case *callExpr:
980 switch x.name {
981 case "filter":
982 return ctx.parseCompareFilterFuncResult(directive, x, xValue, !negate), true
983 case "filter-out":
984 return ctx.parseCompareFilterFuncResult(directive, x, xValue, negate), true
985 case "wildcard":
986 return ctx.parseCompareWildcardFuncResult(directive, x, xValue, negate), true
987 case "findstring":
988 return ctx.parseCheckFindstringFuncResult(directive, x, xValue, negate), true
989 case "strip":
990 return ctx.parseCompareStripFuncResult(directive, x, xValue, negate), true
991 case "is-board-platform":
992 if xBad := checkIsSomethingFunction(x); xBad != nil {
993 return xBad, true
994 }
995 return &eqExpr{
996 left: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
997 right: x.args[0],
998 isEq: !negate,
999 }, true
1000 case "is-board-platform-in-list":
1001 if xBad := checkIsSomethingFunction(x); xBad != nil {
1002 return xBad, true
1003 }
1004 return &inExpr{
1005 expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
1006 list: maybeConvertToStringList(x.args[0]),
1007 isNot: negate,
1008 }, true
1009 case "is-product-in-list":
1010 if xBad := checkIsSomethingFunction(x); xBad != nil {
1011 return xBad, true
1012 }
1013 return &inExpr{
1014 expr: &variableRefExpr{ctx.addVariable("TARGET_PRODUCT"), true},
1015 list: maybeConvertToStringList(x.args[0]),
1016 isNot: negate,
1017 }, true
1018 case "is-vendor-board-platform":
1019 if xBad := checkIsSomethingFunction(x); xBad != nil {
1020 return xBad, true
1021 }
1022 s, ok := maybeString(x.args[0])
1023 if !ok {
1024 return ctx.newBadExpr(directive, "cannot handle non-constant argument to is-vendor-board-platform"), true
1025 }
1026 return &inExpr{
1027 expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
1028 list: &variableRefExpr{ctx.addVariable(s + "_BOARD_PLATFORMS"), true},
1029 isNot: negate,
1030 }, true
1031 default:
1032 return ctx.newBadExpr(directive, "Unknown function in ifeq: %s", x.name), true
1033 }
1034 case *badExpr:
1035 return x, true
1036 default:
1037 return nil, false
1038 }
1039}
1040
1041func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
1042 filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
1043 // We handle:
Sasha Smundak0554d762021-07-08 18:26:12 -07001044 // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...]
1045 // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001046 // * ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"]
1047 // TODO(Asmundak): check the last case works for filter-out, too.
1048 xPattern := filterFuncCall.args[0]
1049 xText := filterFuncCall.args[1]
1050 var xInList *stringLiteralExpr
Sasha Smundak0554d762021-07-08 18:26:12 -07001051 var expr starlarkExpr
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001052 var ok bool
1053 switch x := xValue.(type) {
1054 case *stringLiteralExpr:
1055 if x.literal != "" {
1056 return ctx.newBadExpr(cond, "filter comparison to non-empty value: %s", xValue)
1057 }
1058 // Either pattern or text should be const, and the
1059 // non-const one should be varRefExpr
1060 if xInList, ok = xPattern.(*stringLiteralExpr); ok {
Sasha Smundak0554d762021-07-08 18:26:12 -07001061 expr = xText
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001062 } else if xInList, ok = xText.(*stringLiteralExpr); ok {
Sasha Smundak0554d762021-07-08 18:26:12 -07001063 expr = xPattern
1064 } else {
1065 return &callExpr{
1066 object: nil,
1067 name: filterFuncCall.name,
1068 args: filterFuncCall.args,
1069 returnType: starlarkTypeBool,
1070 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001071 }
1072 case *variableRefExpr:
1073 if v, ok := xPattern.(*variableRefExpr); ok {
1074 if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() {
1075 // ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate,
1076 // it's the opposite to what is done when comparing to empty.
Sasha Smundak0554d762021-07-08 18:26:12 -07001077 expr = xPattern
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001078 negate = !negate
1079 }
1080 }
1081 }
Sasha Smundak0554d762021-07-08 18:26:12 -07001082 if expr != nil && xInList != nil {
1083 slExpr := newStringListExpr(strings.Fields(xInList.literal))
1084 // Generate simpler code for the common cases:
1085 if expr.typ() == starlarkTypeList {
1086 if len(slExpr.items) == 1 {
1087 // Checking that a string belongs to list
1088 return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}
1089 } else {
1090 // TODO(asmundak):
1091 panic("TBD")
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001092 }
Sasha Smundak0554d762021-07-08 18:26:12 -07001093 } else if len(slExpr.items) == 1 {
1094 return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}
1095 } else {
1096 return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001097 }
1098 }
1099 return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump())
1100}
1101
1102func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
1103 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
Sasha Smundak0554d762021-07-08 18:26:12 -07001104 if !isEmptyString(xValue) {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001105 return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
1106 }
1107 callFunc := wildcardExistsPhony
1108 if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") {
1109 callFunc = fileExistsPhony
1110 }
1111 var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool}
1112 if !negate {
1113 cc = &notExpr{cc}
1114 }
1115 return cc
1116}
1117
1118func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
1119 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
Sasha Smundak0554d762021-07-08 18:26:12 -07001120 if isEmptyString(xValue) {
1121 return &eqExpr{
1122 left: &callExpr{
1123 object: xCall.args[1],
1124 name: "find",
1125 args: []starlarkExpr{xCall.args[0]},
1126 returnType: starlarkTypeInt,
1127 },
1128 right: &intLiteralExpr{-1},
1129 isEq: !negate,
1130 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001131 }
Sasha Smundak0554d762021-07-08 18:26:12 -07001132 return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001133}
1134
1135func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
1136 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
1137 if _, ok := xValue.(*stringLiteralExpr); !ok {
1138 return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
1139 }
1140 return &eqExpr{
1141 left: &callExpr{
1142 name: "strip",
1143 args: xCall.args,
1144 returnType: starlarkTypeString,
1145 },
1146 right: xValue, isEq: !negate}
1147}
1148
1149// parses $(...), returning an expression
1150func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
1151 ref.TrimLeftSpaces()
1152 ref.TrimRightSpaces()
1153 refDump := ref.Dump()
1154
1155 // Handle only the case where the first (or only) word is constant
1156 words := ref.SplitN(" ", 2)
1157 if !words[0].Const() {
1158 return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
1159 }
1160
1161 // If it is a single word, it can be a simple variable
1162 // reference or a function call
1163 if len(words) == 1 {
1164 if isMakeControlFunc(refDump) || refDump == "shell" {
1165 return &callExpr{
1166 name: refDump,
1167 args: []starlarkExpr{&stringLiteralExpr{""}},
1168 returnType: starlarkTypeUnknown,
1169 }
1170 }
1171 if v := ctx.addVariable(refDump); v != nil {
1172 return &variableRefExpr{v, ctx.lastAssignment(v.name()) != nil}
1173 }
1174 return ctx.newBadExpr(node, "unknown variable %s", refDump)
1175 }
1176
1177 expr := &callExpr{name: words[0].Dump(), returnType: starlarkTypeUnknown}
1178 args := words[1]
1179 args.TrimLeftSpaces()
1180 // Make control functions and shell need special treatment as everything
1181 // after the name is a single text argument
1182 if isMakeControlFunc(expr.name) || expr.name == "shell" {
1183 x := ctx.parseMakeString(node, args)
1184 if xBad, ok := x.(*badExpr); ok {
1185 return xBad
1186 }
1187 expr.args = []starlarkExpr{x}
1188 return expr
1189 }
1190 if expr.name == "call" {
1191 words = args.SplitN(",", 2)
1192 if words[0].Empty() || !words[0].Const() {
Sasha Smundakf2c9f8b2021-07-27 10:44:48 -07001193 return ctx.newBadExpr(node, "cannot handle %s", refDump)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001194 }
1195 expr.name = words[0].Dump()
1196 if len(words) < 2 {
Sasha Smundak6609ba72021-07-22 18:32:56 -07001197 args = &mkparser.MakeString{}
1198 } else {
1199 args = words[1]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001200 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001201 }
1202 if kf, found := knownFunctions[expr.name]; found {
1203 expr.returnType = kf.returnType
1204 } else {
1205 return ctx.newBadExpr(node, "cannot handle invoking %s", expr.name)
1206 }
1207 switch expr.name {
1208 case "word":
1209 return ctx.parseWordFunc(node, args)
Sasha Smundak6609ba72021-07-22 18:32:56 -07001210 case "my-dir":
1211 return &variableRefExpr{ctx.addVariable("LOCAL_PATH"), true}
Sasha Smundak94b41c72021-07-12 18:30:42 -07001212 case "subst", "patsubst":
1213 return ctx.parseSubstFunc(node, expr.name, args)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001214 default:
1215 for _, arg := range args.Split(",") {
1216 arg.TrimLeftSpaces()
1217 arg.TrimRightSpaces()
1218 x := ctx.parseMakeString(node, arg)
1219 if xBad, ok := x.(*badExpr); ok {
1220 return xBad
1221 }
1222 expr.args = append(expr.args, x)
1223 }
1224 }
1225 return expr
1226}
1227
Sasha Smundak94b41c72021-07-12 18:30:42 -07001228func (ctx *parseContext) parseSubstFunc(node mkparser.Node, fname string, args *mkparser.MakeString) starlarkExpr {
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001229 words := args.Split(",")
1230 if len(words) != 3 {
Sasha Smundak94b41c72021-07-12 18:30:42 -07001231 return ctx.newBadExpr(node, "%s function should have 3 arguments", fname)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001232 }
1233 if !words[0].Const() || !words[1].Const() {
Sasha Smundak94b41c72021-07-12 18:30:42 -07001234 return ctx.newBadExpr(node, "%s function's from and to arguments should be constant", fname)
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001235 }
1236 from := words[0].Strings[0]
1237 to := words[1].Strings[0]
1238 words[2].TrimLeftSpaces()
1239 words[2].TrimRightSpaces()
1240 obj := ctx.parseMakeString(node, words[2])
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001241 typ := obj.typ()
Sasha Smundak94b41c72021-07-12 18:30:42 -07001242 if typ == starlarkTypeString && fname == "subst" {
1243 // Optimization: if it's $(subst from, to, string), emit string.replace(from, to)
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001244 return &callExpr{
1245 object: obj,
1246 name: "replace",
1247 args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}},
1248 returnType: typ,
1249 }
1250 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001251 return &callExpr{
Sasha Smundak94b41c72021-07-12 18:30:42 -07001252 name: fname,
Sasha Smundak9d011ab2021-07-09 16:00:57 -07001253 args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}, obj},
1254 returnType: obj.typ(),
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001255 }
1256}
1257
1258func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
1259 words := args.Split(",")
1260 if len(words) != 2 {
1261 return ctx.newBadExpr(node, "word function should have 2 arguments")
1262 }
1263 var index uint64 = 0
1264 if words[0].Const() {
1265 index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64)
1266 }
1267 if index < 1 {
1268 return ctx.newBadExpr(node, "word index should be constant positive integer")
1269 }
1270 words[1].TrimLeftSpaces()
1271 words[1].TrimRightSpaces()
1272 array := ctx.parseMakeString(node, words[1])
1273 if xBad, ok := array.(*badExpr); ok {
1274 return xBad
1275 }
1276 if array.typ() != starlarkTypeList {
1277 array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
1278 }
1279 return indexExpr{array, &intLiteralExpr{int(index - 1)}}
1280}
1281
1282func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
1283 if mk.Const() {
1284 return &stringLiteralExpr{mk.Dump()}
1285 }
1286 if mkRef, ok := mk.SingleVariable(); ok {
1287 return ctx.parseReference(node, mkRef)
1288 }
1289 // If we reached here, it's neither string literal nor a simple variable,
1290 // we need a full-blown interpolation node that will generate
1291 // "a%b%c" % (X, Y) for a$(X)b$(Y)c
1292 xInterp := &interpolateExpr{args: make([]starlarkExpr, len(mk.Variables))}
1293 for i, ref := range mk.Variables {
1294 arg := ctx.parseReference(node, ref.Name)
1295 if x, ok := arg.(*badExpr); ok {
1296 return x
1297 }
1298 xInterp.args[i] = arg
1299 }
1300 xInterp.chunks = append(xInterp.chunks, mk.Strings...)
1301 return xInterp
1302}
1303
1304// Handles the statements whose treatment is the same in all contexts: comment,
1305// assignment, variable (which is a macro call in reality) and all constructs that
1306// do not handle in any context ('define directive and any unrecognized stuff).
1307// Return true if we handled it.
1308func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool {
1309 handled := true
1310 switch x := node.(type) {
1311 case *mkparser.Comment:
1312 ctx.insertComment("#" + x.Comment)
1313 case *mkparser.Assignment:
1314 ctx.handleAssignment(x)
1315 case *mkparser.Variable:
1316 ctx.handleVariable(x)
1317 case *mkparser.Directive:
1318 switch x.Name {
1319 case "define":
1320 ctx.handleDefine(x)
1321 case "include", "-include":
1322 ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-')
1323 default:
1324 handled = false
1325 }
1326 default:
1327 ctx.errorf(x, "unsupported line %s", x.Dump())
1328 }
1329 return handled
1330}
1331
1332func (ctx *parseContext) insertComment(s string) {
1333 ctx.receiver.newNode(&commentNode{strings.TrimSpace(s)})
1334}
1335
1336func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) {
1337 for _, line := range strings.Split(failedNode.Dump(), "\n") {
1338 ctx.insertComment("# " + line)
1339 }
1340}
1341
1342// records that the given node failed to be converted and includes an explanatory message
1343func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) {
1344 if ctx.errorLogger != nil {
1345 ctx.errorLogger.NewError(message, failedNode, args...)
1346 }
1347 message = fmt.Sprintf(message, args...)
1348 ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", message))
1349 ctx.carryAsComment(failedNode)
1350 ctx.script.hasErrors = true
1351}
1352
1353func (ctx *parseContext) wrapBadExpr(xBad *badExpr) {
1354 ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", xBad.message))
1355 ctx.carryAsComment(xBad.node)
1356}
1357
1358func (ctx *parseContext) loadedModulePath(path string) string {
1359 // During the transition to Roboleaf some of the product configuration files
1360 // will be converted and checked in while the others will be generated on the fly
1361 // and run. The runner (rbcrun application) accommodates this by allowing three
1362 // different ways to specify the loaded file location:
1363 // 1) load(":<file>",...) loads <file> from the same directory
1364 // 2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
1365 // 3) load("/absolute/path/to/<file> absolute path
1366 // If the file being generated and the file it wants to load are in the same directory,
1367 // generate option 1.
1368 // Otherwise, if output directory is not specified, generate 2)
1369 // Finally, if output directory has been specified and the file being generated and
1370 // the file it wants to load from are in the different directories, generate 2) or 3):
1371 // * if the file being loaded exists in the source tree, generate 2)
1372 // * otherwise, generate 3)
1373 // Finally, figure out the loaded module path and name and create a node for it
1374 loadedModuleDir := filepath.Dir(path)
1375 base := filepath.Base(path)
1376 loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
1377 if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
1378 return ":" + loadedModuleName
1379 }
1380 if ctx.outputDir == "" {
1381 return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
1382 }
1383 if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
1384 return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
1385 }
1386 return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
1387}
1388
1389func (ss *StarlarkScript) String() string {
1390 return NewGenerateContext(ss).emit()
1391}
1392
1393func (ss *StarlarkScript) SubConfigFiles() []string {
Sasha Smundak6609ba72021-07-22 18:32:56 -07001394
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001395 var subs []string
1396 for _, src := range ss.inherited {
1397 subs = append(subs, src.originalPath)
1398 }
1399 return subs
1400}
1401
1402func (ss *StarlarkScript) HasErrors() bool {
1403 return ss.hasErrors
1404}
1405
1406// Convert reads and parses a makefile. If successful, parsed tree
1407// is returned and then can be passed to String() to get the generated
1408// Starlark file.
1409func Convert(req Request) (*StarlarkScript, error) {
1410 reader := req.Reader
1411 if reader == nil {
1412 mkContents, err := ioutil.ReadFile(req.MkFile)
1413 if err != nil {
1414 return nil, err
1415 }
1416 reader = bytes.NewBuffer(mkContents)
1417 }
1418 parser := mkparser.NewParser(req.MkFile, reader)
1419 nodes, errs := parser.Parse()
1420 if len(errs) > 0 {
1421 for _, e := range errs {
1422 fmt.Fprintln(os.Stderr, "ERROR:", e)
1423 }
1424 return nil, fmt.Errorf("bad makefile %s", req.MkFile)
1425 }
1426 starScript := &StarlarkScript{
1427 moduleName: moduleNameForFile(req.MkFile),
1428 mkFile: req.MkFile,
1429 topDir: req.RootDir,
1430 traceCalls: req.TraceCalls,
1431 warnPartialSuccess: req.WarnPartialSuccess,
Sasha Smundak6609ba72021-07-22 18:32:56 -07001432 sourceFS: req.SourceFS,
1433 makefileFinder: req.MakefileFinder,
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001434 }
1435 ctx := newParseContext(starScript, nodes)
1436 ctx.outputSuffix = req.OutputSuffix
1437 ctx.outputDir = req.OutputDir
1438 ctx.errorLogger = req.ErrorLogger
1439 if len(req.TracedVariables) > 0 {
1440 ctx.tracedVariables = make(map[string]bool)
1441 for _, v := range req.TracedVariables {
1442 ctx.tracedVariables[v] = true
1443 }
1444 }
1445 ctx.pushReceiver(starScript)
1446 for ctx.hasNodes() && ctx.fatalError == nil {
1447 node := ctx.getNode()
1448 if ctx.handleSimpleStatement(node) {
1449 continue
1450 }
1451 switch x := node.(type) {
1452 case *mkparser.Directive:
1453 switch x.Name {
1454 case "ifeq", "ifneq", "ifdef", "ifndef":
1455 ctx.handleIfBlock(x)
1456 default:
1457 ctx.errorf(x, "unexpected directive %s", x.Name)
1458 }
1459 default:
1460 ctx.errorf(x, "unsupported line")
1461 }
1462 }
1463 if ctx.fatalError != nil {
1464 return nil, ctx.fatalError
1465 }
1466 return starScript, nil
1467}
1468
1469func Launcher(path, name string) string {
1470 var buf bytes.Buffer
1471 fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
1472 fmt.Fprintf(&buf, "load(%q, \"init\")\n", path)
1473 fmt.Fprintf(&buf, "g, config = %s(%q, init)\n", cfnMain, name)
1474 fmt.Fprintf(&buf, "%s(g, config)\n", cfnPrintVars)
1475 return buf.String()
1476}
1477
1478func MakePath2ModuleName(mkPath string) string {
1479 return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
1480}