mk2rbc/mk2rbc.go

// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Convert makefile containing device configuration to Starlark file
// The conversion can handle the following constructs in a makefile:
//   * comments
//   * simple variable assignments
//   * $(call init-product,<file>)
//   * $(call inherit-product-if-exists
//   * if directives
// All other constructs are carried over to the output starlark file as comments.
//
package mk2rbc

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/scanner"

	mkparser "android/soong/androidmk/parser"
)

const (
	baseUri = "//build/make/core:product_config.rbc"
	// The name of the struct exported by the product_config.rbc
	// that contains the functions and variables available to
	// product configuration Starlark files.
	baseName = "rblf"

	// And here are the functions and variables:
	cfnGetCfg          = baseName + ".cfg"
	cfnMain            = baseName + ".product_configuration"
	cfnPrintVars       = baseName + ".printvars"
	cfnWarning         = baseName + ".warning"
	cfnLocalAppend     = baseName + ".local_append"
	cfnLocalSetDefault = baseName + ".local_set_default"
	cfnInherit         = baseName + ".inherit"
	cfnSetListDefault  = baseName + ".setdefault"
)

const (
	// Phony makefile functions, they are eventually rewritten
	// according to knownFunctions map
	fileExistsPhony     = "$file_exists"
	wildcardExistsPhony = "$wildcard_exists"
)

const (
	callLoadAlways = "inherit-product"
	callLoadIf     = "inherit-product-if-exists"
)

var knownFunctions = map[string]struct {
	// The name of the runtime function this function call in makefiles maps to.
	// If it starts with !, then this makefile function call is rewritten to
	// something else.
	runtimeName string
	returnType  starlarkType
}{
	fileExistsPhony:                       {baseName + ".file_exists", starlarkTypeBool},
	wildcardExistsPhony:                   {baseName + ".file_wildcard_exists", starlarkTypeBool},
	"add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList},
	"addprefix":                           {baseName + ".addprefix", starlarkTypeList},
	"addsuffix":                           {baseName + ".addsuffix", starlarkTypeList},
	"enforce-product-packages-exist":      {baseName + ".enforce_product_packages_exist", starlarkTypeVoid},
	"error":                               {baseName + ".mkerror", starlarkTypeVoid},
	"findstring":                          {"!findstring", starlarkTypeInt},
	"find-copy-subdir-files":              {baseName + ".find_and_copy", starlarkTypeList},
	"filter":                              {baseName + ".filter", starlarkTypeList},
	"filter-out":                          {baseName + ".filter_out", starlarkTypeList},
	"info":                                {baseName + ".mkinfo", starlarkTypeVoid},
	"is-board-platform":                   {"!is-board-platform", starlarkTypeBool},
	"is-board-platform-in-list":           {"!is-board-platform-in-list", starlarkTypeBool},
	"is-product-in-list":                  {"!is-product-in-list", starlarkTypeBool},
	"is-vendor-board-platform":            {"!is-vendor-board-platform", starlarkTypeBool},
	callLoadAlways:                        {"!inherit-product", starlarkTypeVoid},
	callLoadIf:                            {"!inherit-product-if-exists", starlarkTypeVoid},
	"patsubst":                            {baseName + ".mkpatsubst", starlarkTypeString},
	"produce_copy_files":                  {baseName + ".produce_copy_files", starlarkTypeList},
	"require-artifacts-in-path":           {baseName + ".require_artifacts_in_path", starlarkTypeVoid},
	"require-artifacts-in-path-relaxed":   {baseName + ".require_artifacts_in_path_relaxed", starlarkTypeVoid},
	// TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
	"shell":    {baseName + ".shell", starlarkTypeString},
	"strip":    {baseName + ".mkstrip", starlarkTypeString},
	"subst":    {baseName + ".mksubst", starlarkTypeString},
	"warning":  {baseName + ".mkwarning", starlarkTypeVoid},
	"word":     {baseName + "!word", starlarkTypeString},
	"wildcard": {baseName + ".expand_wildcard", starlarkTypeList},
}

var builtinFuncRex = regexp.MustCompile(
	"^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" +
		"|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" +
		"|if|info|join|lastword|notdir|or|origin|patsubst|realpath" +
		"|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" +
		"|wildcard)")

// Conversion request parameters
type Request struct {
	MkFile             string    // file to convert
	Reader             io.Reader // if set, read input from this stream instead
	RootDir            string    // root directory path used to resolve included files
	OutputSuffix       string    // generated Starlark files suffix
	OutputDir          string    // if set, root of the output hierarchy
	ErrorLogger        ErrorMonitorCB
	TracedVariables    []string // trace assignment to these variables
	TraceCalls         bool
	WarnPartialSuccess bool
}

// An error sink allowing to gather error statistics.
// NewError is called on every error encountered during processing.
type ErrorMonitorCB interface {
	NewError(s string, node mkparser.Node, args ...interface{})
}

// Derives module name for a given file. It is base name
// (file name without suffix), with some characters replaced to make it a Starlark identifier
func moduleNameForFile(mkFile string) string {
	base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
	// TODO(asmundak): what else can be in the product file names?
	return strings.ReplaceAll(base, "-", "_")
}

func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
	r := &mkparser.MakeString{StringPos: mkString.StringPos}
	r.Strings = append(r.Strings, mkString.Strings...)
	r.Variables = append(r.Variables, mkString.Variables...)
	return r
}

func isMakeControlFunc(s string) bool {
	return s == "error" || s == "warning" || s == "info"
}

// Starlark output generation context
type generationContext struct {
	buf          strings.Builder
	starScript   *StarlarkScript
	indentLevel  int
	inAssignment bool
	tracedCount  int
}

func NewGenerateContext(ss *StarlarkScript) *generationContext {
	return &generationContext{starScript: ss}
}

// emit returns generated script
func (gctx *generationContext) emit() string {
	ss := gctx.starScript

	// The emitted code has the following layout:
	//    <initial comments>
	//    preamble, i.e.,
	//      load statement for the runtime support
	//      load statement for each unique submodule pulled in by this one
	//    def init(g, handle):
	//      cfg = rblf.cfg(handle)
	//      <statements>
	//      <warning if conversion was not clean>

	iNode := len(ss.nodes)
	for i, node := range ss.nodes {
		if _, ok := node.(*commentNode); !ok {
			iNode = i
			break
		}
		node.emit(gctx)
	}

	gctx.emitPreamble()

	gctx.newLine()
	// The arguments passed to the init function are the global dictionary
	// ('g') and the product configuration dictionary ('cfg')
	gctx.write("def init(g, handle):")
	gctx.indentLevel++
	if gctx.starScript.traceCalls {
		gctx.newLine()
		gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
	}
	gctx.newLine()
	gctx.writef("cfg = %s(handle)", cfnGetCfg)
	for _, node := range ss.nodes[iNode:] {
		node.emit(gctx)
	}

	if ss.hasErrors && ss.warnPartialSuccess {
		gctx.newLine()
		gctx.writef("%s(%q, %q)", cfnWarning, filepath.Base(ss.mkFile), "partially successful conversion")
	}
	if gctx.starScript.traceCalls {
		gctx.newLine()
		gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
	}
	gctx.indentLevel--
	gctx.write("\n")
	return gctx.buf.String()
}

func (gctx *generationContext) emitPreamble() {
	gctx.newLine()
	gctx.writef("load(%q, %q)", baseUri, baseName)
	// Emit exactly one load statement for each URI.
	loadedSubConfigs := make(map[string]string)
	for _, sc := range gctx.starScript.inherited {
		uri := sc.path
		if m, ok := loadedSubConfigs[uri]; ok {
			// No need to emit load statement, but fix module name.
			sc.moduleLocalName = m
			continue
		}
		if !sc.loadAlways {
			uri += "|init"
		}
		gctx.newLine()
		gctx.writef("load(%q, %s = \"init\")", uri, sc.entryName())
		loadedSubConfigs[uri] = sc.moduleLocalName
	}
	gctx.write("\n")
}

func (gctx *generationContext) emitPass() {
	gctx.newLine()
	gctx.write("pass")
}

func (gctx *generationContext) write(ss ...string) {
	for _, s := range ss {
		gctx.buf.WriteString(s)
	}
}

func (gctx *generationContext) writef(format string, args ...interface{}) {
	gctx.write(fmt.Sprintf(format, args...))
}

func (gctx *generationContext) newLine() {
	if gctx.buf.Len() == 0 {
		return
	}
	gctx.write("\n")
	gctx.writef("%*s", 2*gctx.indentLevel, "")
}

type knownVariable struct {
	name      string
	class     varClass
	valueType starlarkType
}

type knownVariables map[string]knownVariable

func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
	v, exists := pcv[name]
	if !exists {
		pcv[name] = knownVariable{name, varClass, valueType}
		return
	}
	// Conflict resolution:
	//    * config class trumps everything
	//    * any type trumps unknown type
	match := varClass == v.class
	if !match {
		if varClass == VarClassConfig {
			v.class = VarClassConfig
			match = true
		} else if v.class == VarClassConfig {
			match = true
		}
	}
	if valueType != v.valueType {
		if valueType != starlarkTypeUnknown {
			if v.valueType == starlarkTypeUnknown {
				v.valueType = valueType
			} else {
				match = false
			}
		}
	}
	if !match {
		fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
			name, varClass, valueType, v.class, v.valueType)
	}
}

// All known product variables.
var KnownVariables = make(knownVariables)

func init() {
	for _, kv := range []string{
		// Kernel-related variables that we know are lists.
		"BOARD_VENDOR_KERNEL_MODULES",
		"BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
		"BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
		"BOARD_RECOVERY_KERNEL_MODULES",
		// Other variables we knwo are lists
		"ART_APEX_JARS",
	} {
		KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
	}
}

type nodeReceiver interface {
	newNode(node starlarkNode)
}

// Information about the generated Starlark script.
type StarlarkScript struct {
	mkFile             string
	moduleName         string
	mkPos              scanner.Position
	nodes              []starlarkNode
	inherited          []*inheritedModule
	hasErrors          bool
	topDir             string
	traceCalls         bool // print enter/exit each init function
	warnPartialSuccess bool
}

func (ss *StarlarkScript) newNode(node starlarkNode) {
	ss.nodes = append(ss.nodes, node)
}

// varAssignmentScope points to the last assignment for each variable
// in the current block. It is used during the parsing to chain
// the assignments to a variable together.
type varAssignmentScope struct {
	outer *varAssignmentScope
	vars  map[string]*assignmentNode
}

// parseContext holds the script we are generating and all the ephemeral data
// needed during the parsing.
type parseContext struct {
	script           *StarlarkScript
	nodes            []mkparser.Node // Makefile as parsed by mkparser
	currentNodeIndex int             // Node in it we are processing
	ifNestLevel      int
	moduleNameCount  map[string]int // count of imported modules with given basename
	fatalError       error
	builtinMakeVars  map[string]starlarkExpr
	outputSuffix     string
	errorLogger      ErrorMonitorCB
	tracedVariables  map[string]bool // variables to be traced in the generated script
	variables        map[string]variable
	varAssignments   *varAssignmentScope
	receiver         nodeReceiver // receptacle for the generated starlarkNode's
	receiverStack    []nodeReceiver
	outputDir        string
}

func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
	predefined := []struct{ name, value string }{
		{"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
		{"LOCAL_PATH", filepath.Dir(ss.mkFile)},
		{"TOPDIR", ss.topDir},
		// TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
		{"TARGET_COPY_OUT_SYSTEM", "system"},
		{"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
		{"TARGET_COPY_OUT_DATA", "data"},
		{"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
		{"TARGET_COPY_OUT_OEM", "oem"},
		{"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
		{"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
		{"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
		{"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
		{"TARGET_COPY_OUT_ROOT", "root"},
		{"TARGET_COPY_OUT_RECOVERY", "recovery"},
		{"TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"},
		{"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
		{"TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"},
		{"TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"},
		{"TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"},
		{"TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"},
		{"TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"},
		{"TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"},
		// TODO(asmundak): to process internal config files, we need the following variables:
		//    BOARD_CONFIG_VENDOR_PATH
		//    TARGET_VENDOR
		//    target_base_product
		//

		// the following utility variables are set in build/make/common/core.mk:
		{"empty", ""},
		{"space", " "},
		{"comma", ","},
		{"newline", "\n"},
		{"pound", "#"},
		{"backslash", "\\"},
	}
	ctx := &parseContext{
		script:           ss,
		nodes:            nodes,
		currentNodeIndex: 0,
		ifNestLevel:      0,
		moduleNameCount:  make(map[string]int),
		builtinMakeVars:  map[string]starlarkExpr{},
		variables:        make(map[string]variable),
	}
	ctx.pushVarAssignments()
	for _, item := range predefined {
		ctx.variables[item.name] = &predefinedVariable{
			baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
			value:        &stringLiteralExpr{item.value},
		}
	}

	return ctx
}

func (ctx *parseContext) lastAssignment(name string) *assignmentNode {
	for va := ctx.varAssignments; va != nil; va = va.outer {
		if v, ok := va.vars[name]; ok {
			return v
		}
	}
	return nil
}

func (ctx *parseContext) setLastAssignment(name string, asgn *assignmentNode) {
	ctx.varAssignments.vars[name] = asgn
}

func (ctx *parseContext) pushVarAssignments() {
	va := &varAssignmentScope{
		outer: ctx.varAssignments,
		vars:  make(map[string]*assignmentNode),
	}
	ctx.varAssignments = va
}

func (ctx *parseContext) popVarAssignments() {
	ctx.varAssignments = ctx.varAssignments.outer
}

func (ctx *parseContext) pushReceiver(rcv nodeReceiver) {
	ctx.receiverStack = append(ctx.receiverStack, ctx.receiver)
	ctx.receiver = rcv
}

func (ctx *parseContext) popReceiver() {
	last := len(ctx.receiverStack) - 1
	if last < 0 {
		panic(fmt.Errorf("popReceiver: receiver stack empty"))
	}
	ctx.receiver = ctx.receiverStack[last]
	ctx.receiverStack = ctx.receiverStack[0:last]
}

func (ctx *parseContext) hasNodes() bool {
	return ctx.currentNodeIndex < len(ctx.nodes)
}

func (ctx *parseContext) getNode() mkparser.Node {
	if !ctx.hasNodes() {
		return nil
	}
	node := ctx.nodes[ctx.currentNodeIndex]
	ctx.currentNodeIndex++
	return node
}

func (ctx *parseContext) backNode() {
	if ctx.currentNodeIndex <= 0 {
		panic("Cannot back off")
	}
	ctx.currentNodeIndex--
}

func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) {
	// Handle only simple variables
	if !a.Name.Const() {
		ctx.errorf(a, "Only simple variables are handled")
		return
	}
	name := a.Name.Strings[0]
	lhs := ctx.addVariable(name)
	if lhs == nil {
		ctx.errorf(a, "unknown variable %s", name)
		return
	}
	_, isTraced := ctx.tracedVariables[name]
	asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced}
	if lhs.valueType() == starlarkTypeUnknown {
		// Try to divine variable type from the RHS
		asgn.value = ctx.parseMakeString(a, a.Value)
		if xBad, ok := asgn.value.(*badExpr); ok {
			ctx.wrapBadExpr(xBad)
			return
		}
		inferred_type := asgn.value.typ()
		if inferred_type != starlarkTypeUnknown {
			lhs.setValueType(inferred_type)
		}
	}
	if lhs.valueType() == starlarkTypeList {
		xConcat := ctx.buildConcatExpr(a)
		if xConcat == nil {
			return
		}
		switch len(xConcat.items) {
		case 0:
			asgn.value = &listExpr{}
		case 1:
			asgn.value = xConcat.items[0]
		default:
			asgn.value = xConcat
		}
	} else {
		asgn.value = ctx.parseMakeString(a, a.Value)
		if xBad, ok := asgn.value.(*badExpr); ok {
			ctx.wrapBadExpr(xBad)
			return
		}
	}

	// TODO(asmundak): move evaluation to a separate pass
	asgn.value, _ = asgn.value.eval(ctx.builtinMakeVars)

	asgn.previous = ctx.lastAssignment(name)
	ctx.setLastAssignment(name, asgn)
	switch a.Type {
	case "=", ":=":
		asgn.flavor = asgnSet
	case "+=":
		if asgn.previous == nil && !asgn.lhs.isPreset() {
			asgn.flavor = asgnMaybeAppend
		} else {
			asgn.flavor = asgnAppend
		}
	case "?=":
		asgn.flavor = asgnMaybeSet
	default:
		panic(fmt.Errorf("unexpected assignment type %s", a.Type))
	}

	ctx.receiver.newNode(asgn)
}

func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr {
	xConcat := &concatExpr{}
	var xItemList *listExpr
	addToItemList := func(x ...starlarkExpr) {
		if xItemList == nil {
			xItemList = &listExpr{[]starlarkExpr{}}
		}
		xItemList.items = append(xItemList.items, x...)
	}
	finishItemList := func() {
		if xItemList != nil {
			xConcat.items = append(xConcat.items, xItemList)
			xItemList = nil
		}
	}

	items := a.Value.Words()
	for _, item := range items {
		// A function call in RHS is supposed to return a list, all other item
		// expressions return individual elements.
		switch x := ctx.parseMakeString(a, item).(type) {
		case *badExpr:
			ctx.wrapBadExpr(x)
			return nil
		case *stringLiteralExpr:
			addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
		default:
			switch x.typ() {
			case starlarkTypeList:
				finishItemList()
				xConcat.items = append(xConcat.items, x)
			case starlarkTypeString:
				finishItemList()
				xConcat.items = append(xConcat.items, &callExpr{
					object:     x,
					name:       "split",
					args:       nil,
					returnType: starlarkTypeList,
				})
			default:
				addToItemList(x)
			}
		}
	}
	if xItemList != nil {
		xConcat.items = append(xConcat.items, xItemList)
	}
	return xConcat
}

func (ctx *parseContext) newInheritedModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) *inheritedModule {
	var path string
	x, _ := pathExpr.eval(ctx.builtinMakeVars)
	s, ok := x.(*stringLiteralExpr)
	if !ok {
		ctx.errorf(v, "inherit-product/include argument is too complex")
		return nil
	}

	path = s.literal
	moduleName := moduleNameForFile(path)
	moduleLocalName := "_" + moduleName
	n, found := ctx.moduleNameCount[moduleName]
	if found {
		moduleLocalName += fmt.Sprintf("%d", n)
	}
	ctx.moduleNameCount[moduleName] = n + 1
	ln := &inheritedModule{
		path:            ctx.loadedModulePath(path),
		originalPath:    path,
		moduleName:      moduleName,
		moduleLocalName: moduleLocalName,
		loadAlways:      loadAlways,
	}
	ctx.script.inherited = append(ctx.script.inherited, ln)
	return ln
}

func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
	if im := ctx.newInheritedModule(v, pathExpr, loadAlways); im != nil {
		ctx.receiver.newNode(&inheritNode{im})
	}
}

func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
	if ln := ctx.newInheritedModule(v, pathExpr, loadAlways); ln != nil {
		ctx.receiver.newNode(&includeNode{ln})
	}
}

func (ctx *parseContext) handleVariable(v *mkparser.Variable) {
	// Handle:
	//   $(call inherit-product,...)
	//   $(call inherit-product-if-exists,...)
	//   $(info xxx)
	//   $(warning xxx)
	//   $(error xxx)
	expr := ctx.parseReference(v, v.Name)
	switch x := expr.(type) {
	case *callExpr:
		if x.name == callLoadAlways || x.name == callLoadIf {
			ctx.handleInheritModule(v, x.args[0], x.name == callLoadAlways)
		} else if isMakeControlFunc(x.name) {
			// File name is the first argument
			args := []starlarkExpr{
				&stringLiteralExpr{ctx.script.mkFile},
				x.args[0],
			}
			ctx.receiver.newNode(&exprNode{
				&callExpr{name: x.name, args: args, returnType: starlarkTypeUnknown},
			})
		} else {
			ctx.receiver.newNode(&exprNode{expr})
		}
	case *badExpr:
		ctx.wrapBadExpr(x)
		return
	default:
		ctx.errorf(v, "cannot handle %s", v.Dump())
		return
	}
}

func (ctx *parseContext) handleDefine(directive *mkparser.Directive) {
	tokens := strings.Fields(directive.Args.Strings[0])
	ctx.errorf(directive, "define is not supported: %s", tokens[0])
}

func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) {
	ssSwitch := &switchNode{}
	ctx.pushReceiver(ssSwitch)
	for ctx.processBranch(ifDirective); ctx.hasNodes() && ctx.fatalError == nil; {
		node := ctx.getNode()
		switch x := node.(type) {
		case *mkparser.Directive:
			switch x.Name {
			case "else", "elifdef", "elifndef", "elifeq", "elifneq":
				ctx.processBranch(x)
			case "endif":
				ctx.popReceiver()
				ctx.receiver.newNode(ssSwitch)
				return
			default:
				ctx.errorf(node, "unexpected directive %s", x.Name)
			}
		default:
			ctx.errorf(ifDirective, "unexpected statement")
		}
	}
	if ctx.fatalError == nil {
		ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
	}
	ctx.popReceiver()
}

// processBranch processes a single branch (if/elseif/else) until the next directive
// on the same level.
func (ctx *parseContext) processBranch(check *mkparser.Directive) {
	block := switchCase{gate: ctx.parseCondition(check)}
	defer func() {
		ctx.popVarAssignments()
		ctx.ifNestLevel--

	}()
	ctx.pushVarAssignments()
	ctx.ifNestLevel++

	ctx.pushReceiver(&block)
	for ctx.hasNodes() {
		node := ctx.getNode()
		if ctx.handleSimpleStatement(node) {
			continue
		}
		switch d := node.(type) {
		case *mkparser.Directive:
			switch d.Name {
			case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
				ctx.popReceiver()
				ctx.receiver.newNode(&block)
				ctx.backNode()
				return
			case "ifdef", "ifndef", "ifeq", "ifneq":
				ctx.handleIfBlock(d)
			default:
				ctx.errorf(d, "unexpected directive %s", d.Name)
			}
		default:
			ctx.errorf(node, "unexpected statement")
		}
	}
	ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
	ctx.popReceiver()
}

func (ctx *parseContext) newIfDefinedNode(check *mkparser.Directive) (starlarkExpr, bool) {
	if !check.Args.Const() {
		return ctx.newBadExpr(check, "ifdef variable ref too complex: %s", check.Args.Dump()), false
	}
	v := ctx.addVariable(check.Args.Strings[0])
	return &variableDefinedExpr{v}, true
}

func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
	switch check.Name {
	case "ifdef", "ifndef", "elifdef", "elifndef":
		v, ok := ctx.newIfDefinedNode(check)
		if ok && strings.HasSuffix(check.Name, "ndef") {
			v = &notExpr{v}
		}
		return &ifNode{
			isElif: strings.HasPrefix(check.Name, "elif"),
			expr:   v,
		}
	case "ifeq", "ifneq", "elifeq", "elifneq":
		return &ifNode{
			isElif: strings.HasPrefix(check.Name, "elif"),
			expr:   ctx.parseCompare(check),
		}
	case "else":
		return &elseNode{}
	default:
		panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
	}
}

func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
	message := fmt.Sprintf(text, args...)
	if ctx.errorLogger != nil {
		ctx.errorLogger.NewError(text, node, args)
	}
	ctx.script.hasErrors = true
	return &badExpr{node, message}
}

func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
	// Strip outer parentheses
	mkArg := cloneMakeString(cond.Args)
	mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
	n := len(mkArg.Strings)
	mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
	args := mkArg.Split(",")
	// TODO(asmundak): handle the case where the arguments are in quotes and space-separated
	if len(args) != 2 {
		return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
	}
	args[0].TrimRightSpaces()
	args[1].TrimLeftSpaces()

	isEq := !strings.HasSuffix(cond.Name, "neq")
	switch xLeft := ctx.parseMakeString(cond, args[0]).(type) {
	case *stringLiteralExpr, *variableRefExpr:
		switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
		case *stringLiteralExpr, *variableRefExpr:
			return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
		case *badExpr:
			return xRight
		default:
			expr, ok := ctx.parseCheckFunctionCallResult(cond, xLeft, args[1])
			if ok {
				return expr
			}
			return ctx.newBadExpr(cond, "right operand is too complex: %s", args[1].Dump())
		}
	case *badExpr:
		return xLeft
	default:
		switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
		case *stringLiteralExpr, *variableRefExpr:
			expr, ok := ctx.parseCheckFunctionCallResult(cond, xRight, args[0])
			if ok {
				return expr
			}
			return ctx.newBadExpr(cond, "left operand is too complex: %s", args[0].Dump())
		case *badExpr:
			return xRight
		default:
			return ctx.newBadExpr(cond, "operands are too complex: (%s,%s)", args[0].Dump(), args[1].Dump())
		}
	}
}

func (ctx *parseContext) parseCheckFunctionCallResult(directive *mkparser.Directive, xValue starlarkExpr,
	varArg *mkparser.MakeString) (starlarkExpr, bool) {
	mkSingleVar, ok := varArg.SingleVariable()
	if !ok {
		return nil, false
	}
	expr := ctx.parseReference(directive, mkSingleVar)
	negate := strings.HasSuffix(directive.Name, "neq")
	checkIsSomethingFunction := func(xCall *callExpr) starlarkExpr {
		s, ok := maybeString(xValue)
		if !ok || s != "true" {
			return ctx.newBadExpr(directive,
				fmt.Sprintf("the result of %s can be compared only to 'true'", xCall.name))
		}
		if len(xCall.args) < 1 {
			return ctx.newBadExpr(directive, "%s requires an argument", xCall.name)
		}
		return nil
	}
	switch x := expr.(type) {
	case *callExpr:
		switch x.name {
		case "filter":
			return ctx.parseCompareFilterFuncResult(directive, x, xValue, !negate), true
		case "filter-out":
			return ctx.parseCompareFilterFuncResult(directive, x, xValue, negate), true
		case "wildcard":
			return ctx.parseCompareWildcardFuncResult(directive, x, xValue, negate), true
		case "findstring":
			return ctx.parseCheckFindstringFuncResult(directive, x, xValue, negate), true
		case "strip":
			return ctx.parseCompareStripFuncResult(directive, x, xValue, negate), true
		case "is-board-platform":
			if xBad := checkIsSomethingFunction(x); xBad != nil {
				return xBad, true
			}
			return &eqExpr{
				left:  &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
				right: x.args[0],
				isEq:  !negate,
			}, true
		case "is-board-platform-in-list":
			if xBad := checkIsSomethingFunction(x); xBad != nil {
				return xBad, true
			}
			return &inExpr{
				expr:  &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
				list:  maybeConvertToStringList(x.args[0]),
				isNot: negate,
			}, true
		case "is-product-in-list":
			if xBad := checkIsSomethingFunction(x); xBad != nil {
				return xBad, true
			}
			return &inExpr{
				expr:  &variableRefExpr{ctx.addVariable("TARGET_PRODUCT"), true},
				list:  maybeConvertToStringList(x.args[0]),
				isNot: negate,
			}, true
		case "is-vendor-board-platform":
			if xBad := checkIsSomethingFunction(x); xBad != nil {
				return xBad, true
			}
			s, ok := maybeString(x.args[0])
			if !ok {
				return ctx.newBadExpr(directive, "cannot handle non-constant argument to is-vendor-board-platform"), true
			}
			return &inExpr{
				expr:  &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
				list:  &variableRefExpr{ctx.addVariable(s + "_BOARD_PLATFORMS"), true},
				isNot: negate,
			}, true
		default:
			return ctx.newBadExpr(directive, "Unknown function in ifeq: %s", x.name), true
		}
	case *badExpr:
		return x, true
	default:
		return nil, false
	}
}

func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
	filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
	// We handle:
	// *  ifeq/ifneq (,$(filter v1 v2 ..., $(VAR)) becomes if VAR not in/in ["v1", "v2", ...]
	// *  ifeq/ifneq (,$(filter $(VAR), v1 v2 ...) becomes if VAR not in/in ["v1", "v2", ...]
	// *  ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"]
	// TODO(Asmundak): check the last case works for filter-out, too.
	xPattern := filterFuncCall.args[0]
	xText := filterFuncCall.args[1]
	var xInList *stringLiteralExpr
	var xVar starlarkExpr
	var ok bool
	switch x := xValue.(type) {
	case *stringLiteralExpr:
		if x.literal != "" {
			return ctx.newBadExpr(cond, "filter comparison to non-empty value: %s", xValue)
		}
		// Either pattern or text should be const, and the
		// non-const one should be varRefExpr
		if xInList, ok = xPattern.(*stringLiteralExpr); ok {
			xVar = xText
		} else if xInList, ok = xText.(*stringLiteralExpr); ok {
			xVar = xPattern
		}
	case *variableRefExpr:
		if v, ok := xPattern.(*variableRefExpr); ok {
			if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() {
				// ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate,
				// it's the opposite to what is done when comparing to empty.
				xVar = xPattern
				negate = !negate
			}
		}
	}
	if xVar != nil && xInList != nil {
		if _, ok := xVar.(*variableRefExpr); ok {
			slExpr := newStringListExpr(strings.Fields(xInList.literal))
			// Generate simpler code for the common cases:
			if xVar.typ() == starlarkTypeList {
				if len(slExpr.items) == 1 {
					// Checking that a string belongs to list
					return &inExpr{isNot: negate, list: xVar, expr: slExpr.items[0]}
				} else {
					// TODO(asmundak):
					panic("TBD")
				}
			}
			return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: xVar}
		}
	}
	return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump())
}

func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
	xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
	if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
		return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
	}
	callFunc := wildcardExistsPhony
	if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") {
		callFunc = fileExistsPhony
	}
	var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool}
	if !negate {
		cc = &notExpr{cc}
	}
	return cc
}

func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
	xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
	if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
		return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue)
	}
	return &eqExpr{
		left: &callExpr{
			object:     xCall.args[1],
			name:       "find",
			args:       []starlarkExpr{xCall.args[0]},
			returnType: starlarkTypeInt,
		},
		right: &intLiteralExpr{-1},
		isEq:  !negate,
	}
}

func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
	xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
	if _, ok := xValue.(*stringLiteralExpr); !ok {
		return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
	}
	return &eqExpr{
		left: &callExpr{
			name:       "strip",
			args:       xCall.args,
			returnType: starlarkTypeString,
		},
		right: xValue, isEq: !negate}
}

// parses $(...), returning an expression
func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
	ref.TrimLeftSpaces()
	ref.TrimRightSpaces()
	refDump := ref.Dump()

	// Handle only the case where the first (or only) word is constant
	words := ref.SplitN(" ", 2)
	if !words[0].Const() {
		return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
	}

	// If it is a single word, it can be a simple variable
	// reference or a function call
	if len(words) == 1 {
		if isMakeControlFunc(refDump) || refDump == "shell" {
			return &callExpr{
				name:       refDump,
				args:       []starlarkExpr{&stringLiteralExpr{""}},
				returnType: starlarkTypeUnknown,
			}
		}
		if v := ctx.addVariable(refDump); v != nil {
			return &variableRefExpr{v, ctx.lastAssignment(v.name()) != nil}
		}
		return ctx.newBadExpr(node, "unknown variable %s", refDump)
	}

	expr := &callExpr{name: words[0].Dump(), returnType: starlarkTypeUnknown}
	args := words[1]
	args.TrimLeftSpaces()
	// Make control functions and shell need special treatment as everything
	// after the name is a single text argument
	if isMakeControlFunc(expr.name) || expr.name == "shell" {
		x := ctx.parseMakeString(node, args)
		if xBad, ok := x.(*badExpr); ok {
			return xBad
		}
		expr.args = []starlarkExpr{x}
		return expr
	}
	if expr.name == "call" {
		words = args.SplitN(",", 2)
		if words[0].Empty() || !words[0].Const() {
			return ctx.newBadExpr(nil, "cannot handle %s", refDump)
		}
		expr.name = words[0].Dump()
		if len(words) < 2 {
			return expr
		}
		args = words[1]
	}
	if kf, found := knownFunctions[expr.name]; found {
		expr.returnType = kf.returnType
	} else {
		return ctx.newBadExpr(node, "cannot handle invoking %s", expr.name)
	}
	switch expr.name {
	case "word":
		return ctx.parseWordFunc(node, args)
	case "subst", "patsubst":
		return ctx.parseSubstFunc(node, expr.name, args)
	default:
		for _, arg := range args.Split(",") {
			arg.TrimLeftSpaces()
			arg.TrimRightSpaces()
			x := ctx.parseMakeString(node, arg)
			if xBad, ok := x.(*badExpr); ok {
				return xBad
			}
			expr.args = append(expr.args, x)
		}
	}
	return expr
}

func (ctx *parseContext) parseSubstFunc(node mkparser.Node, fname string, args *mkparser.MakeString) starlarkExpr {
	words := args.Split(",")
	if len(words) != 3 {
		return ctx.newBadExpr(node, "%s function should have 3 arguments", fname)
	}
	if !words[0].Const() || !words[1].Const() {
		return ctx.newBadExpr(node, "%s function's from and to arguments should be constant", fname)
	}
	from := words[0].Strings[0]
	to := words[1].Strings[0]
	words[2].TrimLeftSpaces()
	words[2].TrimRightSpaces()
	obj := ctx.parseMakeString(node, words[2])
	typ := obj.typ()
	if typ == starlarkTypeString && fname == "subst" {
		// Optimization: if it's $(subst from, to, string), emit string.replace(from, to)
		return &callExpr{
			object:     obj,
			name:       "replace",
			args:       []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}},
			returnType: typ,
		}
	}
	return &callExpr{
		name:       fname,
		args:       []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}, obj},
		returnType: obj.typ(),
	}
}

func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
	words := args.Split(",")
	if len(words) != 2 {
		return ctx.newBadExpr(node, "word function should have 2 arguments")
	}
	var index uint64 = 0
	if words[0].Const() {
		index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64)
	}
	if index < 1 {
		return ctx.newBadExpr(node, "word index should be constant positive integer")
	}
	words[1].TrimLeftSpaces()
	words[1].TrimRightSpaces()
	array := ctx.parseMakeString(node, words[1])
	if xBad, ok := array.(*badExpr); ok {
		return xBad
	}
	if array.typ() != starlarkTypeList {
		array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
	}
	return indexExpr{array, &intLiteralExpr{int(index - 1)}}
}

func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
	if mk.Const() {
		return &stringLiteralExpr{mk.Dump()}
	}
	if mkRef, ok := mk.SingleVariable(); ok {
		return ctx.parseReference(node, mkRef)
	}
	// If we reached here, it's neither string literal nor a simple variable,
	// we need a full-blown interpolation node that will generate
	// "a%b%c" % (X, Y) for a$(X)b$(Y)c
	xInterp := &interpolateExpr{args: make([]starlarkExpr, len(mk.Variables))}
	for i, ref := range mk.Variables {
		arg := ctx.parseReference(node, ref.Name)
		if x, ok := arg.(*badExpr); ok {
			return x
		}
		xInterp.args[i] = arg
	}
	xInterp.chunks = append(xInterp.chunks, mk.Strings...)
	return xInterp
}

// Handles the statements whose treatment is the same in all contexts: comment,
// assignment, variable (which is a macro call in reality) and all constructs that
// do not handle in any context ('define directive and any unrecognized stuff).
// Return true if we handled it.
func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool {
	handled := true
	switch x := node.(type) {
	case *mkparser.Comment:
		ctx.insertComment("#" + x.Comment)
	case *mkparser.Assignment:
		ctx.handleAssignment(x)
	case *mkparser.Variable:
		ctx.handleVariable(x)
	case *mkparser.Directive:
		switch x.Name {
		case "define":
			ctx.handleDefine(x)
		case "include", "-include":
			ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-')
		default:
			handled = false
		}
	default:
		ctx.errorf(x, "unsupported line %s", x.Dump())
	}
	return handled
}

func (ctx *parseContext) insertComment(s string) {
	ctx.receiver.newNode(&commentNode{strings.TrimSpace(s)})
}

func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) {
	for _, line := range strings.Split(failedNode.Dump(), "\n") {
		ctx.insertComment("# " + line)
	}
}

// records that the given node failed to be converted and includes an explanatory message
func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) {
	if ctx.errorLogger != nil {
		ctx.errorLogger.NewError(message, failedNode, args...)
	}
	message = fmt.Sprintf(message, args...)
	ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", message))
	ctx.carryAsComment(failedNode)
	ctx.script.hasErrors = true
}

func (ctx *parseContext) wrapBadExpr(xBad *badExpr) {
	ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", xBad.message))
	ctx.carryAsComment(xBad.node)
}

func (ctx *parseContext) loadedModulePath(path string) string {
	// During the transition to Roboleaf some of the product configuration files
	// will be converted and checked in while the others will be generated on the fly
	// and run. The runner  (rbcrun application) accommodates this by allowing three
	// different ways to specify the loaded file location:
	//  1) load(":<file>",...) loads <file> from the same directory
	//  2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
	//  3) load("/absolute/path/to/<file> absolute path
	// If the file being generated and the file it wants to load are in the same directory,
	// generate option 1.
	// Otherwise, if output directory is not specified, generate 2)
	// Finally, if output directory has been specified and the file being generated and
	// the file it wants to load from are in the different directories, generate 2) or 3):
	//  * if the file being loaded exists in the source tree, generate 2)
	//  * otherwise, generate 3)
	// Finally, figure out the loaded module path and name and create a node for it
	loadedModuleDir := filepath.Dir(path)
	base := filepath.Base(path)
	loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
	if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
		return ":" + loadedModuleName
	}
	if ctx.outputDir == "" {
		return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
	}
	if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
		return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
	}
	return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
}

func (ss *StarlarkScript) String() string {
	return NewGenerateContext(ss).emit()
}

func (ss *StarlarkScript) SubConfigFiles() []string {
	var subs []string
	for _, src := range ss.inherited {
		subs = append(subs, src.originalPath)
	}
	return subs
}

func (ss *StarlarkScript) HasErrors() bool {
	return ss.hasErrors
}

// Convert reads and parses a makefile. If successful, parsed tree
// is returned and then can be passed to String() to get the generated
// Starlark file.
func Convert(req Request) (*StarlarkScript, error) {
	reader := req.Reader
	if reader == nil {
		mkContents, err := ioutil.ReadFile(req.MkFile)
		if err != nil {
			return nil, err
		}
		reader = bytes.NewBuffer(mkContents)
	}
	parser := mkparser.NewParser(req.MkFile, reader)
	nodes, errs := parser.Parse()
	if len(errs) > 0 {
		for _, e := range errs {
			fmt.Fprintln(os.Stderr, "ERROR:", e)
		}
		return nil, fmt.Errorf("bad makefile %s", req.MkFile)
	}
	starScript := &StarlarkScript{
		moduleName:         moduleNameForFile(req.MkFile),
		mkFile:             req.MkFile,
		topDir:             req.RootDir,
		traceCalls:         req.TraceCalls,
		warnPartialSuccess: req.WarnPartialSuccess,
	}
	ctx := newParseContext(starScript, nodes)
	ctx.outputSuffix = req.OutputSuffix
	ctx.outputDir = req.OutputDir
	ctx.errorLogger = req.ErrorLogger
	if len(req.TracedVariables) > 0 {
		ctx.tracedVariables = make(map[string]bool)
		for _, v := range req.TracedVariables {
			ctx.tracedVariables[v] = true
		}
	}
	ctx.pushReceiver(starScript)
	for ctx.hasNodes() && ctx.fatalError == nil {
		node := ctx.getNode()
		if ctx.handleSimpleStatement(node) {
			continue
		}
		switch x := node.(type) {
		case *mkparser.Directive:
			switch x.Name {
			case "ifeq", "ifneq", "ifdef", "ifndef":
				ctx.handleIfBlock(x)
			default:
				ctx.errorf(x, "unexpected directive %s", x.Name)
			}
		default:
			ctx.errorf(x, "unsupported line")
		}
	}
	if ctx.fatalError != nil {
		return nil, ctx.fatalError
	}
	return starScript, nil
}

func Launcher(path, name string) string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
	fmt.Fprintf(&buf, "load(%q, \"init\")\n", path)
	fmt.Fprintf(&buf, "g, config = %s(%q, init)\n", cfnMain, name)
	fmt.Fprintf(&buf, "%s(g, config)\n", cfnPrintVars)
	return buf.String()
}

func MakePath2ModuleName(mkPath string) string {
	return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
}