Define release flags in starlark instead of make
So that we have a more restricted enviornment for this new configuration
axis that can also be imported into other tools more easily.
Test: Manually (this time also tested setting OUT_DIR outside of the tree)
Change-Id: I01d90e06e45cba756156af16f63e04f575877263
diff --git a/core/envsetup.mk b/core/envsetup.mk
index 8887ddc..3e19198 100644
--- a/core/envsetup.mk
+++ b/core/envsetup.mk
@@ -24,19 +24,24 @@
#$(warning $(call find_and_earlier,A B C,C))
#$(warning $(call find_and_earlier,A B C,D))
-# Runs the starlark file given in $(1), and sets all the variables in its top-level
+# Runs a starlark file, and sets all the variables in its top-level
# variables_to_export_to_make variable as make variables.
#
# In order to avoid running starlark every time the stamp file is checked, we use
# $(KATI_shell_no_rerun). Then, to make sure that we actually do rerun kati when
# modifying the starlark files, we add the starlark files to the kati stamp file with
# $(KATI_extra_file_deps).
+#
+# Arguments:
+# $(1): A single starlark file to use as the entrypoint
+# $(2): An optional list of starlark files to NOT include as kati dependencies.
+# $(3): An optional list of extra flags to pass to rbcrun
define run-starlark
$(eval _starlark_results := $(OUT_DIR)/starlark_results/$(subst /,_,$(1)).mk)
-$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results))
+$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(3) $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results))
$(if $(filter-out 0,$(.SHELLSTATUS)),$(error Starlark failed to run))
$(eval include $(_starlark_results))
-$(KATI_extra_file_deps $(LOADED_STARLARK_FILES))
+$(KATI_extra_file_deps $(filter-out $(2),$(LOADED_STARLARK_FILES)))
$(eval LOADED_STARLARK_FILES :=)
$(eval _starlark_results :=)
endef
diff --git a/core/release_config.bzl b/core/release_config.bzl
new file mode 100644
index 0000000..ecb9ec6
--- /dev/null
+++ b/core/release_config.bzl
@@ -0,0 +1,90 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Partitions that get build system flag summaries
+_flag_partitions = [
+ "product",
+ "system",
+ "system_ext",
+ "vendor",
+]
+
+def _combine_dicts_no_duplicate_keys(dicts):
+ result = {}
+ for d in dicts:
+ for k, v in d.items():
+ if k in result:
+ fail("Duplicate key: " + k)
+ result[k] = v
+ return result
+
+def release_config(target_release, flag_definitions, config_maps, fail_if_no_release_config = True):
+ result = {
+ "_ALL_RELEASE_FLAGS": [flag.name for flag in flag_definitions],
+ }
+ all_flags = {}
+ for flag in flag_definitions:
+ if sorted(dir(flag)) != ["default", "name", "partitions"]:
+ fail("Flag structs must contain 3 fields: name, partitions, and default")
+ if not flag.partitions:
+ fail("At least 1 partition is required")
+ for partition in flag.partitions:
+ if partition == "all":
+ if len(flag.partitions) > 1:
+ fail("\"all\" can't be combined with other partitions: " + str(flag.partitions))
+ elif partition not in _flag_partitions:
+ fail("Invalid partition: " + flag.partition + ", allowed partitions: " + str(_flag_partitions))
+ if not flag.name.startswith("RELEASE_"):
+ fail("Release flag names must start with RELEASE_")
+ if " " in flag.name or "\t" in flag.name or "\n" in flag.name:
+ fail("Flag names must not contain whitespace.")
+ if flag.name in all_flags:
+ fail("Duplicate declaration of flag " + flag.name)
+ all_flags[flag.name] = True
+
+ default = flag.default
+ if type(default) == "bool":
+ default = "true" if default else ""
+
+ result["_ALL_RELEASE_FLAGS." + flag.name + ".PARTITIONS"] = flag.partitions
+ result["_ALL_RELEASE_FLAGS." + flag.name + ".DEFAULT"] = default
+ result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = default
+
+ # If TARGET_RELEASE is set, fail if there is no matching release config
+ # If it isn't set, no release config files will be included and all flags
+ # will get their default values.
+ if target_release:
+ config_map = _combine_dicts_no_duplicate_keys(config_maps)
+ if target_release not in config_map:
+ fail("No release config found for TARGET_RELEASE: " + target_release + ". Available releases are: " + str(config_map.keys()))
+ release_config = config_map[target_release]
+ if sorted(dir(release_config)) != ["flags", "release_version"]:
+ fail("A release config must be a struct with a flags and release_version fields")
+ result["_RELEASE_VERSION"] = release_config.release_version
+ for flag in release_config.flags:
+ if sorted(dir(flag)) != ["name", "value"]:
+ fail("A flag must be a struct with name and value fields, got: " + str(sorted(dir(flag))))
+ if flag.name not in all_flags:
+ fail("Undeclared build flag: " + flag.name)
+ value = flag.value
+ if type(value) == "bool":
+ value = "true" if value else ""
+ result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = value
+ elif fail_if_no_release_config:
+ fail("FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not")
+ else:
+ # No TARGET_RELEASE means release version 0
+ result["_RELEASE_VERSION"] = 0
+
+ return result
diff --git a/core/release_config.mk b/core/release_config.mk
index fdfc6a0..ce57855 100644
--- a/core/release_config.mk
+++ b/core/release_config.mk
@@ -12,78 +12,45 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Partitions that get build system flag summaries
-_FLAG_PARTITIONS := system vendor system_ext product
-
-# All possible release flags. Defined in the build_flags.mk files
-# throughout the tree
-_ALL_RELEASE_FLAGS :=
-
-# -----------------------------------------------------------------
-# Choose the flag files
-# Do this first, because we're going to unset TARGET_RELEASE before
-# including anyone, so they don't start making conditionals based on it.
-
# If this is a google source tree, restrict it to only the one file
# which has OWNERS control. If it isn't let others define their own.
# TODO: Remove wildcard for build/release one when all branch manifests
# have updated.
-config_map_files := $(wildcard build/release/release_config_map.mk) \
- $(if $(wildcard vendor/google/release/release_config_map.mk), \
- vendor/google/release/release_config_map.mk, \
+flag_declaration_files := $(wildcard build/release/build_flags.bzl) \
+ $(if $(wildcard vendor/google/release/build_flags.bzl), \
+ vendor/google/release/build_flags.bzl, \
$(sort \
- $(wildcard device/*/release/release_config_map.mk) \
- $(wildcard device/*/*/release/release_config_map.mk) \
- $(wildcard vendor/*/release/release_config_map.mk) \
- $(wildcard vendor/*/*/release/release_config_map.mk) \
+ $(wildcard device/*/release/build_flags.bzl) \
+ $(wildcard device/*/*/release/build_flags.bzl) \
+ $(wildcard vendor/*/release/build_flags.bzl) \
+ $(wildcard vendor/*/*/release/build_flags.bzl) \
+ ) \
+ )
+config_map_files := $(wildcard build/release/release_config_map.bzl) \
+ $(if $(wildcard vendor/google/release/release_config_map.bzl), \
+ vendor/google/release/release_config_map.bzl, \
+ $(sort \
+ $(wildcard device/*/release/release_config_map.bzl) \
+ $(wildcard device/*/*/release/release_config_map.bzl) \
+ $(wildcard vendor/*/release/release_config_map.bzl) \
+ $(wildcard vendor/*/*/release/release_config_map.bzl) \
) \
)
-# $1 config name
-# $2 release config files
-define declare-release-config
- $(eval # No duplicates)
- $(if $(filter $(_all_release_configs), $(strip $(1))), \
- $(error declare-release-config: config $(strip $(1)) declared in: $(_included) Previously declared here: $(_all_release_configs.$(strip $(1)).DECLARED_IN)) \
- )
- $(eval # Must have release config files)
- $(if $(strip $(2)),, \
- $(error declare-release-config: config $(strip $(1)) must have release config files) \
- )
- $(eval _all_release_configs := $(sort $(_all_release_configs) $(strip $(1))))
- $(eval _all_release_configs.$(strip $(1)).DECLARED_IN := $(_included))
- $(eval _all_release_configs.$(strip $(1)).FILES := $(strip $(2)))
-endef
-
-# Include the config map files
-$(foreach f, $(config_map_files), \
- $(eval _included := $(f)) \
- $(eval include $(f)) \
-)
-
-# If TARGET_RELEASE is set, fail if there is no matching release config
-# If it isn't set, no release config files will be included and all flags
-# will get their default values.
-ifneq ($(TARGET_RELEASE),)
-ifeq ($(filter $(_all_release_configs), $(TARGET_RELEASE)),)
- $(error No release config found for TARGET_RELEASE: $(TARGET_RELEASE). Available releases are: $(_all_release_configs))
-else
- # Choose flag files
- # Don't sort this, use it in the order they gave us.
- _release_config_files := $(_all_release_configs.$(TARGET_RELEASE).FILES)
-endif
-else
-# Useful for finding scripts etc that aren't passing or setting TARGET_RELEASE
-ifneq ($(FAIL_IF_NO_RELEASE_CONFIG),)
- $(error FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not)
-endif
-_release_config_files :=
-endif
-
-# Unset variables so they can't use it
-define declare-release-config
-$(error declare-release-config can only be called from inside release_config_map.mk files)
-endef
+# Because starlark can't find files with $(wildcard), write an entrypoint starlark script that
+# contains the result of the above wildcards for the starlark code to use.
+filename_to_starlark=$(subst /,_,$(subst .,_,$(1)))
+_c:=load("//build/make/core/release_config.bzl", "release_config")
+_c+=$(foreach f,$(flag_declaration_files),$(newline)load("//$(f)", flags_$(call filename_to_starlark,$(f)) = "flags"))
+_c+=$(foreach f,$(config_map_files),$(newline)load("//$(f)", config_maps_$(call filename_to_starlark,$(f)) = "config_maps"))
+_c+=$(newline)all_flags = [] $(foreach f,$(flag_declaration_files),+ flags_$(call filename_to_starlark,$(f)))
+_c+=$(newline)all_config_maps = [$(foreach f,$(config_map_files),config_maps_$(call filename_to_starlark,$(f))$(comma))]
+_c+=$(newline)target_release = "$(TARGET_RELEASE)"
+_c+=$(newline)fail_if_no_release_config = True if "$(FAIL_IF_NO_RELEASE_CONFIG)" else False
+_c+=$(newline)variables_to_export_to_make = release_config(target_release, all_flags, all_config_maps, fail_if_no_release_config)
+$(file >$(OUT_DIR)/release_config_entrypoint.bzl,$(_c))
+_c:=
+filename_to_starlark:=
# TODO: Remove this check after enough people have sourced lunch that we don't
# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023
@@ -96,116 +63,13 @@
endif
.KATI_READONLY := TARGET_RELEASE
-$(foreach config, $(_all_release_configs), \
- $(eval _all_release_configs.$(config).DECLARED_IN:= ) \
- $(eval _all_release_configs.$(config).FILES:= ) \
-)
-_all_release_configs:=
-config_map_files:=
-
-# -----------------------------------------------------------------
-# Declare the flags
-
-# $1 partition(s)
-# $2 flag name. Must start with RELEASE_
-# $3 default. True or false
-define declare-build-flag
- $(if $(filter-out all $(_FLAG_PARTITIONS), $(strip $(1))), \
- $(error declare-build-flag: invalid partitions: $(strip $(1))) \
- )
- $(if $(and $(filter all,$(strip $(1))),$(filter-out all, $(strip $(1)))), \
- $(error declare-build-flag: "all" can't be combined with other partitions: $(strip $(1))), \
- $(eval declare-build-flag.partition := $(_FLAG_PARTITIONS)) \
- )
- $(if $(filter-out RELEASE_%, $(strip $(2))), \
- $(error declare-build-flag: Release flag names must start with RELEASE_: $(strip $(2))) \
- )
- $(eval _ALL_RELEASE_FLAGS += $(strip $(2)))
- $(foreach partition, $(declare-build-flag.partition), \
- $(eval _ALL_RELEASE_FLAGS.PARTITIONS.$(partition) := $(sort \
- $(_ALL_RELEASE_FLAGS.PARTITIONS.$(partition)) $(strip $(2)))) \
- )
- $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).PARTITIONS := $(declare-build-flag.partition))
- $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).DEFAULT := $(strip $(3)))
- $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).DECLARED_IN := $(_included))
- $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).VALUE := $(strip $(3)))
- $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).SET_IN := $(_included))
- $(eval declare-build-flag.partition:=)
-endef
-
-
-# Choose the files
-# If this is a google source tree, restrict it to only the one file
-# which has OWNERS control. If it isn't let others define their own.
-flag_declaration_files := $(wildcard build/release/build_flags.mk) \
- $(if $(wildcard vendor/google/release/build_flags.mk), \
- vendor/google/release/build_flags.mk, \
- $(sort \
- $(wildcard device/*/release/build_flags.mk) \
- $(wildcard device/*/*/release/build_flags.mk) \
- $(wildcard vendor/*/release/build_flags.mk) \
- $(wildcard vendor/*/*/release/build_flags.mk) \
- ) \
- )
-
-# Include the files
-$(foreach f, $(flag_declaration_files), \
- $(eval _included := $(f)) \
- $(eval include $(f)) \
-)
-
-# Don't let anyone declare build flags after here
-define declare-build-flag
-$(error declare-build-flag can only be called from inside flag definition files.)
-endef
-
-# No more flags from here on
-.KATI_READONLY := _ALL_RELEASE_FLAGS
-
-# -----------------------------------------------------------------
-# Set the flags
-
-# $(1): Flag name. Must start with RELEASE_ and have been defined by declare-build-flag
-# $(2): Value. True or false
-define set-build-flag
- $(if $(filter-out $(_ALL_RELEASE_FLAGS), $(strip $(1))), \
- $(error set-build-flag: Undeclared build flag: $(strip $(1))) \
- )
- $(eval _ALL_RELEASE_FLAGS.$(strip $(1)).VALUE := $(strip $(2)))
- $(eval _ALL_RELEASE_FLAGS.$(strip $(1)).SET_IN := $(_included))
-endef
-
-# This writes directly to a file so that the version never exists in make for
-# people to write conditionals upon.
-define set-release-version
- $(eval _RELEASE_VERSION := $(strip $(1)))
-endef
-
-# Include the files (if there are any)
-ifneq ($(strip $(_release_config_files)),)
- $(foreach f, $(_release_config_files), \
- $(eval _included := $(f)) \
- $(eval include $(f)) \
- )
-else
- # No TARGET_RELEASE means release version 0
- $(call set-release-version, 0)
-endif
-
-
-ifeq ($(_RELEASE_VERSION)),)
- $(error No release config file called set-release-version. Included files were: $(_release_config_files))
-endif
-
-# Don't let anyone declare build flags after here
-define set-build-flag
-$(error set-build-flag can only be called from inside release config files.)
-endef
-
-# Don't let anyone set the release version after here
-define set-release-version
-$(error set-release-version can only be called from inside release config files.)
-endef
+# Exclude the entrypoint file as a dependency (by passing it as the 2nd argument) so that we don't
+# rerun kati every build. Kati will replay the $(file) command that generates it every build,
+# updating its timestamp.
+#
+# We also need to pass --allow_external_entrypoint to rbcrun in case the OUT_DIR is set to something
+# outside of the source tree.
+$(call run-starlark,$(OUT_DIR)/release_config_entrypoint.bzl,$(OUT_DIR)/release_config_entrypoint.bzl,--allow_external_entrypoint)
# Set the flag values, and don't allow any one to modify them.
$(foreach flag, $(_ALL_RELEASE_FLAGS), \
@@ -213,10 +77,3 @@
$(eval .KATI_READONLY := $(flag)) \
)
-
-# -----------------------------------------------------------------
-# Clear out vars
-flag_declaration_files:=
-flag_files:=
-_included:=
-_release_config_files:=
diff --git a/tools/rbcrun/host.go b/tools/rbcrun/host.go
index a0fb9e1..1d68d43 100644
--- a/tools/rbcrun/host.go
+++ b/tools/rbcrun/host.go
@@ -34,9 +34,10 @@
ExecutionModeMake ExecutionMode = iota
)
+const allowExternalEntrypointKey = "allowExternalEntrypoint"
const callerDirKey = "callerDir"
-const shellKey = "shell"
const executionModeKey = "executionMode"
+const shellKey = "shell"
type modentry struct {
globals starlark.StringDict
@@ -64,7 +65,7 @@
// Takes a module name (the first argument to the load() function) and returns the path
// it's trying to load, stripping out leading //, and handling leading :s.
-func cleanModuleName(moduleName string, callerDir string) (string, error) {
+func cleanModuleName(moduleName string, callerDir string, allowExternalPaths bool) (string, error) {
if strings.Count(moduleName, ":") > 1 {
return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName)
}
@@ -82,7 +83,7 @@
} else if strings.HasPrefix(moduleName, ":") {
moduleName = moduleName[1:]
localLoad = true
- } else {
+ } else if !allowExternalPaths {
return "", fmt.Errorf("load path must start with // or :")
}
@@ -93,11 +94,13 @@
if filepath.Clean(moduleName) != moduleName {
return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName))
}
- if strings.HasPrefix(moduleName, "../") {
- return "", fmt.Errorf("load path must not start with ../: %s", moduleName)
- }
- if strings.HasPrefix(moduleName, "/") {
- return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)
+ if !allowExternalPaths {
+ if strings.HasPrefix(moduleName, "../") {
+ return "", fmt.Errorf("load path must not start with ../: %s", moduleName)
+ }
+ if strings.HasPrefix(moduleName, "/") {
+ return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)
+ }
}
if localLoad {
@@ -114,17 +117,18 @@
// bound to None if file is missing.
func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) {
mode := thread.Local(executionModeKey).(ExecutionMode)
+ allowExternalEntrypoint := thread.Local(allowExternalEntrypointKey).(bool)
var defaultSymbol string
mustLoad := true
if mode == ExecutionModeRbc {
pipePos := strings.LastIndex(module, "|")
- mustLoad = pipePos < 0
- if !mustLoad {
+ if pipePos >= 0 {
+ mustLoad = false
defaultSymbol = module[pipePos+1:]
module = module[:pipePos]
}
}
- modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string))
+ modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string), allowExternalEntrypoint)
if err != nil {
return nil, err
}
@@ -155,9 +159,11 @@
childThread.SetLocal(testReporterKey, v)
}
+ // Only the entrypoint starlark file allows external loads.
+ childThread.SetLocal(allowExternalEntrypointKey, false)
childThread.SetLocal(callerDirKey, filepath.Dir(modulePath))
- childThread.SetLocal(shellKey, thread.Local(shellKey))
childThread.SetLocal(executionModeKey, mode)
+ childThread.SetLocal(shellKey, thread.Local(shellKey))
if mode == ExecutionModeRbc {
globals, err := starlark.ExecFile(childThread, modulePath, nil, rbcBuiltins)
e = &modentry{globals, err}
@@ -318,7 +324,7 @@
// * src is an optional source of bytes to use instead of filename
// (it can be a string, or a byte array, or an io.Reader instance)
// Returns the top-level starlark variables, the list of starlark files loaded, and an error
-func Run(filename string, src interface{}, mode ExecutionMode) (starlark.StringDict, []string, error) {
+func Run(filename string, src interface{}, mode ExecutionMode, allowExternalEntrypoint bool) (starlark.StringDict, []string, error) {
// NOTE(asmundak): OS-specific. Behave similar to Linux `system` call,
// which always uses /bin/sh to run the command
shellPath := "/bin/sh"
@@ -347,7 +353,7 @@
if err != nil {
return nil, nil, err
}
- if strings.HasPrefix(filename, "../") {
+ if !allowExternalEntrypoint && strings.HasPrefix(filename, "../") {
return nil, nil, fmt.Errorf("path could not be made relative to workspace root: %s", filename)
}
} else {
@@ -358,9 +364,10 @@
moduleCache[filename] = nil
var results starlark.StringDict
+ mainThread.SetLocal(allowExternalEntrypointKey, allowExternalEntrypoint)
mainThread.SetLocal(callerDirKey, filepath.Dir(filename))
- mainThread.SetLocal(shellKey, shellPath)
mainThread.SetLocal(executionModeKey, mode)
+ mainThread.SetLocal(shellKey, shellPath)
if mode == ExecutionModeRbc {
results, err = starlark.ExecFile(mainThread, filename, src, rbcBuiltins)
} else if mode == ExecutionModeMake {
diff --git a/tools/rbcrun/host_test.go b/tools/rbcrun/host_test.go
index 10cac62..10ce55e 100644
--- a/tools/rbcrun/host_test.go
+++ b/tools/rbcrun/host_test.go
@@ -125,6 +125,7 @@
if err := os.Chdir(filepath.Dir(dir)); err != nil {
t.Fatal(err)
}
+ thread.SetLocal(allowExternalEntrypointKey, false)
thread.SetLocal(callerDirKey, dir)
thread.SetLocal(executionModeKey, ExecutionModeRbc)
if _, err := starlark.ExecFile(thread, "testdata/load.star", nil, rbcBuiltins); err != nil {
diff --git a/tools/rbcrun/rbcrun/rbcrun.go b/tools/rbcrun/rbcrun/rbcrun.go
index b5182f0..a15b867 100644
--- a/tools/rbcrun/rbcrun/rbcrun.go
+++ b/tools/rbcrun/rbcrun/rbcrun.go
@@ -26,6 +26,7 @@
)
var (
+ allowExternalEntrypoint = flag.Bool("allow_external_entrypoint", false, "allow the entrypoint starlark file to be outside of the source tree")
modeFlag = flag.String("mode", "", "the general behavior of rbcrun. Can be \"rbc\" or \"make\". Required.")
rootdir = flag.String("d", ".", "the value of // for load paths")
perfFile = flag.String("perf", "", "save performance data")
@@ -159,7 +160,7 @@
quit("%s\n", err)
}
}
- variables, loadedStarlarkFiles, err := rbcrun.Run(filename, nil, mode)
+ variables, loadedStarlarkFiles, err := rbcrun.Run(filename, nil, mode, *allowExternalEntrypoint)
rc := 0
if *perfFile != "" {
if err2 := starlark.StopProfile(); err2 != nil {