Merge "Include all_aconfig_declarations in docs dist" into main
diff --git a/android/Android.bp b/android/Android.bp
index 02b7444..03619f4 100644
--- a/android/Android.bp
+++ b/android/Android.bp
@@ -16,7 +16,6 @@
"soong-remoteexec",
"soong-response",
"soong-shared",
- "soong-starlark",
"soong-starlark-format",
"soong-ui-metrics_proto",
"soong-android-allowlists",
diff --git a/android/config.go b/android/config.go
index 396b685..1bb1a22 100644
--- a/android/config.go
+++ b/android/config.go
@@ -1345,13 +1345,16 @@
panic(fmt.Errorf("Cannot parse vendor API level %s to an integer: %s",
c.VendorApiLevel(), err))
}
- if vendorApiLevel < 202404 || vendorApiLevel%100 != 4 {
- panic("Unknown vendor API level " + c.VendorApiLevel())
- }
// The version before trunk stable is 34.
if vendorApiLevel == 202404 {
return "34"
}
+ if vendorApiLevel >= 1 && vendorApiLevel <= 34 {
+ return strconv.Itoa(vendorApiLevel - 1)
+ }
+ if vendorApiLevel < 202404 || vendorApiLevel%100 != 4 {
+ panic("Unknown vendor API level " + c.VendorApiLevel())
+ }
return strconv.Itoa(vendorApiLevel - 100)
}
diff --git a/android/ninja_deps.go b/android/ninja_deps.go
index 1d50a47..bdf465e 100644
--- a/android/ninja_deps.go
+++ b/android/ninja_deps.go
@@ -15,7 +15,6 @@
package android
import (
- "android/soong/starlark_import"
"sort"
)
@@ -43,11 +42,4 @@
func (ninjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) {
ctx.AddNinjaFileDeps(ctx.Config().ninjaFileDeps()...)
-
- deps, err := starlark_import.GetNinjaDeps()
- if err != nil {
- ctx.Errorf("Error running starlark code: %s", err)
- } else {
- ctx.AddNinjaFileDeps(deps...)
- }
}
diff --git a/cmd/soong_build/queryview.go b/cmd/soong_build/queryview.go
index 5c2316a..eafd67a 100644
--- a/cmd/soong_build/queryview.go
+++ b/cmd/soong_build/queryview.go
@@ -22,7 +22,6 @@
"android/soong/android"
"android/soong/bp2build"
- "android/soong/starlark_import"
)
// A helper function to generate a Read-only Bazel workspace in outDir
@@ -47,14 +46,6 @@
}
}
- // Add starlark deps here, so that they apply to both queryview and apibp2build which
- // both run this function.
- starlarkDeps, err2 := starlark_import.GetNinjaDeps()
- if err2 != nil {
- return err2
- }
- ctx.AddNinjaFileDeps(starlarkDeps...)
-
return nil
}
diff --git a/sdk/update.go b/sdk/update.go
index 095e0c2..a731414 100644
--- a/sdk/update.go
+++ b/sdk/update.go
@@ -89,19 +89,6 @@
indentLevel int
}
-// generatedFile abstracts operations for writing contents into a file and emit a build rule
-// for the file.
-type generatedFile struct {
- generatedContents
- path android.OutputPath
-}
-
-func newGeneratedFile(ctx android.ModuleContext, path ...string) *generatedFile {
- return &generatedFile{
- path: android.PathForModuleOut(ctx, path...).OutputPath,
- }
-}
-
func (gc *generatedContents) Indent() {
gc.indentLevel++
}
@@ -122,26 +109,6 @@
_, _ = fmt.Fprintf(&(gc.content), format, args...)
}
-func (gf *generatedFile) build(pctx android.PackageContext, ctx android.BuilderContext, implicits android.Paths) {
- rb := android.NewRuleBuilder(pctx, ctx)
-
- content := gf.content.String()
-
- // ninja consumes newline characters in rspfile_content. Prevent it by
- // escaping the backslash in the newline character. The extra backslash
- // is removed when the rspfile is written to the actual script file
- content = strings.ReplaceAll(content, "\n", "\\n")
-
- rb.Command().
- Implicits(implicits).
- Text("echo -n").Text(proptools.ShellEscape(content)).
- // convert \\n to \n
- Text("| sed 's/\\\\n/\\n/g' >").Output(gf.path)
- rb.Command().
- Text("chmod a+x").Output(gf.path)
- rb.Build(gf.path.Base(), "Build "+gf.path.Base())
-}
-
// Collect all the members.
//
// Updates the sdk module with a list of sdkMemberVariantDep instances and details as to which
@@ -170,7 +137,7 @@
var container android.Module
if parent != ctx.Module() {
- container = parent.(android.Module)
+ container = parent
}
minApiLevel := android.MinApiLevelForSdkSnapshot(ctx, child)
@@ -179,7 +146,7 @@
s.memberVariantDeps = append(s.memberVariantDeps, sdkMemberVariantDep{
sdkVariant: s,
memberType: memberType,
- variant: child.(android.Module),
+ variant: child,
minApiLevel: minApiLevel,
container: container,
export: export,
@@ -375,7 +342,7 @@
snapshotDir := android.PathForModuleOut(ctx, "snapshot")
- bp := newGeneratedFile(ctx, "snapshot", "Android.bp")
+ bp := android.PathForModuleOut(ctx, "snapshot", "Android.bp")
bpFile := &bpFile{
modules: make(map[string]*bpModule),
@@ -389,7 +356,7 @@
sdk: s,
snapshotDir: snapshotDir.OutputPath,
copies: make(map[string]string),
- filesToZip: []android.Path{bp.path},
+ filesToZip: []android.Path{bp},
bpFile: bpFile,
prebuiltModules: make(map[string]*bpModule),
allMembersByName: allMembersByName,
@@ -463,17 +430,14 @@
}
// generate Android.bp
- bp = newGeneratedFile(ctx, "snapshot", "Android.bp")
- generateBpContents(&bp.generatedContents, bpFile)
-
- contents := bp.content.String()
+ contents := generateBpContents(bpFile)
// If the snapshot is being generated for the current build release then check the syntax to make
// sure that it is compatible.
if targetBuildRelease == buildReleaseCurrent {
syntaxCheckSnapshotBpFile(ctx, contents)
}
- bp.build(pctx, ctx, nil)
+ android.WriteFileRuleVerbatim(ctx, bp, contents)
// Copy the build number file into the snapshot.
builder.CopyToSnapshot(ctx.Config().BuildNumberFile(ctx), BUILD_NUMBER_FILE)
@@ -522,16 +486,14 @@
modules := s.generateInfoData(ctx, memberVariantDeps)
// Output the modules information as pretty printed JSON.
- info := newGeneratedFile(ctx, fmt.Sprintf("%s%s.info", ctx.ModuleName(), snapshotFileSuffix))
+ info := android.PathForModuleOut(ctx, fmt.Sprintf("%s%s.info", ctx.ModuleName(), snapshotFileSuffix))
output, err := json.MarshalIndent(modules, "", " ")
if err != nil {
ctx.ModuleErrorf("error generating %q: %s", info, err)
}
builder.infoContents = string(output)
- info.generatedContents.UnindentedPrintf("%s", output)
- info.build(pctx, ctx, nil)
- infoPath := info.path
- installedInfo := ctx.InstallFile(android.PathForMainlineSdksInstall(ctx), infoPath.Base(), infoPath)
+ android.WriteFileRuleVerbatim(ctx, info, builder.infoContents)
+ installedInfo := ctx.InstallFile(android.PathForMainlineSdksInstall(ctx), info.Base(), info)
s.infoFile = android.OptionalPathForPath(installedInfo)
// Install the zip, making sure that the info file has been installed as well.
@@ -718,105 +680,6 @@
}
}
-// snapshotModuleStaticProperties contains snapshot static (i.e. not dynamically generated) properties.
-type snapshotModuleStaticProperties struct {
- Compile_multilib string `android:"arch_variant"`
-}
-
-// combinedSnapshotModuleProperties are the properties that are associated with the snapshot module.
-type combinedSnapshotModuleProperties struct {
- // The sdk variant from which this information was collected.
- sdkVariant *sdk
-
- // Static snapshot module properties.
- staticProperties *snapshotModuleStaticProperties
-
- // The dynamically generated member list properties.
- dynamicProperties interface{}
-}
-
-// collateSnapshotModuleInfo collates all the snapshot module info from supplied sdk variants.
-func (s *sdk) collateSnapshotModuleInfo(ctx android.BaseModuleContext, sdkVariants []*sdk, memberVariantDeps []sdkMemberVariantDep) []*combinedSnapshotModuleProperties {
- sdkVariantToCombinedProperties := map[*sdk]*combinedSnapshotModuleProperties{}
- var list []*combinedSnapshotModuleProperties
- for _, sdkVariant := range sdkVariants {
- staticProperties := &snapshotModuleStaticProperties{
- Compile_multilib: sdkVariant.multilibUsages.String(),
- }
- dynamicProperties := s.dynamicSdkMemberTypes.createMemberTypeListProperties()
-
- combinedProperties := &combinedSnapshotModuleProperties{
- sdkVariant: sdkVariant,
- staticProperties: staticProperties,
- dynamicProperties: dynamicProperties,
- }
- sdkVariantToCombinedProperties[sdkVariant] = combinedProperties
-
- list = append(list, combinedProperties)
- }
-
- for _, memberVariantDep := range memberVariantDeps {
- // If the member dependency is internal then do not add the dependency to the snapshot member
- // list properties.
- if !memberVariantDep.export {
- continue
- }
-
- combined := sdkVariantToCombinedProperties[memberVariantDep.sdkVariant]
- memberListProperty := s.memberTypeListProperty(memberVariantDep.memberType)
- memberName := ctx.OtherModuleName(memberVariantDep.variant)
-
- if memberListProperty.getter == nil {
- continue
- }
-
- // Append the member to the appropriate list, if it is not already present in the list.
- memberList := memberListProperty.getter(combined.dynamicProperties)
- if !android.InList(memberName, memberList) {
- memberList = append(memberList, memberName)
- }
- memberListProperty.setter(combined.dynamicProperties, memberList)
- }
-
- return list
-}
-
-func (s *sdk) optimizeSnapshotModuleProperties(ctx android.ModuleContext, list []*combinedSnapshotModuleProperties) *combinedSnapshotModuleProperties {
-
- // Extract the dynamic properties and add them to a list of propertiesContainer.
- propertyContainers := []propertiesContainer{}
- for _, i := range list {
- propertyContainers = append(propertyContainers, sdkVariantPropertiesContainer{
- sdkVariant: i.sdkVariant,
- properties: i.dynamicProperties,
- })
- }
-
- // Extract the common members, removing them from the original properties.
- commonDynamicProperties := s.dynamicSdkMemberTypes.createMemberTypeListProperties()
- extractor := newCommonValueExtractor(commonDynamicProperties)
- extractCommonProperties(ctx, extractor, commonDynamicProperties, propertyContainers)
-
- // Extract the static properties and add them to a list of propertiesContainer.
- propertyContainers = []propertiesContainer{}
- for _, i := range list {
- propertyContainers = append(propertyContainers, sdkVariantPropertiesContainer{
- sdkVariant: i.sdkVariant,
- properties: i.staticProperties,
- })
- }
-
- commonStaticProperties := &snapshotModuleStaticProperties{}
- extractor = newCommonValueExtractor(commonStaticProperties)
- extractCommonProperties(ctx, extractor, &commonStaticProperties, propertyContainers)
-
- return &combinedSnapshotModuleProperties{
- sdkVariant: nil,
- staticProperties: commonStaticProperties,
- dynamicProperties: commonDynamicProperties,
- }
-}
-
type propertyTag struct {
name string
}
@@ -885,7 +748,8 @@
}
}
-func generateBpContents(contents *generatedContents, bpFile *bpFile) {
+func generateBpContents(bpFile *bpFile) string {
+ contents := &generatedContents{}
contents.IndentedPrintf("// This is auto-generated. DO NOT EDIT.\n")
for _, bpModule := range bpFile.order {
contents.IndentedPrintf("\n")
@@ -893,6 +757,7 @@
outputPropertySet(contents, bpModule.bpPropertySet)
contents.IndentedPrintf("}\n")
}
+ return contents.content.String()
}
func outputPropertySet(contents *generatedContents, set *bpPropertySet) {
@@ -1007,7 +872,7 @@
contents.IndentedPrintf("}")
default:
- panic(fmt.Errorf("Unknown type: %T of value %#v", value, value))
+ panic(fmt.Errorf("unknown type: %T of value %#v", value, value))
}
}
@@ -1018,9 +883,7 @@
}
func (s *sdk) GetAndroidBpContentsForTests() string {
- contents := &generatedContents{}
- generateBpContents(contents, s.builderForTests.bpFile)
- return contents.content.String()
+ return generateBpContents(s.builderForTests.bpFile)
}
func (s *sdk) GetInfoContentsForTests() string {
@@ -1334,7 +1197,7 @@
case "lib64":
return m | multilib64
default:
- panic(fmt.Errorf("Unknown Multilib field in ArchType, expected 'lib32' or 'lib64', found %q", multilib))
+ panic(fmt.Errorf("unknown Multilib field in ArchType, expected 'lib32' or 'lib64', found %q", multilib))
}
}
@@ -1349,7 +1212,7 @@
case multilibBoth:
return "both"
default:
- panic(fmt.Errorf("Unknown multilib value, found %b, expected one of %b, %b, %b or %b",
+ panic(fmt.Errorf("unknown multilib value, found %b, expected one of %b, %b, %b or %b",
m, multilibNone, multilib32, multilib64, multilibBoth))
}
}
@@ -2302,20 +2165,6 @@
optimizableProperties() interface{}
}
-// A wrapper for sdk variant related properties to allow them to be optimized.
-type sdkVariantPropertiesContainer struct {
- sdkVariant *sdk
- properties interface{}
-}
-
-func (c sdkVariantPropertiesContainer) optimizableProperties() interface{} {
- return c.properties
-}
-
-func (c sdkVariantPropertiesContainer) String() string {
- return c.sdkVariant.String()
-}
-
// Extract common properties from a slice of property structures of the same type.
//
// All the property structures must be of the same type.
diff --git a/starlark_import/Android.bp b/starlark_import/Android.bp
deleted file mode 100644
index b43217b..0000000
--- a/starlark_import/Android.bp
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2023 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package {
- default_applicable_licenses: ["Android-Apache-2.0"],
-}
-
-bootstrap_go_package {
- name: "soong-starlark",
- pkgPath: "android/soong/starlark_import",
- srcs: [
- "starlark_import.go",
- "unmarshal.go",
- ],
- testSrcs: [
- "starlark_import_test.go",
- "unmarshal_test.go",
- ],
- deps: [
- "go-starlark-starlark",
- "go-starlark-starlarkstruct",
- "go-starlark-starlarkjson",
- "go-starlark-starlarktest",
- ],
-}
diff --git a/starlark_import/README.md b/starlark_import/README.md
deleted file mode 100644
index e444759..0000000
--- a/starlark_import/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-# starlark_import package
-
-This allows soong to read constant information from starlark files. At package initialization
-time, soong will read `build/bazel/constants_exported_to_soong.bzl`, and then make the
-variables from that file available via `starlark_import.GetStarlarkValue()`. So to import
-a new variable, it must be added to `constants_exported_to_soong.bzl` and then it can
-be accessed by name.
-
-Only constant information can be read, since this is not a full bazel execution but a
-standalone starlark interpreter. This means you can't use bazel contructs like `rule`,
-`provider`, `select`, `glob`, etc.
-
-All starlark files that were loaded must be added as ninja deps that cause soong to rerun.
-The loaded files can be retrieved via `starlark_import.GetNinjaDeps()`.
diff --git a/starlark_import/starlark_import.go b/starlark_import/starlark_import.go
deleted file mode 100644
index ebe4247..0000000
--- a/starlark_import/starlark_import.go
+++ /dev/null
@@ -1,306 +0,0 @@
-// Copyright 2023 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package starlark_import
-
-import (
- "fmt"
- "os"
- "path/filepath"
- "sort"
- "strings"
- "sync"
- "time"
-
- "go.starlark.net/starlark"
- "go.starlark.net/starlarkjson"
- "go.starlark.net/starlarkstruct"
-)
-
-func init() {
- go func() {
- startTime := time.Now()
- v, d, err := runStarlarkFile("//build/bazel/constants_exported_to_soong.bzl")
- endTime := time.Now()
- //fmt.Fprintf(os.Stderr, "starlark run time: %s\n", endTime.Sub(startTime).String())
- globalResult.Set(starlarkResult{
- values: v,
- ninjaDeps: d,
- err: err,
- startTime: startTime,
- endTime: endTime,
- })
- }()
-}
-
-type starlarkResult struct {
- values starlark.StringDict
- ninjaDeps []string
- err error
- startTime time.Time
- endTime time.Time
-}
-
-// setOnce wraps a value and exposes Set() and Get() accessors for it.
-// The Get() calls will block until a Set() has been called.
-// A second call to Set() will panic.
-// setOnce must be created using newSetOnce()
-type setOnce[T any] struct {
- value T
- lock sync.Mutex
- wg sync.WaitGroup
- isSet bool
-}
-
-func (o *setOnce[T]) Set(value T) {
- o.lock.Lock()
- defer o.lock.Unlock()
- if o.isSet {
- panic("Value already set")
- }
-
- o.value = value
- o.isSet = true
- o.wg.Done()
-}
-
-func (o *setOnce[T]) Get() T {
- if !o.isSet {
- o.wg.Wait()
- }
- return o.value
-}
-
-func newSetOnce[T any]() *setOnce[T] {
- result := &setOnce[T]{}
- result.wg.Add(1)
- return result
-}
-
-var globalResult = newSetOnce[starlarkResult]()
-
-func GetStarlarkValue[T any](key string) (T, error) {
- result := globalResult.Get()
- if result.err != nil {
- var zero T
- return zero, result.err
- }
- if !result.values.Has(key) {
- var zero T
- return zero, fmt.Errorf("a starlark variable by that name wasn't found, did you update //build/bazel/constants_exported_to_soong.bzl?")
- }
- return Unmarshal[T](result.values[key])
-}
-
-func GetNinjaDeps() ([]string, error) {
- result := globalResult.Get()
- if result.err != nil {
- return nil, result.err
- }
- return result.ninjaDeps, nil
-}
-
-func getTopDir() (string, error) {
- // It's hard to communicate the top dir to this package in any other way than reading the
- // arguments directly, because we need to know this at package initialization time. Many
- // soong constants that we'd like to read from starlark are initialized during package
- // initialization.
- for i, arg := range os.Args {
- if arg == "--top" {
- if i < len(os.Args)-1 && os.Args[i+1] != "" {
- return os.Args[i+1], nil
- }
- }
- }
-
- // When running tests, --top is not passed. Instead, search for the top dir manually
- cwd, err := os.Getwd()
- if err != nil {
- return "", err
- }
- for cwd != "/" {
- if _, err := os.Stat(filepath.Join(cwd, "build/soong/soong_ui.bash")); err == nil {
- return cwd, nil
- }
- cwd = filepath.Dir(cwd)
- }
- return "", fmt.Errorf("could not find top dir")
-}
-
-const callerDirKey = "callerDir"
-
-type modentry struct {
- globals starlark.StringDict
- err error
-}
-
-func unsupportedMethod(t *starlark.Thread, fn *starlark.Builtin, _ starlark.Tuple, _ []starlark.Tuple) (starlark.Value, error) {
- return nil, fmt.Errorf("%sthis file is read by soong, and must therefore be pure starlark and include only constant information. %q is not allowed", t.CallStack().String(), fn.Name())
-}
-
-var builtins = starlark.StringDict{
- "aspect": starlark.NewBuiltin("aspect", unsupportedMethod),
- "glob": starlark.NewBuiltin("glob", unsupportedMethod),
- "json": starlarkjson.Module,
- "provider": starlark.NewBuiltin("provider", unsupportedMethod),
- "rule": starlark.NewBuiltin("rule", unsupportedMethod),
- "struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
- "select": starlark.NewBuiltin("select", unsupportedMethod),
- "transition": starlark.NewBuiltin("transition", unsupportedMethod),
-}
-
-// Takes a module name (the first argument to the load() function) and returns the path
-// it's trying to load, stripping out leading //, and handling leading :s.
-func cleanModuleName(moduleName string, callerDir string) (string, error) {
- if strings.Count(moduleName, ":") > 1 {
- return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName)
- }
-
- // We don't have full support for external repositories, but at least support skylib's dicts.
- if moduleName == "@bazel_skylib//lib:dicts.bzl" {
- return "external/bazel-skylib/lib/dicts.bzl", nil
- }
-
- localLoad := false
- if strings.HasPrefix(moduleName, "@//") {
- moduleName = moduleName[3:]
- } else if strings.HasPrefix(moduleName, "//") {
- moduleName = moduleName[2:]
- } else if strings.HasPrefix(moduleName, ":") {
- moduleName = moduleName[1:]
- localLoad = true
- } else {
- return "", fmt.Errorf("load path must start with // or :")
- }
-
- if ix := strings.LastIndex(moduleName, ":"); ix >= 0 {
- moduleName = moduleName[:ix] + string(os.PathSeparator) + moduleName[ix+1:]
- }
-
- if filepath.Clean(moduleName) != moduleName {
- return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName))
- }
- if strings.HasPrefix(moduleName, "../") {
- return "", fmt.Errorf("load path must not start with ../: %s", moduleName)
- }
- if strings.HasPrefix(moduleName, "/") {
- return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)
- }
-
- if localLoad {
- return filepath.Join(callerDir, moduleName), nil
- }
-
- return moduleName, nil
-}
-
-// loader implements load statement. The format of the loaded module URI is
-//
-// [//path]:base
-//
-// The file path is $ROOT/path/base if path is present, <caller_dir>/base otherwise.
-func loader(thread *starlark.Thread, module string, topDir string, moduleCache map[string]*modentry, moduleCacheLock *sync.Mutex, filesystem map[string]string) (starlark.StringDict, error) {
- modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string))
- if err != nil {
- return nil, err
- }
- moduleCacheLock.Lock()
- e, ok := moduleCache[modulePath]
- if e == nil {
- if ok {
- moduleCacheLock.Unlock()
- return nil, fmt.Errorf("cycle in load graph")
- }
-
- // Add a placeholder to indicate "load in progress".
- moduleCache[modulePath] = nil
- moduleCacheLock.Unlock()
-
- childThread := &starlark.Thread{Name: "exec " + module, Load: thread.Load}
-
- // Cheating for the sake of testing:
- // propagate starlarktest's Reporter key, otherwise testing
- // the load function may cause panic in starlarktest code.
- const testReporterKey = "Reporter"
- if v := thread.Local(testReporterKey); v != nil {
- childThread.SetLocal(testReporterKey, v)
- }
-
- childThread.SetLocal(callerDirKey, filepath.Dir(modulePath))
-
- if filesystem != nil {
- globals, err := starlark.ExecFile(childThread, filepath.Join(topDir, modulePath), filesystem[modulePath], builtins)
- e = &modentry{globals, err}
- } else {
- globals, err := starlark.ExecFile(childThread, filepath.Join(topDir, modulePath), nil, builtins)
- e = &modentry{globals, err}
- }
-
- // Update the cache.
- moduleCacheLock.Lock()
- moduleCache[modulePath] = e
- }
- moduleCacheLock.Unlock()
- return e.globals, e.err
-}
-
-// Run runs the given starlark file and returns its global variables and a list of all starlark
-// files that were loaded. The top dir for starlark's // is found via getTopDir().
-func runStarlarkFile(filename string) (starlark.StringDict, []string, error) {
- topDir, err := getTopDir()
- if err != nil {
- return nil, nil, err
- }
- return runStarlarkFileWithFilesystem(filename, topDir, nil)
-}
-
-func runStarlarkFileWithFilesystem(filename string, topDir string, filesystem map[string]string) (starlark.StringDict, []string, error) {
- if !strings.HasPrefix(filename, "//") && !strings.HasPrefix(filename, ":") {
- filename = "//" + filename
- }
- filename, err := cleanModuleName(filename, "")
- if err != nil {
- return nil, nil, err
- }
- moduleCache := make(map[string]*modentry)
- moduleCache[filename] = nil
- moduleCacheLock := &sync.Mutex{}
- mainThread := &starlark.Thread{
- Name: "main",
- Print: func(_ *starlark.Thread, msg string) {
- // Ignore prints
- },
- Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
- return loader(thread, module, topDir, moduleCache, moduleCacheLock, filesystem)
- },
- }
- mainThread.SetLocal(callerDirKey, filepath.Dir(filename))
-
- var result starlark.StringDict
- if filesystem != nil {
- result, err = starlark.ExecFile(mainThread, filepath.Join(topDir, filename), filesystem[filename], builtins)
- } else {
- result, err = starlark.ExecFile(mainThread, filepath.Join(topDir, filename), nil, builtins)
- }
- return result, sortedStringKeys(moduleCache), err
-}
-
-func sortedStringKeys(m map[string]*modentry) []string {
- s := make([]string, 0, len(m))
- for k := range m {
- s = append(s, k)
- }
- sort.Strings(s)
- return s
-}
diff --git a/starlark_import/starlark_import_test.go b/starlark_import/starlark_import_test.go
deleted file mode 100644
index 8a58e3b..0000000
--- a/starlark_import/starlark_import_test.go
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2023 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package starlark_import
-
-import (
- "strings"
- "testing"
-
- "go.starlark.net/starlark"
-)
-
-func TestBasic(t *testing.T) {
- globals, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
- "a.bzl": `
-my_string = "hello, world!"
-`})
- if err != nil {
- t.Error(err)
- return
- }
-
- if globals["my_string"].(starlark.String) != "hello, world!" {
- t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String())
- }
-}
-
-func TestLoad(t *testing.T) {
- globals, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
- "a.bzl": `
-load("//b.bzl", _b_string = "my_string")
-my_string = "hello, " + _b_string
-`,
- "b.bzl": `
-my_string = "world!"
-`})
- if err != nil {
- t.Error(err)
- return
- }
-
- if globals["my_string"].(starlark.String) != "hello, world!" {
- t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String())
- }
-}
-
-func TestLoadRelative(t *testing.T) {
- globals, ninjaDeps, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
- "a.bzl": `
-load(":b.bzl", _b_string = "my_string")
-load("//foo/c.bzl", _c_string = "my_string")
-my_string = "hello, " + _b_string
-c_string = _c_string
-`,
- "b.bzl": `
-my_string = "world!"
-`,
- "foo/c.bzl": `
-load(":d.bzl", _d_string = "my_string")
-my_string = "hello, " + _d_string
-`,
- "foo/d.bzl": `
-my_string = "world!"
-`})
- if err != nil {
- t.Error(err)
- return
- }
-
- if globals["my_string"].(starlark.String) != "hello, world!" {
- t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String())
- }
-
- expectedNinjaDeps := []string{
- "a.bzl",
- "b.bzl",
- "foo/c.bzl",
- "foo/d.bzl",
- }
- if !slicesEqual(ninjaDeps, expectedNinjaDeps) {
- t.Errorf("Expected %v ninja deps, got %v", expectedNinjaDeps, ninjaDeps)
- }
-}
-
-func TestLoadCycle(t *testing.T) {
- _, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
- "a.bzl": `
-load(":b.bzl", _b_string = "my_string")
-my_string = "hello, " + _b_string
-`,
- "b.bzl": `
-load(":a.bzl", _a_string = "my_string")
-my_string = "hello, " + _a_string
-`})
- if err == nil || !strings.Contains(err.Error(), "cycle in load graph") {
- t.Errorf("Expected cycle in load graph, got: %v", err)
- return
- }
-}
-
-func slicesEqual[T comparable](a []T, b []T) bool {
- if len(a) != len(b) {
- return false
- }
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
- return true
-}
diff --git a/starlark_import/unmarshal.go b/starlark_import/unmarshal.go
deleted file mode 100644
index b243471..0000000
--- a/starlark_import/unmarshal.go
+++ /dev/null
@@ -1,304 +0,0 @@
-// Copyright 2023 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package starlark_import
-
-import (
- "fmt"
- "math"
- "reflect"
- "unsafe"
-
- "go.starlark.net/starlark"
- "go.starlark.net/starlarkstruct"
-)
-
-func Unmarshal[T any](value starlark.Value) (T, error) {
- x, err := UnmarshalReflect(value, reflect.TypeOf((*T)(nil)).Elem())
- return x.Interface().(T), err
-}
-
-func UnmarshalReflect(value starlark.Value, ty reflect.Type) (reflect.Value, error) {
- if ty == reflect.TypeOf((*starlark.Value)(nil)).Elem() {
- return reflect.ValueOf(value), nil
- }
- zero := reflect.Zero(ty)
- if value == nil {
- panic("nil value")
- }
- var result reflect.Value
- if ty.Kind() == reflect.Interface {
- var err error
- ty, err = typeOfStarlarkValue(value)
- if err != nil {
- return zero, err
- }
- }
- if ty.Kind() == reflect.Map {
- result = reflect.MakeMap(ty)
- } else {
- result = reflect.Indirect(reflect.New(ty))
- }
-
- switch v := value.(type) {
- case starlark.String:
- if result.Type().Kind() != reflect.String {
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- result.SetString(v.GoString())
- case starlark.Int:
- signedValue, signedOk := v.Int64()
- unsignedValue, unsignedOk := v.Uint64()
- switch result.Type().Kind() {
- case reflect.Int64:
- if !signedOk {
- return zero, fmt.Errorf("starlark int didn't fit in go int64")
- }
- result.SetInt(signedValue)
- case reflect.Int32:
- if !signedOk || signedValue > math.MaxInt32 || signedValue < math.MinInt32 {
- return zero, fmt.Errorf("starlark int didn't fit in go int32")
- }
- result.SetInt(signedValue)
- case reflect.Int16:
- if !signedOk || signedValue > math.MaxInt16 || signedValue < math.MinInt16 {
- return zero, fmt.Errorf("starlark int didn't fit in go int16")
- }
- result.SetInt(signedValue)
- case reflect.Int8:
- if !signedOk || signedValue > math.MaxInt8 || signedValue < math.MinInt8 {
- return zero, fmt.Errorf("starlark int didn't fit in go int8")
- }
- result.SetInt(signedValue)
- case reflect.Int:
- if !signedOk || signedValue > math.MaxInt || signedValue < math.MinInt {
- return zero, fmt.Errorf("starlark int didn't fit in go int")
- }
- result.SetInt(signedValue)
- case reflect.Uint64:
- if !unsignedOk {
- return zero, fmt.Errorf("starlark int didn't fit in go uint64")
- }
- result.SetUint(unsignedValue)
- case reflect.Uint32:
- if !unsignedOk || unsignedValue > math.MaxUint32 {
- return zero, fmt.Errorf("starlark int didn't fit in go uint32")
- }
- result.SetUint(unsignedValue)
- case reflect.Uint16:
- if !unsignedOk || unsignedValue > math.MaxUint16 {
- return zero, fmt.Errorf("starlark int didn't fit in go uint16")
- }
- result.SetUint(unsignedValue)
- case reflect.Uint8:
- if !unsignedOk || unsignedValue > math.MaxUint8 {
- return zero, fmt.Errorf("starlark int didn't fit in go uint8")
- }
- result.SetUint(unsignedValue)
- case reflect.Uint:
- if !unsignedOk || unsignedValue > math.MaxUint {
- return zero, fmt.Errorf("starlark int didn't fit in go uint")
- }
- result.SetUint(unsignedValue)
- default:
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- case starlark.Float:
- f := float64(v)
- switch result.Type().Kind() {
- case reflect.Float64:
- result.SetFloat(f)
- case reflect.Float32:
- if f > math.MaxFloat32 || f < -math.MaxFloat32 {
- return zero, fmt.Errorf("starlark float didn't fit in go float32")
- }
- result.SetFloat(f)
- default:
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- case starlark.Bool:
- if result.Type().Kind() != reflect.Bool {
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- result.SetBool(bool(v))
- case starlark.Tuple:
- if result.Type().Kind() != reflect.Slice {
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- elemType := result.Type().Elem()
- // TODO: Add this grow call when we're on go 1.20
- //result.Grow(v.Len())
- for i := 0; i < v.Len(); i++ {
- elem, err := UnmarshalReflect(v.Index(i), elemType)
- if err != nil {
- return zero, err
- }
- result = reflect.Append(result, elem)
- }
- case *starlark.List:
- if result.Type().Kind() != reflect.Slice {
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- elemType := result.Type().Elem()
- // TODO: Add this grow call when we're on go 1.20
- //result.Grow(v.Len())
- for i := 0; i < v.Len(); i++ {
- elem, err := UnmarshalReflect(v.Index(i), elemType)
- if err != nil {
- return zero, err
- }
- result = reflect.Append(result, elem)
- }
- case *starlark.Dict:
- if result.Type().Kind() != reflect.Map {
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- keyType := result.Type().Key()
- valueType := result.Type().Elem()
- for _, pair := range v.Items() {
- key := pair.Index(0)
- value := pair.Index(1)
-
- unmarshalledKey, err := UnmarshalReflect(key, keyType)
- if err != nil {
- return zero, err
- }
- unmarshalledValue, err := UnmarshalReflect(value, valueType)
- if err != nil {
- return zero, err
- }
-
- result.SetMapIndex(unmarshalledKey, unmarshalledValue)
- }
- case *starlarkstruct.Struct:
- if result.Type().Kind() != reflect.Struct {
- return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
- }
- if result.NumField() != len(v.AttrNames()) {
- return zero, fmt.Errorf("starlark struct and go struct have different number of fields (%d and %d)", len(v.AttrNames()), result.NumField())
- }
- for _, attrName := range v.AttrNames() {
- attr, err := v.Attr(attrName)
- if err != nil {
- return zero, err
- }
-
- // TODO(b/279787235): this should probably support tags to rename the field
- resultField := result.FieldByName(attrName)
- if resultField == (reflect.Value{}) {
- return zero, fmt.Errorf("starlark struct had field %s, but requested struct type did not", attrName)
- }
- // This hack allows us to change unexported fields
- resultField = reflect.NewAt(resultField.Type(), unsafe.Pointer(resultField.UnsafeAddr())).Elem()
- x, err := UnmarshalReflect(attr, resultField.Type())
- if err != nil {
- return zero, err
- }
- resultField.Set(x)
- }
- default:
- return zero, fmt.Errorf("unimplemented starlark type: %s", value.Type())
- }
-
- return result, nil
-}
-
-func typeOfStarlarkValue(value starlark.Value) (reflect.Type, error) {
- var err error
- switch v := value.(type) {
- case starlark.String:
- return reflect.TypeOf(""), nil
- case *starlark.List:
- innerType := reflect.TypeOf("")
- if v.Len() > 0 {
- innerType, err = typeOfStarlarkValue(v.Index(0))
- if err != nil {
- return nil, err
- }
- }
- for i := 1; i < v.Len(); i++ {
- innerTypeI, err := typeOfStarlarkValue(v.Index(i))
- if err != nil {
- return nil, err
- }
- if innerType != innerTypeI {
- return nil, fmt.Errorf("List must contain elements of entirely the same type, found %v and %v", innerType, innerTypeI)
- }
- }
- return reflect.SliceOf(innerType), nil
- case *starlark.Dict:
- keyType := reflect.TypeOf("")
- valueType := reflect.TypeOf("")
- keys := v.Keys()
- if v.Len() > 0 {
- firstKey := keys[0]
- keyType, err = typeOfStarlarkValue(firstKey)
- if err != nil {
- return nil, err
- }
- firstValue, found, err := v.Get(firstKey)
- if !found {
- err = fmt.Errorf("value not found")
- }
- if err != nil {
- return nil, err
- }
- valueType, err = typeOfStarlarkValue(firstValue)
- if err != nil {
- return nil, err
- }
- }
- for _, key := range keys {
- keyTypeI, err := typeOfStarlarkValue(key)
- if err != nil {
- return nil, err
- }
- if keyType != keyTypeI {
- return nil, fmt.Errorf("dict must contain elements of entirely the same type, found %v and %v", keyType, keyTypeI)
- }
- value, found, err := v.Get(key)
- if !found {
- err = fmt.Errorf("value not found")
- }
- if err != nil {
- return nil, err
- }
- valueTypeI, err := typeOfStarlarkValue(value)
- if valueType.Kind() != reflect.Interface && valueTypeI != valueType {
- // If we see conflicting value types, change the result value type to an empty interface
- valueType = reflect.TypeOf([]interface{}{}).Elem()
- }
- }
- return reflect.MapOf(keyType, valueType), nil
- case starlark.Int:
- return reflect.TypeOf(0), nil
- case starlark.Float:
- return reflect.TypeOf(0.0), nil
- case starlark.Bool:
- return reflect.TypeOf(true), nil
- default:
- return nil, fmt.Errorf("unimplemented starlark type: %s", value.Type())
- }
-}
-
-// UnmarshalNoneable is like Unmarshal, but it will accept None as the top level (but not nested)
-// starlark value. If the value is None, a nil pointer will be returned, otherwise a pointer
-// to the result of Unmarshal will be returned.
-func UnmarshalNoneable[T any](value starlark.Value) (*T, error) {
- if _, ok := value.(starlark.NoneType); ok {
- return nil, nil
- }
- ret, err := Unmarshal[T](value)
- return &ret, err
-}
diff --git a/starlark_import/unmarshal_test.go b/starlark_import/unmarshal_test.go
deleted file mode 100644
index bc0ea4c..0000000
--- a/starlark_import/unmarshal_test.go
+++ /dev/null
@@ -1,148 +0,0 @@
-// Copyright 2023 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package starlark_import
-
-import (
- "reflect"
- "testing"
-
- "go.starlark.net/starlark"
-)
-
-func createStarlarkValue(t *testing.T, code string) starlark.Value {
- t.Helper()
- result, err := starlark.ExecFile(&starlark.Thread{}, "main.bzl", "x = "+code, builtins)
- if err != nil {
- panic(err)
- }
- return result["x"]
-}
-
-func TestUnmarshalConcreteType(t *testing.T) {
- x, err := Unmarshal[string](createStarlarkValue(t, `"foo"`))
- if err != nil {
- t.Error(err)
- return
- }
- if x != "foo" {
- t.Errorf(`Expected "foo", got %q`, x)
- }
-}
-
-func TestUnmarshalConcreteTypeWithInterfaces(t *testing.T) {
- x, err := Unmarshal[map[string]map[string]interface{}](createStarlarkValue(t,
- `{"foo": {"foo2": "foo3"}, "bar": {"bar2": ["bar3"]}}`))
- if err != nil {
- t.Error(err)
- return
- }
- expected := map[string]map[string]interface{}{
- "foo": {"foo2": "foo3"},
- "bar": {"bar2": []string{"bar3"}},
- }
- if !reflect.DeepEqual(x, expected) {
- t.Errorf(`Expected %v, got %v`, expected, x)
- }
-}
-
-func TestUnmarshalToStarlarkValue(t *testing.T) {
- x, err := Unmarshal[map[string]starlark.Value](createStarlarkValue(t,
- `{"foo": "Hi", "bar": None}`))
- if err != nil {
- t.Error(err)
- return
- }
- if x["foo"].(starlark.String).GoString() != "Hi" {
- t.Errorf("Expected \"Hi\", got: %q", x["foo"].(starlark.String).GoString())
- }
- if x["bar"].Type() != "NoneType" {
- t.Errorf("Expected \"NoneType\", got: %q", x["bar"].Type())
- }
-}
-
-func TestUnmarshal(t *testing.T) {
- testCases := []struct {
- input string
- expected interface{}
- }{
- {
- input: `"foo"`,
- expected: "foo",
- },
- {
- input: `5`,
- expected: 5,
- },
- {
- input: `["foo", "bar"]`,
- expected: []string{"foo", "bar"},
- },
- {
- input: `("foo", "bar")`,
- expected: []string{"foo", "bar"},
- },
- {
- input: `("foo",5)`,
- expected: []interface{}{"foo", 5},
- },
- {
- input: `{"foo": 5, "bar": 10}`,
- expected: map[string]int{"foo": 5, "bar": 10},
- },
- {
- input: `{"foo": ["qux"], "bar": []}`,
- expected: map[string][]string{"foo": {"qux"}, "bar": nil},
- },
- {
- input: `struct(Foo="foo", Bar=5)`,
- expected: struct {
- Foo string
- Bar int
- }{Foo: "foo", Bar: 5},
- },
- {
- // Unexported fields version of the above
- input: `struct(foo="foo", bar=5)`,
- expected: struct {
- foo string
- bar int
- }{foo: "foo", bar: 5},
- },
- {
- input: `{"foo": "foo2", "bar": ["bar2"], "baz": 5, "qux": {"qux2": "qux3"}, "quux": {"quux2": "quux3", "quux4": 5}}`,
- expected: map[string]interface{}{
- "foo": "foo2",
- "bar": []string{"bar2"},
- "baz": 5,
- "qux": map[string]string{"qux2": "qux3"},
- "quux": map[string]interface{}{
- "quux2": "quux3",
- "quux4": 5,
- },
- },
- },
- }
-
- for _, tc := range testCases {
- x, err := UnmarshalReflect(createStarlarkValue(t, tc.input), reflect.TypeOf(tc.expected))
- if err != nil {
- t.Error(err)
- continue
- }
- if !reflect.DeepEqual(x.Interface(), tc.expected) {
- t.Errorf(`Expected %#v, got %#v`, tc.expected, x.Interface())
- }
- }
-}