Add directory support to nsjail genrule
Trusty build requires a lot of files. Rather than globbing all such
files (more than 10,000 files), allowing bind-mounting directories makes
nsjail much more efficient. With this change, directories can be
directly bind-mounted with dirgroup modules and dir_srcs property of
genrule module.
dirgroup module and dir_srcs property will be allowed only to Trusty
builds.
Bug: 358302178
Test: m lk.elf.x86_64 lk.elf.arm64
Change-Id: I5938a57b7fb65f8fecce4a8f40aa4aedbf991135
diff --git a/android/Android.bp b/android/Android.bp
index 44cddcc..3b54326 100644
--- a/android/Android.bp
+++ b/android/Android.bp
@@ -52,6 +52,7 @@
"defs.go",
"depset_generic.go",
"deptag.go",
+ "dirgroup.go",
"early_module_context.go",
"expand.go",
"filegroup.go",
diff --git a/android/dirgroup.go b/android/dirgroup.go
new file mode 100644
index 0000000..20c4d13
--- /dev/null
+++ b/android/dirgroup.go
@@ -0,0 +1,63 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
+)
+
+func init() {
+ RegisterDirgroupBuildComponents(InitRegistrationContext)
+}
+
+func RegisterDirgroupBuildComponents(ctx RegistrationContext) {
+ ctx.RegisterModuleType("dirgroup", DirGroupFactory)
+}
+
+type dirGroupProperties struct {
+ // dirs lists directories that will be included in this dirgroup
+ Dirs proptools.Configurable[[]string] `android:"path"`
+}
+
+type dirGroup struct {
+ ModuleBase
+ DefaultableModuleBase
+ properties dirGroupProperties
+}
+
+type DirInfo struct {
+ // TODO(b/358302178): Use DirectoryPaths instead of Paths
+ Dirs Paths
+}
+
+var DirProvider = blueprint.NewProvider[DirInfo]()
+
+// dirgroup contains a list of dirs that are referenced by other modules
+// properties using the syntax ":<name>". dirgroup are also be used to export
+// dirs across package boundaries. Currently the only allowed usage is genrule's
+// dir_srcs property.
+func DirGroupFactory() Module {
+ module := &dirGroup{}
+ module.AddProperties(&module.properties)
+ InitAndroidModule(module)
+ InitDefaultableModule(module)
+ return module
+}
+
+func (fg *dirGroup) GenerateAndroidBuildActions(ctx ModuleContext) {
+ dirs := DirectoryPathsForModuleSrc(ctx, fg.properties.Dirs.GetOrDefault(ctx, nil))
+ SetProvider(ctx, DirProvider, DirInfo{Dirs: dirs})
+}
diff --git a/android/neverallow.go b/android/neverallow.go
index e93763b..041c9a0 100644
--- a/android/neverallow.go
+++ b/android/neverallow.go
@@ -60,6 +60,7 @@
AddNeverAllowRules(createCcStubsRule())
AddNeverAllowRules(createProhibitHeaderOnlyRule())
AddNeverAllowRules(createLimitNdkExportRule()...)
+ AddNeverAllowRules(createLimitDirgroupRule()...)
}
// Add a NeverAllow rule to the set of rules to apply.
@@ -275,6 +276,23 @@
}
}
+func createLimitDirgroupRule() []Rule {
+ reason := "dirgroup module and dir_srcs property of genrule is allowed only to Trusty build rule."
+ return []Rule{
+ NeverAllow().
+ ModuleType("dirgroup").
+ WithMatcher("visibility", NotInList([]string{"//trusty/vendor/google/aosp/scripts"})).Because(reason),
+ NeverAllow().
+ ModuleType("dirgroup").
+ Without("visibility", "//trusty/vendor/google/aosp/scripts").Because(reason),
+ NeverAllow().
+ ModuleType("genrule").
+ Without("name", "lk.elf.arm64").
+ Without("name", "lk.elf.x86_64").
+ WithMatcher("dir_srcs", isSetMatcherInstance).Because(reason),
+ }
+}
+
func neverallowMutator(ctx BottomUpMutatorContext) {
m, ok := ctx.Module().(Module)
if !ok {
diff --git a/android/paths.go b/android/paths.go
index ec05831..371aed8 100644
--- a/android/paths.go
+++ b/android/paths.go
@@ -550,6 +550,58 @@
return ret
}
+// DirectoryPathsForModuleSrcExcludes returns a Paths{} containing the resolved references in
+// directory paths. Elements of paths are resolved as:
+// - filepath, relative to local module directory, resolves as a filepath relative to the local
+// source directory
+// - other modules using the ":name" syntax. These modules must implement DirProvider.
+//
+// TODO(b/358302178): Implement DirectoryPath and change the return type.
+func DirectoryPathsForModuleSrc(ctx ModuleMissingDepsPathContext, paths []string) Paths {
+ var ret Paths
+
+ for _, path := range paths {
+ if m, t := SrcIsModuleWithTag(path); m != "" {
+ module := GetModuleFromPathDep(ctx, m, t)
+ if module == nil {
+ ctx.ModuleErrorf(`missing dependency on %q, is the property annotated with android:"path"?`, m)
+ continue
+ }
+ if t != "" {
+ ctx.ModuleErrorf("DirProvider dependency %q does not support the tag %q", module, t)
+ continue
+ }
+ mctx, ok := ctx.(OtherModuleProviderContext)
+ if !ok {
+ panic(fmt.Errorf("%s is not an OtherModuleProviderContext", ctx))
+ }
+ if dirProvider, ok := OtherModuleProvider(mctx, module, DirProvider); ok {
+ ret = append(ret, dirProvider.Dirs...)
+ } else {
+ ReportPathErrorf(ctx, "module %q does not implement DirProvider", module)
+ }
+ } else {
+ p := pathForModuleSrc(ctx, path)
+ if isDir, err := ctx.Config().fs.IsDir(p.String()); err != nil {
+ ReportPathErrorf(ctx, "%s: %s", p, err.Error())
+ } else if !isDir {
+ ReportPathErrorf(ctx, "module directory path %q is not a directory", p)
+ } else {
+ ret = append(ret, p)
+ }
+ }
+ }
+
+ seen := make(map[Path]bool, len(ret))
+ for _, path := range ret {
+ if seen[path] {
+ ReportPathErrorf(ctx, "duplicated path %q", path)
+ }
+ seen[path] = true
+ }
+ return ret
+}
+
// OutputPaths is a slice of OutputPath objects, with helpers to operate on the collection.
type OutputPaths []OutputPath
diff --git a/genrule/genrule.go b/genrule/genrule.go
index 349615f..f2a761c 100644
--- a/genrule/genrule.go
+++ b/genrule/genrule.go
@@ -219,6 +219,7 @@
// For nsjail tasks
useNsjail bool
+ dirSrcs android.Paths
}
func (g *Module) GeneratedSourceFiles() android.Paths {
@@ -579,10 +580,12 @@
}
if task.useNsjail {
- for _, input := range task.in {
- // can fail if input is a file.
+ for _, input := range task.dirSrcs {
+ cmd.Implicit(input)
if paths, err := ctx.GlobWithDeps(filepath.Join(input.String(), "**/*"), nil); err == nil {
rule.NsjailImplicits(android.PathsForSource(ctx, paths))
+ } else {
+ ctx.PropertyErrorf("dir_srcs", "can't glob %q", input.String())
}
}
}
@@ -858,6 +861,12 @@
taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
useNsjail := Bool(properties.Use_nsjail)
+ dirSrcs := android.DirectoryPathsForModuleSrc(ctx, properties.Dir_srcs)
+ if len(dirSrcs) > 0 && !useNsjail {
+ ctx.PropertyErrorf("dir_srcs", "can't use dir_srcs if use_nsjail is false")
+ return nil
+ }
+
outs := make(android.WritablePaths, len(properties.Out))
for i, out := range properties.Out {
outs[i] = android.PathForModuleGen(ctx, out)
@@ -868,6 +877,7 @@
genDir: android.PathForModuleGen(ctx),
cmd: rawCommand,
useNsjail: useNsjail,
+ dirSrcs: dirSrcs,
}}
}
@@ -884,6 +894,10 @@
type genRuleProperties struct {
Use_nsjail *bool
+ // List of input directories. Can be set only when use_nsjail is true. Currently, usage of
+ // dir_srcs is limited only to Trusty build.
+ Dir_srcs []string `android:"path"`
+
// names of the output files that will be generated
Out []string `android:"arch_variant"`
}