Merge "Add myself as an owner for NDK things."
diff --git a/Android.bp b/Android.bp
index 82be0fa..ebf1038 100644
--- a/Android.bp
+++ b/Android.bp
@@ -2,6 +2,10 @@
"androidmk",
"bpfix",
"cmd/*",
+ "fs",
+ "finder",
+ "jar",
+ "zip",
"third_party/zip",
"ui/*",
]
@@ -46,22 +50,29 @@
"android/makevars.go",
"android/module.go",
"android/mutator.go",
+ "android/namespace.go",
"android/onceper.go",
"android/package_ctx.go",
"android/paths.go",
"android/prebuilt.go",
+ "android/proto.go",
"android/register.go",
+ "android/singleton.go",
"android/testing.go",
"android/util.go",
"android/variable.go",
+ "android/writedocs.go",
// Lock down environment access last
"android/env.go",
],
testSrcs: [
+ "android/config_test.go",
"android/expand_test.go",
+ "android/namespace_test.go",
"android/paths_test.go",
"android/prebuilt_test.go",
+ "android/util_test.go",
"android/variable_test.go",
],
}
@@ -113,7 +124,9 @@
"cc/check.go",
"cc/coverage.go",
"cc/gen.go",
+ "cc/lto.go",
"cc/makevars.go",
+ "cc/pgo.go",
"cc/prebuilt.go",
"cc/proto.go",
"cc/relocation_packer.go",
@@ -125,6 +138,7 @@
"cc/tidy.go",
"cc/util.go",
"cc/vndk.go",
+ "cc/vndk_prebuilt.go",
"cc/cmakelists.go",
"cc/compiler.go",
@@ -145,9 +159,13 @@
"cc/llndk_library.go",
"cc/kernel_headers.go",
+
+ "cc/genrule.go",
],
testSrcs: [
"cc/cc_test.go",
+ "cc/gen_test.go",
+ "cc/library_test.go",
"cc/test_data_test.go",
],
pluginFor: ["soong_build"],
@@ -195,15 +213,21 @@
"soong-java-config",
],
srcs: [
+ "java/aapt2.go",
"java/androidmk.go",
"java/app_builder.go",
"java/app.go",
"java/builder.go",
"java/gen.go",
+ "java/genrule.go",
+ "java/jacoco.go",
"java/java.go",
+ "java/proto.go",
"java/resources.go",
+ "java/system_modules.go",
],
testSrcs: [
+ "java/app_test.go",
"java/java_test.go",
],
pluginFor: ["soong_build"],
@@ -218,6 +242,8 @@
],
srcs: [
"java/config/config.go",
+ "java/config/error_prone.go",
+ "java/config/kotlin.go",
"java/config/makevars.go",
],
}
@@ -233,6 +259,7 @@
"python/androidmk.go",
"python/binary.go",
"python/builder.go",
+ "python/defaults.go",
"python/installer.go",
"python/library.go",
"python/python.go",
@@ -301,7 +328,7 @@
enabled: false,
target: {
windows: {
- enabled: true
+ enabled: true,
},
},
}
@@ -320,3 +347,45 @@
name: "device_kernel_headers",
vendor: true,
}
+
+cc_genrule {
+ name: "host_bionic_linker_asm",
+ host_supported: true,
+ device_supported: false,
+ target: {
+ linux_bionic: {
+ enabled: true,
+ },
+ linux_glibc: {
+ enabled: false,
+ },
+ darwin: {
+ enabled: false,
+ },
+ },
+ tools: ["extract_linker"],
+ cmd: "$(location) -s $(out) $(in)",
+ srcs: [":linker"],
+ out: ["linker.s"],
+}
+
+cc_genrule {
+ name: "host_bionic_linker_script",
+ host_supported: true,
+ device_supported: false,
+ target: {
+ linux_bionic: {
+ enabled: true,
+ },
+ linux_glibc: {
+ enabled: false,
+ },
+ darwin: {
+ enabled: false,
+ },
+ },
+ tools: ["extract_linker"],
+ cmd: "$(location) -T $(out) $(in)",
+ srcs: [":linker"],
+ out: ["linker.script"],
+}
diff --git a/README.md b/README.md
index bc019ae..4013a2a 100644
--- a/README.md
+++ b/README.md
@@ -61,6 +61,7 @@
first assignment, and properties statically by the module type. The supported
types are:
* Bool (`true` or `false`)
+* Integers (`int`)
* Strings (`"string"`)
* Lists of strings (`["string1", "string2"]`)
* Maps (`{key1: "value1", key2: ["value2"]}`)
@@ -71,8 +72,9 @@
### Operators
Strings, lists of strings, and maps can be appended using the `+` operator.
-Appending a map produces the union of keys in both maps, appending the values
-of any keys that are present in both maps.
+Integers can be summed up using the `+` operator. Appending a map produces the
+union of keys in both maps, appending the values of any keys that are present
+in both maps.
### Defaults modules
@@ -93,6 +95,38 @@
}
```
+### Name resolution
+
+Soong provides the ability for modules in different directories to specify
+the same name, as long as each module is declared within a separate namespace.
+A namespace can be declared like this:
+
+```
+soong_namespace {
+ imports: ["path/to/otherNamespace1", "path/to/otherNamespace2"],
+}
+```
+
+Each Soong module is assigned a namespace based on its location in the tree.
+Each Soong module is considered to be in the namespace defined by the
+soong_namespace found in an Android.bp in the current directory or closest
+ancestor directory, unless no such soong_namespace module is found, in which
+case the module is considered to be in the implicit root namespace.
+
+When Soong attempts to resolve dependency D declared my module M in namespace
+N which imports namespaces I1, I2, I3..., then if D is a fully-qualified name
+of the form "//namespace:module", only the specified namespace will be searched
+for the specified module name. Otherwise, Soong will first look for a module
+named D declared in namespace N. If that module does not exist, Soong will look
+for a module named D in namespaces I1, I2, I3... Lastly, Soong will look in the
+root namespace.
+
+Until we have fully converted from Make to Soong, it will be necessary for the
+Make product config to specify a value of PRODUCT_SOONG_NAMESPACES. Its value
+should be a space-separated list of namespaces that Soong export to Make to be
+built by the `m` command. After we have fully converted from Make to Soong, the
+details of enabling namespaces could potentially change.
+
### Formatter
Soong includes a canonical formatter for blueprint files, similar to
diff --git a/android/androidmk.go b/android/androidmk.go
index 6197f59..fb3934f 100644
--- a/android/androidmk.go
+++ b/android/androidmk.go
@@ -25,7 +25,6 @@
"strings"
"github.com/google/blueprint"
- "github.com/google/blueprint/proptools"
)
func init() {
@@ -33,7 +32,7 @@
}
type AndroidMkDataProvider interface {
- AndroidMk() (AndroidMkData, error)
+ AndroidMk() AndroidMkData
BaseModuleName() string
}
@@ -42,36 +41,38 @@
SubName string
OutputFile OptionalPath
Disabled bool
+ Include string
+ Required []string
- Custom func(w io.Writer, name, prefix, moduleDir string) error
+ Custom func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData)
- Extra []func(w io.Writer, outputFile Path) error
+ Extra []AndroidMkExtraFunc
+
+ preamble bytes.Buffer
}
-func AndroidMkSingleton() blueprint.Singleton {
+type AndroidMkExtraFunc func(w io.Writer, outputFile Path)
+
+func AndroidMkSingleton() Singleton {
return &androidMkSingleton{}
}
type androidMkSingleton struct{}
-func (c *androidMkSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
- config := ctx.Config().(Config)
-
- if !config.EmbeddedInMake() {
+func (c *androidMkSingleton) GenerateBuildActions(ctx SingletonContext) {
+ if !ctx.Config().EmbeddedInMake() {
return
}
var androidMkModulesList []Module
- ctx.VisitAllModules(func(module blueprint.Module) {
- if amod, ok := module.(Module); ok {
- androidMkModulesList = append(androidMkModulesList, amod)
- }
+ ctx.VisitAllModules(func(module Module) {
+ androidMkModulesList = append(androidMkModulesList, module)
})
sort.Sort(AndroidModulesByName{androidMkModulesList, ctx})
- transMk := PathForOutput(ctx, "Android"+proptools.String(config.ProductVariables.Make_suffix)+".mk")
+ transMk := PathForOutput(ctx, "Android"+String(ctx.Config().ProductVariables.Make_suffix)+".mk")
if ctx.Failed() {
return
}
@@ -81,14 +82,13 @@
ctx.Errorf(err.Error())
}
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: blueprint.Phony,
- Outputs: []string{transMk.String()},
- Optional: true,
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Output: transMk,
})
}
-func translateAndroidMk(ctx blueprint.SingletonContext, mkFile string, mods []Module) error {
+func translateAndroidMk(ctx SingletonContext, mkFile string, mods []Module) error {
buf := &bytes.Buffer{}
fmt.Fprintln(buf, "LOCAL_MODULE_MAKEFILE := $(lastword $(MAKEFILE_LIST))")
@@ -140,7 +140,7 @@
return ioutil.WriteFile(mkFile, buf.Bytes(), 0666)
}
-func translateAndroidMkModule(ctx blueprint.SingletonContext, w io.Writer, mod blueprint.Module) error {
+func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.Module) error {
provider, ok := mod.(AndroidMkDataProvider)
if !ok {
return nil
@@ -157,58 +157,50 @@
return nil
}
- data, err := provider.AndroidMk()
- if err != nil {
- return err
+ if !amod.commonProperties.NamespaceExportedToMake {
+ // TODO(jeffrygaston) do we want to validate that there are no modules being
+ // exported to Kati that depend on this module?
+ return nil
}
+ data := provider.AndroidMk()
+
+ if data.Include == "" {
+ data.Include = "$(BUILD_PREBUILT)"
+ }
+
+ data.Required = amod.commonProperties.Required
+
// Make does not understand LinuxBionic
if amod.Os() == LinuxBionic {
return nil
}
- if data.SubName != "" {
- name += data.SubName
- }
+ prefix := ""
+ if amod.ArchSpecific() {
+ switch amod.Os().Class {
+ case Host:
+ prefix = "HOST_"
+ case HostCross:
+ prefix = "HOST_CROSS_"
+ case Device:
+ prefix = "TARGET_"
- if data.Custom != nil {
- prefix := ""
- if amod.ArchSpecific() {
- switch amod.Os().Class {
- case Host:
- prefix = "HOST_"
- case HostCross:
- prefix = "HOST_CROSS_"
- case Device:
- prefix = "TARGET_"
-
- }
-
- config := ctx.Config().(Config)
- if amod.Arch().ArchType != config.Targets[amod.Os().Class][0].Arch.ArchType {
- prefix = "2ND_" + prefix
- }
}
- return data.Custom(w, name, prefix, filepath.Dir(ctx.BlueprintFile(mod)))
+ if amod.Arch().ArchType != ctx.Config().Targets[amod.Os().Class][0].Arch.ArchType {
+ prefix = "2ND_" + prefix
+ }
}
- if data.Disabled {
- return nil
- }
+ fmt.Fprintln(&data.preamble, "\ninclude $(CLEAR_VARS)")
+ fmt.Fprintln(&data.preamble, "LOCAL_PATH :=", filepath.Dir(ctx.BlueprintFile(mod)))
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE :=", name+data.SubName)
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE_CLASS :=", data.Class)
+ fmt.Fprintln(&data.preamble, "LOCAL_PREBUILT_MODULE_FILE :=", data.OutputFile.String())
- if !data.OutputFile.Valid() {
- return err
- }
-
- fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)")
- fmt.Fprintln(w, "LOCAL_PATH :=", filepath.Dir(ctx.BlueprintFile(mod)))
- fmt.Fprintln(w, "LOCAL_MODULE :=", name)
- fmt.Fprintln(w, "LOCAL_MODULE_CLASS :=", data.Class)
- fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", data.OutputFile.String())
-
- if len(amod.commonProperties.Required) > 0 {
- fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(amod.commonProperties.Required, " "))
+ if len(data.Required) > 0 {
+ fmt.Fprintln(&data.preamble, "LOCAL_REQUIRED_MODULES := "+strings.Join(data.Required, " "))
}
archStr := amod.Arch().ArchType.String()
@@ -217,51 +209,72 @@
case Host:
// Make cannot identify LOCAL_MODULE_HOST_ARCH:= common.
if archStr != "common" {
- fmt.Fprintln(w, "LOCAL_MODULE_HOST_ARCH :=", archStr)
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_ARCH :=", archStr)
}
host = true
case HostCross:
// Make cannot identify LOCAL_MODULE_HOST_CROSS_ARCH:= common.
if archStr != "common" {
- fmt.Fprintln(w, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr)
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr)
}
host = true
case Device:
// Make cannot identify LOCAL_MODULE_TARGET_ARCH:= common.
if archStr != "common" {
- fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", archStr)
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE_TARGET_ARCH :=", archStr)
}
- if len(amod.commonProperties.Logtags) > 0 {
- fmt.Fprintln(w, "LOCAL_LOGTAGS_FILES := ", strings.Join(amod.commonProperties.Logtags, " "))
- }
if len(amod.commonProperties.Init_rc) > 0 {
- fmt.Fprintln(w, "LOCAL_INIT_RC := ", strings.Join(amod.commonProperties.Init_rc, " "))
+ fmt.Fprintln(&data.preamble, "LOCAL_INIT_RC := ", strings.Join(amod.commonProperties.Init_rc, " "))
}
- if amod.commonProperties.Proprietary {
- fmt.Fprintln(w, "LOCAL_PROPRIETARY_MODULE := true")
+ if Bool(amod.commonProperties.Proprietary) {
+ fmt.Fprintln(&data.preamble, "LOCAL_PROPRIETARY_MODULE := true")
}
- if amod.commonProperties.Vendor {
- fmt.Fprintln(w, "LOCAL_VENDOR_MODULE := true")
+ if Bool(amod.commonProperties.Vendor) {
+ fmt.Fprintln(&data.preamble, "LOCAL_VENDOR_MODULE := true")
}
if amod.commonProperties.Owner != nil {
- fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", *amod.commonProperties.Owner)
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE_OWNER :=", *amod.commonProperties.Owner)
+ }
+ if amod.commonProperties.Notice != nil {
+ fmt.Fprintln(&data.preamble, "LOCAL_NOTICE_FILE :=", "$(LOCAL_PATH)/"+*amod.commonProperties.Notice)
}
}
if host {
- fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", amod.Os().String())
- fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true")
+ makeOs := amod.Os().String()
+ if amod.Os() == Linux || amod.Os() == LinuxBionic {
+ makeOs = "linux"
+ }
+ fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_OS :=", makeOs)
+ fmt.Fprintln(&data.preamble, "LOCAL_IS_HOST_MODULE := true")
}
+ blueprintDir := filepath.Dir(ctx.BlueprintFile(mod))
+
+ if data.Custom != nil {
+ data.Custom(w, name, prefix, blueprintDir, data)
+ } else {
+ WriteAndroidMkData(w, data)
+ }
+
+ return nil
+}
+
+func WriteAndroidMkData(w io.Writer, data AndroidMkData) {
+ if data.Disabled {
+ return
+ }
+
+ if !data.OutputFile.Valid() {
+ return
+ }
+
+ w.Write(data.preamble.Bytes())
+
for _, extra := range data.Extra {
- err = extra(w, data.OutputFile.Path())
- if err != nil {
- return err
- }
+ extra(w, data.OutputFile.Path())
}
- fmt.Fprintln(w, "include $(BUILD_PREBUILT)")
-
- return err
+ fmt.Fprintln(w, "include "+data.Include)
}
diff --git a/android/api_levels.go b/android/api_levels.go
index 70b251b..a519117 100644
--- a/android/api_levels.go
+++ b/android/api_levels.go
@@ -16,22 +16,19 @@
import (
"encoding/json"
- "path/filepath"
-
- "github.com/google/blueprint"
)
func init() {
RegisterSingletonType("api_levels", ApiLevelsSingleton)
}
-func ApiLevelsSingleton() blueprint.Singleton {
+func ApiLevelsSingleton() Singleton {
return &apiLevelsSingleton{}
}
type apiLevelsSingleton struct{}
-func createApiLevelsJson(ctx blueprint.SingletonContext, file string,
+func createApiLevelsJson(ctx SingletonContext, file WritablePath,
apiLevelsMap map[string]int) {
jsonStr, err := json.Marshal(apiLevelsMap)
@@ -39,27 +36,40 @@
ctx.Errorf(err.Error())
}
- ctx.Build(pctx, blueprint.BuildParams{
+ ctx.Build(pctx, BuildParams{
Rule: WriteFile,
- Description: "generate " + filepath.Base(file),
- Outputs: []string{file},
+ Description: "generate " + file.Base(),
+ Output: file,
Args: map[string]string{
"content": string(jsonStr[:]),
},
})
}
-func GetApiLevelsJson(ctx PathContext) Path {
+func GetApiLevelsJson(ctx PathContext) WritablePath {
return PathForOutput(ctx, "api_levels.json")
}
-func (a *apiLevelsSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
+func (a *apiLevelsSingleton) GenerateBuildActions(ctx SingletonContext) {
baseApiLevel := 9000
- apiLevelsMap := map[string]int{}
- for i, codename := range ctx.Config().(Config).PlatformVersionAllCodenames() {
+ apiLevelsMap := map[string]int{
+ "G": 9,
+ "I": 14,
+ "J": 16,
+ "J-MR1": 17,
+ "J-MR2": 18,
+ "K": 19,
+ "L": 21,
+ "L-MR1": 22,
+ "M": 23,
+ "N": 24,
+ "N-MR1": 25,
+ "O": 26,
+ }
+ for i, codename := range ctx.Config().PlatformVersionCombinedCodenames() {
apiLevelsMap[codename] = baseApiLevel + i
}
apiLevelsJson := GetApiLevelsJson(ctx)
- createApiLevelsJson(ctx, apiLevelsJson.String(), apiLevelsMap)
+ createApiLevelsJson(ctx, apiLevelsJson, apiLevelsMap)
}
diff --git a/android/arch.go b/android/arch.go
index 08421a1..af3919c 100644
--- a/android/arch.go
+++ b/android/arch.go
@@ -87,7 +87,7 @@
host: {
// Host variants
},
- linux: {
+ linux_glibc: {
// Linux host variants
},
darwin: {
@@ -192,7 +192,7 @@
commonTargetMap = make(map[string]Target)
NoOsType OsType
- Linux = NewOsType("linux", Host, false)
+ Linux = NewOsType("linux_glibc", Host, false)
Darwin = NewOsType("darwin", Host, false)
LinuxBionic = NewOsType("linux_bionic", Host, true)
Windows = NewOsType("windows", HostCross, true)
@@ -242,6 +242,14 @@
return os.Name
}
+func (os OsType) Bionic() bool {
+ return os == Android || os == LinuxBionic
+}
+
+func (os OsType) Linux() bool {
+ return os == Android || os == Linux || os == LinuxBionic
+}
+
func NewOsType(name string, class OsClass, defDisabled bool) OsType {
os := OsType{
Name: name,
@@ -297,19 +305,19 @@
primaryModules := make(map[int]bool)
for _, class := range osClasses {
- targets := mctx.AConfig().Targets[class]
+ targets := mctx.Config().Targets[class]
if len(targets) == 0 {
continue
}
var multilib string
switch class {
case Device:
- multilib = module.base().commonProperties.Target.Android.Compile_multilib
+ multilib = String(module.base().commonProperties.Target.Android.Compile_multilib)
case Host, HostCross:
- multilib = module.base().commonProperties.Target.Host.Compile_multilib
+ multilib = String(module.base().commonProperties.Target.Host.Compile_multilib)
}
if multilib == "" {
- multilib = module.base().commonProperties.Compile_multilib
+ multilib = String(module.base().commonProperties.Compile_multilib)
}
if multilib == "" {
multilib = module.base().commonProperties.Default_multilib
@@ -317,7 +325,7 @@
var prefer32 bool
switch class {
case Device:
- prefer32 = mctx.AConfig().DevicePrefer32BitExecutables()
+ prefer32 = mctx.Config().DevicePrefer32BitExecutables()
case HostCross:
// Windows builds always prefer 32-bit
prefer32 = true
@@ -459,6 +467,8 @@
"Host",
"Android64",
"Android32",
+ "Bionic",
+ "Linux",
"Not_windows",
"Arm_on_x86",
"Arm_on_x86_64",
@@ -468,6 +478,19 @@
for _, archType := range osArchTypeMap[os] {
targets = append(targets, os.Field+"_"+archType.Name)
+
+ if os.Linux() {
+ target := "Linux_" + archType.Name
+ if !inList(target, targets) {
+ targets = append(targets, target)
+ }
+ }
+ if os.Bionic() {
+ target := "Bionic_" + archType.Name
+ if !inList(target, targets) {
+ targets = append(targets, target)
+ }
+ }
}
}
@@ -574,10 +597,6 @@
arch := a.Arch()
os := a.Os()
- if arch.ArchType == Common {
- return
- }
-
for i := range a.generalProperties {
genProps := a.generalProperties[i]
if a.archProperties[i] == nil {
@@ -589,6 +608,9 @@
multilibProp := archProps.FieldByName("Multilib")
targetProp := archProps.FieldByName("Target")
+ var field string
+ var prefix string
+
// Handle arch-specific properties in the form:
// arch: {
// arm64: {
@@ -597,59 +619,61 @@
// },
t := arch.ArchType
- field := proptools.FieldNameForProperty(t.Name)
- prefix := "arch." + t.Name
- archStruct := a.appendProperties(ctx, genProps, archProp, field, prefix)
+ if arch.ArchType != Common {
+ field := proptools.FieldNameForProperty(t.Name)
+ prefix := "arch." + t.Name
+ archStruct := a.appendProperties(ctx, genProps, archProp, field, prefix)
- // Handle arch-variant-specific properties in the form:
- // arch: {
- // variant: {
- // key: value,
- // },
- // },
- v := variantReplacer.Replace(arch.ArchVariant)
- if v != "" {
- field := proptools.FieldNameForProperty(v)
- prefix := "arch." + t.Name + "." + v
- a.appendProperties(ctx, genProps, archStruct, field, prefix)
- }
-
- // Handle cpu-variant-specific properties in the form:
- // arch: {
- // variant: {
- // key: value,
- // },
- // },
- if arch.CpuVariant != arch.ArchVariant {
- c := variantReplacer.Replace(arch.CpuVariant)
- if c != "" {
- field := proptools.FieldNameForProperty(c)
- prefix := "arch." + t.Name + "." + c
+ // Handle arch-variant-specific properties in the form:
+ // arch: {
+ // variant: {
+ // key: value,
+ // },
+ // },
+ v := variantReplacer.Replace(arch.ArchVariant)
+ if v != "" {
+ field := proptools.FieldNameForProperty(v)
+ prefix := "arch." + t.Name + "." + v
a.appendProperties(ctx, genProps, archStruct, field, prefix)
}
- }
- // Handle arch-feature-specific properties in the form:
- // arch: {
- // feature: {
- // key: value,
- // },
- // },
- for _, feature := range arch.ArchFeatures {
- field := proptools.FieldNameForProperty(feature)
- prefix := "arch." + t.Name + "." + feature
- a.appendProperties(ctx, genProps, archStruct, field, prefix)
- }
+ // Handle cpu-variant-specific properties in the form:
+ // arch: {
+ // variant: {
+ // key: value,
+ // },
+ // },
+ if arch.CpuVariant != arch.ArchVariant {
+ c := variantReplacer.Replace(arch.CpuVariant)
+ if c != "" {
+ field := proptools.FieldNameForProperty(c)
+ prefix := "arch." + t.Name + "." + c
+ a.appendProperties(ctx, genProps, archStruct, field, prefix)
+ }
+ }
- // Handle multilib-specific properties in the form:
- // multilib: {
- // lib32: {
- // key: value,
- // },
- // },
- field = proptools.FieldNameForProperty(t.Multilib)
- prefix = "multilib." + t.Multilib
- a.appendProperties(ctx, genProps, multilibProp, field, prefix)
+ // Handle arch-feature-specific properties in the form:
+ // arch: {
+ // feature: {
+ // key: value,
+ // },
+ // },
+ for _, feature := range arch.ArchFeatures {
+ field := proptools.FieldNameForProperty(feature)
+ prefix := "arch." + t.Name + "." + feature
+ a.appendProperties(ctx, genProps, archStruct, field, prefix)
+ }
+
+ // Handle multilib-specific properties in the form:
+ // multilib: {
+ // lib32: {
+ // key: value,
+ // },
+ // },
+ field = proptools.FieldNameForProperty(t.Multilib)
+ prefix = "multilib." + t.Multilib
+ a.appendProperties(ctx, genProps, multilibProp, field, prefix)
+ }
// Handle host-specific properties in the form:
// target: {
@@ -663,18 +687,51 @@
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
+ // Handle target OS generalities of the form:
+ // target: {
+ // bionic: {
+ // key: value,
+ // },
+ // bionic_x86: {
+ // key: value,
+ // },
+ // }
+ if os.Linux() {
+ field = "Linux"
+ prefix = "target.linux"
+ a.appendProperties(ctx, genProps, targetProp, field, prefix)
+
+ if arch.ArchType != Common {
+ field = "Linux_" + arch.ArchType.Name
+ prefix = "target.linux_" + arch.ArchType.Name
+ a.appendProperties(ctx, genProps, targetProp, field, prefix)
+ }
+ }
+
+ if os.Bionic() {
+ field = "Bionic"
+ prefix = "target.bionic"
+ a.appendProperties(ctx, genProps, targetProp, field, prefix)
+
+ if arch.ArchType != Common {
+ field = "Bionic_" + t.Name
+ prefix = "target.bionic_" + t.Name
+ a.appendProperties(ctx, genProps, targetProp, field, prefix)
+ }
+ }
+
// Handle target OS properties in the form:
// target: {
- // linux: {
+ // linux_glibc: {
// key: value,
// },
// not_windows: {
// key: value,
// },
- // linux_x86: {
+ // linux_glibc_x86: {
// key: value,
// },
- // linux_arm: {
+ // linux_glibc_arm: {
// key: value,
// },
// android {
@@ -687,14 +744,15 @@
// key: value,
// },
// },
- // },
field = os.Field
prefix = "target." + os.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
- field = os.Field + "_" + t.Name
- prefix = "target." + os.Name + "_" + t.Name
- a.appendProperties(ctx, genProps, targetProp, field, prefix)
+ if arch.ArchType != Common {
+ field = os.Field + "_" + t.Name
+ prefix = "target." + os.Name + "_" + t.Name
+ a.appendProperties(ctx, genProps, targetProp, field, prefix)
+ }
if (os.Class == Host || os.Class == HostCross) && os != Windows {
field := "Not_windows"
@@ -716,7 +774,7 @@
// that are being compiled for 64-bit. Its expected use case is binaries like linker and
// debuggerd that need to know when they are a 32-bit process running on a 64-bit device
if os.Class == Device {
- if ctx.AConfig().Android64() {
+ if ctx.Config().Android64() {
field := "Android64"
prefix := "target.android64"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
@@ -727,13 +785,13 @@
}
if arch.ArchType == X86 && (hasArmAbi(arch) ||
- hasArmAndroidArch(ctx.AConfig().Targets[Device])) {
+ hasArmAndroidArch(ctx.Config().Targets[Device])) {
field := "Arm_on_x86"
prefix := "target.arm_on_x86"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
if arch.ArchType == X86_64 && (hasArmAbi(arch) ||
- hasArmAndroidArch(ctx.AConfig().Targets[Device])) {
+ hasArmAndroidArch(ctx.Config().Targets[Device])) {
field := "Arm_on_x86_64"
prefix := "target.arm_on_x86_64"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
@@ -879,11 +937,13 @@
{"arm", "armv7-a-neon", "denver", []string{"armeabi-v7a"}},
{"arm", "armv7-a-neon", "krait", []string{"armeabi-v7a"}},
{"arm", "armv7-a-neon", "kryo", []string{"armeabi-v7a"}},
+ {"arm", "armv7-a-neon", "exynos-m1", []string{"armeabi-v7a"}},
{"arm", "armv7-a-neon", "exynos-m2", []string{"armeabi-v7a"}},
{"arm64", "armv8-a", "cortex-a53", []string{"arm64-v8a"}},
{"arm64", "armv8-a", "cortex-a73", []string{"arm64-v8a"}},
{"arm64", "armv8-a", "denver64", []string{"arm64-v8a"}},
{"arm64", "armv8-a", "kryo", []string{"arm64-v8a"}},
+ {"arm64", "armv8-a", "exynos-m1", []string{"arm64-v8a"}},
{"arm64", "armv8-a", "exynos-m2", []string{"arm64-v8a"}},
{"mips", "mips32-fp", "", []string{"mips"}},
{"mips", "mips32r2-fp", "", []string{"mips"}},
@@ -913,8 +973,6 @@
return []archConfig{
{"arm", "armv5te", "", []string{"armeabi"}},
{"arm64", "armv8-a", "", []string{"arm64-v8a"}},
- {"mips", "mips32-fp", "", []string{"mips"}},
- {"mips64", "mips64r6", "", []string{"mips64"}},
{"x86", "", "", []string{"x86"}},
{"x86_64", "", "", []string{"x86_64"}},
}
@@ -1014,19 +1072,30 @@
return ret
}
+func preferTargets(targets []Target, filters ...string) []Target {
+ for _, filter := range filters {
+ buildTargets := filterMultilibTargets(targets, filter)
+ if len(buildTargets) > 0 {
+ return buildTargets
+ }
+ }
+ return nil
+}
+
// Use the module multilib setting to select one or more targets from a target list
func decodeMultilib(multilib string, targets []Target, prefer32 bool) ([]Target, error) {
buildTargets := []Target{}
- if multilib == "first" {
- if prefer32 {
- multilib = "prefer32"
- } else {
- multilib = "prefer64"
- }
- }
+
switch multilib {
case "common":
- buildTargets = append(buildTargets, getCommonTargets(targets)...)
+ buildTargets = getCommonTargets(targets)
+ case "common_first":
+ buildTargets = getCommonTargets(targets)
+ if prefer32 {
+ buildTargets = append(buildTargets, preferTargets(targets, "lib32", "lib64")...)
+ } else {
+ buildTargets = append(buildTargets, preferTargets(targets, "lib64", "lib32")...)
+ }
case "both":
if prefer32 {
buildTargets = append(buildTargets, filterMultilibTargets(targets, "lib32")...)
@@ -1039,16 +1108,14 @@
buildTargets = filterMultilibTargets(targets, "lib32")
case "64":
buildTargets = filterMultilibTargets(targets, "lib64")
+ case "first":
+ if prefer32 {
+ buildTargets = preferTargets(targets, "lib32", "lib64")
+ } else {
+ buildTargets = preferTargets(targets, "lib64", "lib32")
+ }
case "prefer32":
- buildTargets = filterMultilibTargets(targets, "lib32")
- if len(buildTargets) == 0 {
- buildTargets = filterMultilibTargets(targets, "lib64")
- }
- case "prefer64":
- buildTargets = filterMultilibTargets(targets, "lib64")
- if len(buildTargets) == 0 {
- buildTargets = filterMultilibTargets(targets, "lib32")
- }
+ buildTargets = preferTargets(targets, "lib32", "lib64")
default:
return nil, fmt.Errorf(`compile_multilib must be "both", "first", "32", "64", or "prefer32" found %q`,
multilib)
diff --git a/android/config.go b/android/config.go
index 5dcc59b..16870f1 100644
--- a/android/config.go
+++ b/android/config.go
@@ -25,6 +25,7 @@
"strings"
"sync"
+ "github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools"
)
@@ -67,6 +68,7 @@
FileConfigurableOptions
ProductVariables productVariables
+ PrimaryBuilder string
ConfigFileName string
ProductVariablesFileName string
@@ -78,13 +80,20 @@
srcDir string // the path of the root source directory
buildDir string // the path of the build output directory
+ env map[string]string
envLock sync.Mutex
envDeps map[string]string
envFrozen bool
inMake bool
- captureBuild bool // true for tests, saves build parameters for each module
+ captureBuild bool // true for tests, saves build parameters for each module
+ ignoreEnvironment bool // true for tests, returns empty from all Getenv calls
+
+ useOpenJDK9 bool // Use OpenJDK9, but possibly target 1.8
+ targetOpenJDK9 bool // Use OpenJDK9 and target 1.9
+
+ stopBefore bootstrap.StopBefore
OncePer
}
@@ -167,22 +176,51 @@
}
// TestConfig returns a Config object suitable for using for tests
-func TestConfig(buildDir string) Config {
+func TestConfig(buildDir string, env map[string]string) Config {
config := &config{
ProductVariables: productVariables{
- DeviceName: stringPtr("test_device"),
+ DeviceName: stringPtr("test_device"),
+ Platform_sdk_version: intPtr(26),
+ AAPTConfig: &[]string{"normal", "large", "xlarge", "hdpi", "xhdpi", "xxhdpi"},
+ AAPTPreferredConfig: stringPtr("xhdpi"),
+ AAPTCharacteristics: stringPtr("nosdcard"),
+ AAPTPrebuiltDPI: &[]string{"xhdpi", "xxhdpi"},
},
buildDir: buildDir,
captureBuild: true,
+ env: env,
}
config.deviceConfig = &deviceConfig{
config: config,
}
+ if err := config.fromEnv(); err != nil {
+ panic(err)
+ }
+
return Config{config}
}
+// TestConfig returns a Config object suitable for using for tests that need to run the arch mutator
+func TestArchConfig(buildDir string, env map[string]string) Config {
+ testConfig := TestConfig(buildDir, env)
+ config := testConfig.config
+
+ config.Targets = map[OsClass][]Target{
+ Device: []Target{
+ {Android, Arch{ArchType: Arm64, ArchVariant: "armv8-a", Native: true}},
+ {Android, Arch{ArchType: Arm, ArchVariant: "armv7-a-neon", Native: true}},
+ },
+ Host: []Target{
+ {BuildOs, Arch{ArchType: X86_64}},
+ {BuildOs, Arch{ArchType: X86}},
+ },
+ }
+
+ return testConfig
+}
+
// New creates a new Config object. The srcDir argument specifies the path to
// the root source directory. It also loads the config file, if found.
func NewConfig(srcDir, buildDir string) (Config, error) {
@@ -191,6 +229,8 @@
ConfigFileName: filepath.Join(buildDir, configFileName),
ProductVariablesFileName: filepath.Join(buildDir, productVariablesFileName),
+ env: originalEnv,
+
srcDir: srcDir,
buildDir: buildDir,
}
@@ -249,17 +289,49 @@
config.Targets = targets
config.BuildOsVariant = targets[Host][0].String()
+ if err := config.fromEnv(); err != nil {
+ return Config{}, err
+ }
+
return Config{config}, nil
}
-func (c *config) RemoveAbandonedFiles() bool {
- return false
+func (c *config) fromEnv() error {
+ switch c.Getenv("EXPERIMENTAL_USE_OPENJDK9") {
+ case "":
+ // Use OpenJDK8
+ case "false":
+ // Use OpenJDK8
+ case "1.8":
+ // Use OpenJDK9, but target 1.8
+ c.useOpenJDK9 = true
+ case "true":
+ // Use OpenJDK9 and target 1.9
+ c.useOpenJDK9 = true
+ c.targetOpenJDK9 = true
+ default:
+ return fmt.Errorf(`Invalid value for EXPERIMENTAL_USE_OPENJDK9, should be "", "1.8", or "true"`)
+ }
+
+ return nil
}
+func (c *config) StopBefore() bootstrap.StopBefore {
+ return c.stopBefore
+}
+
+func (c *config) SetStopBefore(stopBefore bootstrap.StopBefore) {
+ c.stopBefore = stopBefore
+}
+
+var _ bootstrap.ConfigStopBefore = (*config)(nil)
+
func (c *config) BlueprintToolLocation() string {
return filepath.Join(c.buildDir, "host", c.PrebuiltOS(), "bin")
}
+var _ bootstrap.ConfigBlueprintToolLocation = (*config)(nil)
+
// HostSystemTool looks for non-hermetic tools from the system we're running on.
// Generally shouldn't be used, but useful to find the XCode SDK, etc.
func (c *config) HostSystemTool(name string) string {
@@ -314,7 +386,7 @@
if c.envFrozen {
panic("Cannot access new environment variables after envdeps are frozen")
}
- val, _ = originalEnv[key]
+ val, _ = c.env[key]
c.envDeps[key] = val
}
return val
@@ -362,12 +434,11 @@
return true
}
-func (c *config) ResourceOverlays() []SourcePath {
- return nil
-}
-
-func (c *config) PlatformVersion() string {
- return "M"
+func (c *config) ResourceOverlays() []string {
+ if c.ProductVariables.ResourceOverlays == nil {
+ return nil
+ }
+ return *c.ProductVariables.ResourceOverlays
}
func (c *config) PlatformSdkVersionInt() int {
@@ -378,38 +449,93 @@
return strconv.Itoa(c.PlatformSdkVersionInt())
}
-func (c *config) PlatformVersionAllCodenames() []string {
- return c.ProductVariables.Platform_version_all_codenames
+func (c *config) MinSupportedSdkVersion() int {
+ return 14
}
-func (c *config) BuildNumber() string {
- return "000000"
+func (c *config) DefaultAppTargetSdkInt() int {
+ if Bool(c.ProductVariables.Platform_sdk_final) {
+ return c.PlatformSdkVersionInt()
+ } else {
+ return 10000
+ }
}
-func (c *config) ProductAaptConfig() []string {
- return []string{"normal", "large", "xlarge", "hdpi", "xhdpi", "xxhdpi"}
+func (c *config) AppsDefaultVersionName() string {
+ return String(c.ProductVariables.AppsDefaultVersionName)
}
-func (c *config) ProductAaptPreferredConfig() string {
- return "xhdpi"
+// Codenames that are active in the current lunch target.
+func (c *config) PlatformVersionActiveCodenames() []string {
+ return c.ProductVariables.Platform_version_active_codenames
}
-func (c *config) ProductAaptCharacteristics() string {
- return "nosdcard"
+// Codenames that are available in the branch but not included in the current
+// lunch target.
+func (c *config) PlatformVersionFutureCodenames() []string {
+ return c.ProductVariables.Platform_version_future_codenames
+}
+
+// All possible codenames in the current branch. NB: Not named AllCodenames
+// because "all" has historically meant "active" in make, and still does in
+// build.prop.
+func (c *config) PlatformVersionCombinedCodenames() []string {
+ combined := []string{}
+ combined = append(combined, c.PlatformVersionActiveCodenames()...)
+ combined = append(combined, c.PlatformVersionFutureCodenames()...)
+ return combined
+}
+
+func (c *config) ProductAAPTConfig() []string {
+ return stringSlice(c.ProductVariables.AAPTConfig)
+}
+
+func (c *config) ProductAAPTPreferredConfig() string {
+ return String(c.ProductVariables.AAPTPreferredConfig)
+}
+
+func (c *config) ProductAAPTCharacteristics() string {
+ return String(c.ProductVariables.AAPTCharacteristics)
+}
+
+func (c *config) ProductAAPTPrebuiltDPI() []string {
+ return stringSlice(c.ProductVariables.AAPTPrebuiltDPI)
}
func (c *config) DefaultAppCertificateDir(ctx PathContext) SourcePath {
- return PathForSource(ctx, "build/target/product/security")
+ defaultCert := String(c.ProductVariables.DefaultAppCertificate)
+ if defaultCert != "" {
+ return PathForSource(ctx, filepath.Dir(defaultCert))
+ } else {
+ return PathForSource(ctx, "build/target/product/security")
+ }
}
func (c *config) DefaultAppCertificate(ctx PathContext) SourcePath {
- return c.DefaultAppCertificateDir(ctx).Join(ctx, "testkey")
+ defaultCert := String(c.ProductVariables.DefaultAppCertificate)
+ if defaultCert != "" {
+ return PathForSource(ctx, defaultCert)
+ } else {
+ return c.DefaultAppCertificateDir(ctx).Join(ctx, "testkey")
+ }
}
func (c *config) AllowMissingDependencies() bool {
return Bool(c.ProductVariables.Allow_missing_dependencies)
}
+func (c *config) UnbundledBuild() bool {
+ return Bool(c.ProductVariables.Unbundled_build)
+}
+
+func (c *config) IsPdkBuild() bool {
+ return Bool(c.ProductVariables.Pdk)
+}
+
+func (c *config) MinimizeJavaDebugInfo() bool {
+ return Bool(c.ProductVariables.MinimizeJavaDebugInfo) && !Bool(c.ProductVariables.Eng)
+}
+
func (c *config) DevicePrefer32BitExecutables() bool {
return Bool(c.ProductVariables.DevicePrefer32BitExecutables)
}
@@ -461,6 +587,16 @@
return Bool(c.ProductVariables.UseGoma)
}
+// Returns true if OpenJDK9 prebuilts are being used
+func (c *config) UseOpenJDK9() bool {
+ return c.useOpenJDK9
+}
+
+// Returns true if -source 1.9 -target 1.9 is being passed to javac
+func (c *config) TargetOpenJDK9() bool {
+ return c.targetOpenJDK9
+}
+
func (c *config) ClangTidy() bool {
return Bool(c.ProductVariables.ClangTidy)
}
@@ -508,11 +644,12 @@
return "vendor"
}
-func (c *deviceConfig) CompileVndk() bool {
- if c.config.ProductVariables.DeviceVndkVersion == nil {
- return false
- }
- return *c.config.ProductVariables.DeviceVndkVersion == "current"
+func (c *deviceConfig) VndkVersion() string {
+ return String(c.config.ProductVariables.DeviceVndkVersion)
+}
+
+func (c *deviceConfig) ExtraVndkVersions() []string {
+ return c.config.ProductVariables.ExtraVndkVersions
}
func (c *deviceConfig) BtConfigIncludeDir() string {
@@ -548,3 +685,25 @@
}
return prefixInList(path, *c.ProductVariables.IntegerOverflowExcludePaths)
}
+
+func (c *config) CFIDisabledForPath(path string) bool {
+ if c.ProductVariables.CFIExcludePaths == nil {
+ return false
+ }
+ return prefixInList(path, *c.ProductVariables.CFIExcludePaths)
+}
+
+func (c *config) CFIEnabledForPath(path string) bool {
+ if c.ProductVariables.CFIIncludePaths == nil {
+ return false
+ }
+ return prefixInList(path, *c.ProductVariables.CFIIncludePaths)
+}
+
+func stringSlice(s *[]string) []string {
+ if s != nil {
+ return *s
+ } else {
+ return nil
+ }
+}
diff --git a/android/config_test.go b/android/config_test.go
new file mode 100644
index 0000000..5eb6ed5
--- /dev/null
+++ b/android/config_test.go
@@ -0,0 +1,86 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func validateConfigAnnotations(configurable jsonConfigurable) (err error) {
+ reflectType := reflect.TypeOf(configurable)
+ reflectType = reflectType.Elem()
+ for i := 0; i < reflectType.NumField(); i++ {
+ field := reflectType.Field(i)
+ jsonTag := field.Tag.Get("json")
+ // Check for mistakes in the json tag
+ if jsonTag != "" && !strings.HasPrefix(jsonTag, ",") {
+ if !strings.Contains(jsonTag, ",") {
+ // Probably an accidental rename, most likely "omitempty" instead of ",omitempty"
+ return fmt.Errorf("Field %s.%s has tag %s which specifies to change its json field name to %q.\n"+
+ "Did you mean to use an annotation of %q?\n"+
+ "(Alternatively, to change the json name of the field, rename the field in source instead.)",
+ reflectType.Name(), field.Name, field.Tag, jsonTag, ","+jsonTag)
+ } else {
+ // Although this rename was probably intentional,
+ // a json annotation is still more confusing than renaming the source variable
+ requestedName := strings.Split(jsonTag, ",")[0]
+ return fmt.Errorf("Field %s.%s has tag %s which specifies to change its json field name to %q.\n"+
+ "To change the json name of the field, rename the field in source instead.",
+ reflectType.Name(), field.Name, field.Tag, requestedName)
+
+ }
+ }
+ }
+ return nil
+}
+
+type configType struct {
+ populateMe *bool `json:"omitempty"`
+}
+
+func (c *configType) SetDefaultConfig() {
+}
+
+// tests that ValidateConfigAnnotation works
+func TestValidateConfigAnnotations(t *testing.T) {
+ config := configType{}
+ err := validateConfigAnnotations(&config)
+ expectedError := `Field configType.populateMe has tag json:"omitempty" which specifies to change its json field name to "omitempty".
+Did you mean to use an annotation of ",omitempty"?
+(Alternatively, to change the json name of the field, rename the field in source instead.)`
+ if err.Error() != expectedError {
+ t.Errorf("Incorrect error; expected:\n"+
+ "%s\n"+
+ "got:\n"+
+ "%s",
+ expectedError, err.Error())
+ }
+}
+
+// run validateConfigAnnotations against each type that might have json annotations
+func TestProductConfigAnnotations(t *testing.T) {
+ err := validateConfigAnnotations(&productVariables{})
+ if err != nil {
+ t.Errorf(err.Error())
+ }
+
+ validateConfigAnnotations(&FileConfigurableOptions{})
+ if err != nil {
+ t.Errorf(err.Error())
+ }
+}
diff --git a/android/defaults.go b/android/defaults.go
index 4bf872e..c704529 100644
--- a/android/defaults.go
+++ b/android/defaults.go
@@ -131,7 +131,7 @@
func defaultsMutator(ctx TopDownMutatorContext) {
if defaultable, ok := ctx.Module().(Defaultable); ok && len(defaultable.defaults().Defaults) > 0 {
var defaultsList []Defaults
- ctx.WalkDeps(func(module, parent blueprint.Module) bool {
+ ctx.WalkDeps(func(module, parent Module) bool {
if ctx.OtherModuleDependencyTag(module) == DefaultsDepTag {
if defaults, ok := module.(Defaults); ok {
defaultsList = append(defaultsList, defaults)
diff --git a/android/defs.go b/android/defs.go
index ec8dcf9..cd8b4e3 100644
--- a/android/defs.go
+++ b/android/defs.go
@@ -52,6 +52,13 @@
},
"cpFlags")
+ CpExecutable = pctx.AndroidStaticRule("CpExecutable",
+ blueprint.RuleParams{
+ Command: "rm -f $out && cp $cpPreserveSymlinks $cpFlags $in $out && chmod +x $out",
+ Description: "cp $out",
+ },
+ "cpFlags")
+
// A timestamp touch rule.
Touch = pctx.AndroidStaticRule("Touch",
blueprint.RuleParams{
diff --git a/android/env.go b/android/env.go
index ec5794e..469dfff 100644
--- a/android/env.go
+++ b/android/env.go
@@ -19,8 +19,6 @@
"strings"
"android/soong/env"
-
- "github.com/google/blueprint"
)
// This file supports dependencies on environment variables. During build manifest generation,
@@ -43,14 +41,14 @@
os.Clearenv()
}
-func EnvSingleton() blueprint.Singleton {
+func EnvSingleton() Singleton {
return &envSingleton{}
}
type envSingleton struct{}
-func (c *envSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
- envDeps := ctx.Config().(Config).EnvDeps()
+func (c *envSingleton) GenerateBuildActions(ctx SingletonContext) {
+ envDeps := ctx.Config().EnvDeps()
envFile := PathForOutput(ctx, ".soong.environment")
if ctx.Failed() {
diff --git a/android/hooks.go b/android/hooks.go
index 7530f8d..57560d2 100644
--- a/android/hooks.go
+++ b/android/hooks.go
@@ -16,7 +16,6 @@
import (
"github.com/google/blueprint"
- "github.com/google/blueprint/proptools"
)
// This file implements hooks that external module types can use to inject logic into existing
@@ -27,10 +26,11 @@
// before the module has been split into architecture variants, and before defaults modules have
// been applied.
type LoadHookContext interface {
- // TODO: a new context that includes AConfig() but not Target(), etc.?
+ // TODO: a new context that includes Config() but not Target(), etc.?
BaseContext
AppendProperties(...interface{})
PrependProperties(...interface{})
+ CreateModule(blueprint.ModuleFactory, ...interface{})
}
// Arch hooks are run after the module has been split into architecture variants, and can be used
@@ -51,62 +51,22 @@
h.arch = append(h.arch, hook)
}
-type propertyHookContext struct {
- BaseContext
-
- module *ModuleBase
-}
-
-func (ctx *propertyHookContext) AppendProperties(props ...interface{}) {
- for _, p := range props {
- err := proptools.AppendMatchingProperties(ctx.module.customizableProperties, p, nil)
- if err != nil {
- if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok {
- ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error())
- } else {
- panic(err)
- }
- }
- }
-}
-
-func (ctx *propertyHookContext) PrependProperties(props ...interface{}) {
- for _, p := range props {
- err := proptools.PrependMatchingProperties(ctx.module.customizableProperties, p, nil)
- if err != nil {
- if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok {
- ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error())
- } else {
- panic(err)
- }
- }
- }
-}
-
-func (x *hooks) runLoadHooks(ctx BaseContext, m *ModuleBase) {
+func (x *hooks) runLoadHooks(ctx LoadHookContext, m *ModuleBase) {
if len(x.load) > 0 {
- mctx := &propertyHookContext{
- BaseContext: ctx,
- module: m,
- }
for _, x := range x.load {
- x(mctx)
- if mctx.Failed() {
+ x(ctx)
+ if ctx.Failed() {
return
}
}
}
}
-func (x *hooks) runArchHooks(ctx BaseContext, m *ModuleBase) {
+func (x *hooks) runArchHooks(ctx ArchHookContext, m *ModuleBase) {
if len(x.arch) > 0 {
- mctx := &propertyHookContext{
- BaseContext: ctx,
- module: m,
- }
for _, x := range x.arch {
- x(mctx)
- if mctx.Failed() {
+ x(ctx)
+ if ctx.Failed() {
return
}
}
@@ -165,12 +125,18 @@
func loadHookMutator(ctx TopDownMutatorContext) {
if m, ok := ctx.Module().(Module); ok {
- m.base().hooks.runLoadHooks(ctx, m.base())
+ // Cast through *androidTopDownMutatorContext because AppendProperties is implemented
+ // on *androidTopDownMutatorContext but not exposed through TopDownMutatorContext
+ var loadHookCtx LoadHookContext = ctx.(*androidTopDownMutatorContext)
+ m.base().hooks.runLoadHooks(loadHookCtx, m.base())
}
}
func archHookMutator(ctx TopDownMutatorContext) {
if m, ok := ctx.Module().(Module); ok {
- m.base().hooks.runArchHooks(ctx, m.base())
+ // Cast through *androidTopDownMutatorContext because AppendProperties is implemented
+ // on *androidTopDownMutatorContext but not exposed through TopDownMutatorContext
+ var archHookCtx ArchHookContext = ctx.(*androidTopDownMutatorContext)
+ m.base().hooks.runArchHooks(archHookCtx, m.base())
}
}
diff --git a/android/makevars.go b/android/makevars.go
index 482fbde..00a20f5 100644
--- a/android/makevars.go
+++ b/android/makevars.go
@@ -19,11 +19,19 @@
"fmt"
"io/ioutil"
"os"
+ "strconv"
- "github.com/google/blueprint"
"github.com/google/blueprint/proptools"
)
+func init() {
+ RegisterMakeVarsProvider(pctx, androidMakeVarsProvider)
+}
+
+func androidMakeVarsProvider(ctx MakeVarsContext) {
+ ctx.Strict("MIN_SUPPORTED_SDK_VERSION", strconv.Itoa(ctx.Config().MinSupportedSdkVersion()))
+}
+
///////////////////////////////////////////////////////////////////////////////
// Interface for other packages to use to declare make variables
type MakeVarsContext interface {
@@ -57,7 +65,7 @@
type MakeVarsProvider func(ctx MakeVarsContext)
-func RegisterMakeVarsProvider(pctx blueprint.PackageContext, provider MakeVarsProvider) {
+func RegisterMakeVarsProvider(pctx PackageContext, provider MakeVarsProvider) {
makeVarsProviders = append(makeVarsProviders, makeVarsProvider{pctx, provider})
}
@@ -67,14 +75,14 @@
RegisterSingletonType("makevars", makeVarsSingletonFunc)
}
-func makeVarsSingletonFunc() blueprint.Singleton {
+func makeVarsSingletonFunc() Singleton {
return &makeVarsSingleton{}
}
type makeVarsSingleton struct{}
type makeVarsProvider struct {
- pctx blueprint.PackageContext
+ pctx PackageContext
call MakeVarsProvider
}
@@ -82,8 +90,8 @@
type makeVarsContext struct {
config Config
- ctx blueprint.SingletonContext
- pctx blueprint.PackageContext
+ ctx SingletonContext
+ pctx PackageContext
vars []makeVarsVariable
}
@@ -96,14 +104,12 @@
strict bool
}
-func (s *makeVarsSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
- config := ctx.Config().(Config)
-
- if !config.EmbeddedInMake() {
+func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) {
+ if !ctx.Config().EmbeddedInMake() {
return
}
- outFile := PathForOutput(ctx, "make_vars"+proptools.String(config.ProductVariables.Make_suffix)+".mk").String()
+ outFile := PathForOutput(ctx, "make_vars"+proptools.String(ctx.Config().ProductVariables.Make_suffix)+".mk").String()
if ctx.Failed() {
return
@@ -112,7 +118,7 @@
vars := []makeVarsVariable{}
for _, provider := range makeVarsProviders {
mctx := &makeVarsContext{
- config: config,
+ config: ctx.Config(),
ctx: ctx,
pctx: provider.pctx,
}
diff --git a/android/module.go b/android/module.go
index 3a3d173..3d8f683 100644
--- a/android/module.go
+++ b/android/module.go
@@ -17,7 +17,9 @@
import (
"fmt"
"path/filepath"
+ "sort"
"strings"
+ "text/scanner"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
@@ -32,7 +34,7 @@
HostExecutable = "host_executable"
)
-type ModuleBuildParams struct {
+type BuildParams struct {
Rule blueprint.Rule
Deps blueprint.Deps
Depfile WritablePath
@@ -50,6 +52,8 @@
Args map[string]string
}
+type ModuleBuildParams BuildParams
+
type androidBaseContext interface {
Target() Target
TargetPrimary() bool
@@ -61,30 +65,53 @@
Windows() bool
Debug() bool
PrimaryArch() bool
- Vendor() bool
+ InstallOnVendorPartition() bool
AConfig() Config
DeviceConfig() DeviceConfig
}
type BaseContext interface {
- blueprint.BaseModuleContext
+ BaseModuleContext
androidBaseContext
}
-type ModuleContext interface {
- blueprint.ModuleContext
- androidBaseContext
+// BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns
+// a Config instead of an interface{}.
+type BaseModuleContext interface {
+ ModuleName() string
+ ModuleDir() string
+ Config() Config
- // Similar to Build, but takes Paths instead of []string,
- // and performs more verification.
- ModuleBuild(pctx blueprint.PackageContext, params ModuleBuildParams)
+ ContainsProperty(name string) bool
+ Errorf(pos scanner.Position, fmt string, args ...interface{})
+ ModuleErrorf(fmt string, args ...interface{})
+ PropertyErrorf(property, fmt string, args ...interface{})
+ Failed() bool
+
+ // GlobWithDeps returns a list of files that match the specified pattern but do not match any
+ // of the patterns in excludes. It also adds efficient dependencies to rerun the primary
+ // builder whenever a file matching the pattern as added or removed, without rerunning if a
+ // file that does not match the pattern is added to a searched directory.
+ GlobWithDeps(pattern string, excludes []string) ([]string, error)
+
+ Fs() pathtools.FileSystem
+ AddNinjaFileDeps(deps ...string)
+}
+
+type ModuleContext interface {
+ androidBaseContext
+ BaseModuleContext
+
+ // Deprecated: use ModuleContext.Build instead.
+ ModuleBuild(pctx PackageContext, params ModuleBuildParams)
ExpandSources(srcFiles, excludes []string) Paths
+ ExpandSource(srcFile, prop string) Path
ExpandSourcesSubDir(srcFiles, excludes []string, subDir string) Paths
Glob(globPattern string, excludes []string) Paths
- InstallFile(installPath OutputPath, srcPath Path, deps ...Path) OutputPath
- InstallFileName(installPath OutputPath, name string, srcPath Path, deps ...Path) OutputPath
+ InstallExecutable(installPath OutputPath, name string, srcPath Path, deps ...Path) OutputPath
+ InstallFile(installPath OutputPath, name string, srcPath Path, deps ...Path) OutputPath
InstallSymlink(installPath OutputPath, name string, srcPath OutputPath) OutputPath
CheckbuildFile(srcPath Path)
@@ -94,12 +121,48 @@
InstallInSanitizerDir() bool
RequiredModuleNames() []string
+
+ // android.ModuleContext methods
+ // These are duplicated instead of embedded so that can eventually be wrapped to take an
+ // android.Module instead of a blueprint.Module
+ OtherModuleName(m blueprint.Module) string
+ OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{})
+ OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag
+
+ GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module
+ GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag)
+
+ ModuleSubDir() string
+
+ VisitDirectDepsBlueprint(visit func(blueprint.Module))
+ VisitDirectDeps(visit func(Module))
+ VisitDirectDepsIf(pred func(Module) bool, visit func(Module))
+ VisitDepsDepthFirst(visit func(Module))
+ VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module))
+ WalkDeps(visit func(Module, Module) bool)
+
+ Variable(pctx PackageContext, name, value string)
+ Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule
+ // Similar to blueprint.ModuleContext.Build, but takes Paths instead of []string,
+ // and performs more verification.
+ Build(pctx PackageContext, params BuildParams)
+
+ PrimaryModule() Module
+ FinalModule() Module
+ VisitAllModuleVariants(visit func(Module))
+
+ GetMissingDependencies() []string
+ Namespace() blueprint.Namespace
}
type Module interface {
blueprint.Module
+ // GenerateAndroidBuildActions is analogous to Blueprints' GenerateBuildActions,
+ // but GenerateAndroidBuildActions also has access to Android-specific information.
+ // For more information, see Module.GenerateBuildActions within Blueprint's module_ctx.go
GenerateAndroidBuildActions(ModuleContext)
+
DepsMutator(BottomUpMutatorContext)
base() *ModuleBase
@@ -112,12 +175,12 @@
AddProperties(props ...interface{})
GetProperties() []interface{}
- BuildParamsForTests() []ModuleBuildParams
+ BuildParamsForTests() []BuildParams
}
type nameProperties struct {
// The name of the module. Must be unique across all modules.
- Name string
+ Name *string
}
type commonProperties struct {
@@ -130,31 +193,27 @@
// are "32" (compile for 32-bit only), "64" (compile for 64-bit only), "both" (compile for both
// architectures), or "first" (compile for 64-bit on a 64-bit platform, and 32-bit on a 32-bit
// platform
- Compile_multilib string `android:"arch_variant"`
+ Compile_multilib *string `android:"arch_variant"`
Target struct {
Host struct {
- Compile_multilib string
+ Compile_multilib *string
}
Android struct {
- Compile_multilib string
+ Compile_multilib *string
}
}
Default_multilib string `blueprint:"mutated"`
// whether this is a proprietary vendor module, and should be installed into /vendor
- Proprietary bool
+ Proprietary *bool
// vendor who owns this module
Owner *string
// whether this module is device specific and should be installed into /vendor
- Vendor bool
-
- // *.logtags files, to combine together in order to generate the /system/etc/event-log-tags
- // file
- Logtags []string
+ Vendor *bool
// init.rc files to be installed if this module is installed
Init_rc []string
@@ -162,6 +221,9 @@
// names of other modules to install if this module is installed
Required []string `android:"arch_variant"`
+ // relative path to a file to include in the list of notices for the device
+ Notice *string
+
// Set by TargetMutator
CompileTarget Target `blueprint:"mutated"`
CompilePrimary bool `blueprint:"mutated"`
@@ -171,6 +233,8 @@
ArchSpecific bool `blueprint:"mutated"`
SkipInstall bool `blueprint:"mutated"`
+
+ NamespaceExportedToMake bool `blueprint:"mutated"`
}
type hostAndDeviceProperties struct {
@@ -181,10 +245,11 @@
type Multilib string
const (
- MultilibBoth Multilib = "both"
- MultilibFirst Multilib = "first"
- MultilibCommon Multilib = "common"
- MultilibDefault Multilib = ""
+ MultilibBoth Multilib = "both"
+ MultilibFirst Multilib = "first"
+ MultilibCommon Multilib = "common"
+ MultilibCommonFirst Multilib = "common_first"
+ MultilibDefault Multilib = ""
)
type HostOrDeviceSupported int
@@ -286,8 +351,8 @@
// Used by buildTargetSingleton to create checkbuild and per-directory build targets
// Only set on the final variant of each module
- installTarget string
- checkbuildTarget string
+ installTarget WritablePath
+ checkbuildTarget WritablePath
blueprintDir string
hooks hooks
@@ -295,7 +360,7 @@
registerProps []interface{}
// For tests
- buildParams []ModuleBuildParams
+ buildParams []BuildParams
}
func (a *ModuleBase) AddProperties(props ...interface{}) {
@@ -306,19 +371,19 @@
return a.registerProps
}
-func (a *ModuleBase) BuildParamsForTests() []ModuleBuildParams {
+func (a *ModuleBase) BuildParamsForTests() []BuildParams {
return a.buildParams
}
// Name returns the name of the module. It may be overridden by individual module types, for
// example prebuilts will prepend prebuilt_ to the name.
func (a *ModuleBase) Name() string {
- return a.nameProperties.Name
+ return String(a.nameProperties.Name)
}
// BaseModuleName returns the name of the module as specified in the blueprints file.
func (a *ModuleBase) BaseModuleName() string {
- return a.nameProperties.Name
+ return String(a.nameProperties.Name)
}
func (a *ModuleBase) base() *ModuleBase {
@@ -425,36 +490,40 @@
return false
}
-func (a *ModuleBase) generateModuleTarget(ctx blueprint.ModuleContext) {
+func (a *ModuleBase) generateModuleTarget(ctx ModuleContext) {
allInstalledFiles := Paths{}
allCheckbuildFiles := Paths{}
- ctx.VisitAllModuleVariants(func(module blueprint.Module) {
- a := module.(Module).base()
+ ctx.VisitAllModuleVariants(func(module Module) {
+ a := module.base()
allInstalledFiles = append(allInstalledFiles, a.installFiles...)
allCheckbuildFiles = append(allCheckbuildFiles, a.checkbuildFiles...)
})
- deps := []string{}
+ var deps Paths
+
+ namespacePrefix := ctx.Namespace().(*Namespace).id
+ if namespacePrefix != "" {
+ namespacePrefix = namespacePrefix + "-"
+ }
if len(allInstalledFiles) > 0 {
- name := ctx.ModuleName() + "-install"
- ctx.Build(pctx, blueprint.BuildParams{
+ name := PathForPhony(ctx, namespacePrefix+ctx.ModuleName()+"-install")
+ ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
- Outputs: []string{name},
- Implicits: allInstalledFiles.Strings(),
- Optional: ctx.Config().(Config).EmbeddedInMake(),
+ Output: name,
+ Implicits: allInstalledFiles,
+ Default: !ctx.Config().EmbeddedInMake(),
})
deps = append(deps, name)
a.installTarget = name
}
if len(allCheckbuildFiles) > 0 {
- name := ctx.ModuleName() + "-checkbuild"
- ctx.Build(pctx, blueprint.BuildParams{
+ name := PathForPhony(ctx, namespacePrefix+ctx.ModuleName()+"-checkbuild")
+ ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
- Outputs: []string{name},
- Implicits: allCheckbuildFiles.Strings(),
- Optional: true,
+ Output: name,
+ Implicits: allCheckbuildFiles,
})
deps = append(deps, name)
a.checkbuildTarget = name
@@ -462,15 +531,15 @@
if len(deps) > 0 {
suffix := ""
- if ctx.Config().(Config).EmbeddedInMake() {
+ if ctx.Config().EmbeddedInMake() {
suffix = "-soong"
}
- ctx.Build(pctx, blueprint.BuildParams{
+ name := PathForPhony(ctx, namespacePrefix+ctx.ModuleName()+suffix)
+ ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
- Outputs: []string{ctx.ModuleName() + suffix},
+ Outputs: []WritablePath{name},
Implicits: deps,
- Optional: true,
})
a.blueprintDir = ctx.ModuleDir()
@@ -481,28 +550,28 @@
return androidBaseContextImpl{
target: a.commonProperties.CompileTarget,
targetPrimary: a.commonProperties.CompilePrimary,
- vendor: a.commonProperties.Proprietary || a.commonProperties.Vendor,
+ vendor: Bool(a.commonProperties.Proprietary) || Bool(a.commonProperties.Vendor),
config: ctx.Config().(Config),
}
}
-func (a *ModuleBase) GenerateBuildActions(ctx blueprint.ModuleContext) {
- androidCtx := &androidModuleContext{
+func (a *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) {
+ ctx := &androidModuleContext{
module: a.module,
- ModuleContext: ctx,
- androidBaseContextImpl: a.androidBaseContextFactory(ctx),
- installDeps: a.computeInstallDeps(ctx),
+ ModuleContext: blueprintCtx,
+ androidBaseContextImpl: a.androidBaseContextFactory(blueprintCtx),
+ installDeps: a.computeInstallDeps(blueprintCtx),
installFiles: a.installFiles,
- missingDeps: ctx.GetMissingDependencies(),
+ missingDeps: blueprintCtx.GetMissingDependencies(),
}
desc := "//" + ctx.ModuleDir() + ":" + ctx.ModuleName() + " "
var suffix []string
- if androidCtx.Os().Class != Device && androidCtx.Os().Class != Generic {
- suffix = append(suffix, androidCtx.Os().String())
+ if ctx.Os().Class != Device && ctx.Os().Class != Generic {
+ suffix = append(suffix, ctx.Os().String())
}
- if !androidCtx.PrimaryArch() {
- suffix = append(suffix, androidCtx.Arch().ArchType.String())
+ if !ctx.PrimaryArch() {
+ suffix = append(suffix, ctx.Arch().ArchType.String())
}
ctx.Variable(pctx, "moduleDesc", desc)
@@ -514,13 +583,13 @@
ctx.Variable(pctx, "moduleDescSuffix", s)
if a.Enabled() {
- a.module.GenerateAndroidBuildActions(androidCtx)
+ a.module.GenerateAndroidBuildActions(ctx)
if ctx.Failed() {
return
}
- a.installFiles = append(a.installFiles, androidCtx.installFiles...)
- a.checkbuildFiles = append(a.checkbuildFiles, androidCtx.checkbuildFiles...)
+ a.installFiles = append(a.installFiles, ctx.installFiles...)
+ a.checkbuildFiles = append(a.checkbuildFiles, ctx.checkbuildFiles...)
}
if a == ctx.FinalModule().(Module).base() {
@@ -530,7 +599,7 @@
}
}
- a.buildParams = androidCtx.buildParams
+ a.buildParams = ctx.buildParams
}
type androidBaseContextImpl struct {
@@ -551,11 +620,11 @@
module Module
// For tests
- buildParams []ModuleBuildParams
+ buildParams []BuildParams
}
func (a *androidModuleContext) ninjaError(desc string, outputs []string, err error) {
- a.ModuleContext.Build(pctx, blueprint.BuildParams{
+ a.ModuleContext.Build(pctx.PackageContext, blueprint.BuildParams{
Rule: ErrorRule,
Description: desc,
Outputs: outputs,
@@ -567,25 +636,18 @@
return
}
-func (a *androidModuleContext) Build(pctx blueprint.PackageContext, params blueprint.BuildParams) {
- if a.missingDeps != nil {
- a.ninjaError(params.Description, params.Outputs,
- fmt.Errorf("module %s missing dependencies: %s\n",
- a.ModuleName(), strings.Join(a.missingDeps, ", ")))
- return
- }
-
- params.Optional = true
- a.ModuleContext.Build(pctx, params)
+func (a *androidModuleContext) Config() Config {
+ return a.ModuleContext.Config().(Config)
}
-func (a *androidModuleContext) ModuleBuild(pctx blueprint.PackageContext, params ModuleBuildParams) {
- if a.config.captureBuild {
- a.buildParams = append(a.buildParams, params)
- }
+func (a *androidModuleContext) ModuleBuild(pctx PackageContext, params ModuleBuildParams) {
+ a.Build(pctx, BuildParams(params))
+}
+func convertBuildParams(params BuildParams) blueprint.BuildParams {
bparams := blueprint.BuildParams{
Rule: params.Rule,
+ Description: params.Description,
Deps: params.Deps,
Outputs: params.Outputs.Strings(),
ImplicitOutputs: params.ImplicitOutputs.Strings(),
@@ -596,10 +658,6 @@
Optional: !params.Default,
}
- if params.Description != "" {
- bparams.Description = "${moduleDesc}" + params.Description + "${moduleDescSuffix}"
- }
-
if params.Depfile != nil {
bparams.Depfile = params.Depfile.String()
}
@@ -616,6 +674,30 @@
bparams.Implicits = append(bparams.Implicits, params.Implicit.String())
}
+ return bparams
+}
+
+func (a *androidModuleContext) Variable(pctx PackageContext, name, value string) {
+ a.ModuleContext.Variable(pctx.PackageContext, name, value)
+}
+
+func (a *androidModuleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams,
+ argNames ...string) blueprint.Rule {
+
+ return a.ModuleContext.Rule(pctx.PackageContext, name, params, argNames...)
+}
+
+func (a *androidModuleContext) Build(pctx PackageContext, params BuildParams) {
+ if a.config.captureBuild {
+ a.buildParams = append(a.buildParams, params)
+ }
+
+ bparams := convertBuildParams(params)
+
+ if bparams.Description != "" {
+ bparams.Description = "${moduleDesc}" + params.Description + "${moduleDescSuffix}"
+ }
+
if a.missingDeps != nil {
a.ninjaError(bparams.Description, bparams.Outputs,
fmt.Errorf("module %s missing dependencies: %s\n",
@@ -623,7 +705,7 @@
return
}
- a.ModuleContext.Build(pctx, bparams)
+ a.ModuleContext.Build(pctx.PackageContext, bparams)
}
func (a *androidModuleContext) GetMissingDependencies() []string {
@@ -633,9 +715,107 @@
func (a *androidModuleContext) AddMissingDependencies(deps []string) {
if deps != nil {
a.missingDeps = append(a.missingDeps, deps...)
+ a.missingDeps = FirstUniqueStrings(a.missingDeps)
}
}
+func (a *androidModuleContext) validateAndroidModule(module blueprint.Module) Module {
+ aModule, _ := module.(Module)
+ if aModule == nil {
+ a.ModuleErrorf("module %q not an android module", a.OtherModuleName(aModule))
+ return nil
+ }
+
+ if !aModule.Enabled() {
+ if a.Config().AllowMissingDependencies() {
+ a.AddMissingDependencies([]string{a.OtherModuleName(aModule)})
+ } else {
+ a.ModuleErrorf("depends on disabled module %q", a.OtherModuleName(aModule))
+ }
+ return nil
+ }
+
+ return aModule
+}
+
+func (a *androidModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) {
+ a.ModuleContext.VisitDirectDeps(visit)
+}
+
+func (a *androidModuleContext) VisitDirectDeps(visit func(Module)) {
+ a.ModuleContext.VisitDirectDeps(func(module blueprint.Module) {
+ if aModule := a.validateAndroidModule(module); aModule != nil {
+ visit(aModule)
+ }
+ })
+}
+
+func (a *androidModuleContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) {
+ a.ModuleContext.VisitDirectDepsIf(
+ // pred
+ func(module blueprint.Module) bool {
+ if aModule := a.validateAndroidModule(module); aModule != nil {
+ return pred(aModule)
+ } else {
+ return false
+ }
+ },
+ // visit
+ func(module blueprint.Module) {
+ visit(module.(Module))
+ })
+}
+
+func (a *androidModuleContext) VisitDepsDepthFirst(visit func(Module)) {
+ a.ModuleContext.VisitDepsDepthFirst(func(module blueprint.Module) {
+ if aModule := a.validateAndroidModule(module); aModule != nil {
+ visit(aModule)
+ }
+ })
+}
+
+func (a *androidModuleContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) {
+ a.ModuleContext.VisitDepsDepthFirstIf(
+ // pred
+ func(module blueprint.Module) bool {
+ if aModule := a.validateAndroidModule(module); aModule != nil {
+ return pred(aModule)
+ } else {
+ return false
+ }
+ },
+ // visit
+ func(module blueprint.Module) {
+ visit(module.(Module))
+ })
+}
+
+func (a *androidModuleContext) WalkDeps(visit func(Module, Module) bool) {
+ a.ModuleContext.WalkDeps(func(child, parent blueprint.Module) bool {
+ childAndroidModule := a.validateAndroidModule(child)
+ parentAndroidModule := a.validateAndroidModule(parent)
+ if childAndroidModule != nil && parentAndroidModule != nil {
+ return visit(childAndroidModule, parentAndroidModule)
+ } else {
+ return false
+ }
+ })
+}
+
+func (a *androidModuleContext) VisitAllModuleVariants(visit func(Module)) {
+ a.ModuleContext.VisitAllModuleVariants(func(module blueprint.Module) {
+ visit(module.(Module))
+ })
+}
+
+func (a *androidModuleContext) PrimaryModule() Module {
+ return a.ModuleContext.PrimaryModule().(Module)
+}
+
+func (a *androidModuleContext) FinalModule() Module {
+ return a.ModuleContext.FinalModule().(Module)
+}
+
func (a *androidBaseContextImpl) Target() Target {
return a.target
}
@@ -687,7 +867,7 @@
return DeviceConfig{a.config.deviceConfig}
}
-func (a *androidBaseContextImpl) Vendor() bool {
+func (a *androidBaseContextImpl) InstallOnVendorPartition() bool {
return a.vendor
}
@@ -705,11 +885,11 @@
}
if a.Device() {
- if a.AConfig().SkipDeviceInstall() {
+ if a.Config().SkipDeviceInstall() {
return true
}
- if a.AConfig().SkipMegaDeviceInstall(fullInstallPath.String()) {
+ if a.Config().SkipMegaDeviceInstall(fullInstallPath.String()) {
return true
}
}
@@ -717,8 +897,18 @@
return false
}
-func (a *androidModuleContext) InstallFileName(installPath OutputPath, name string, srcPath Path,
+func (a *androidModuleContext) InstallFile(installPath OutputPath, name string, srcPath Path,
deps ...Path) OutputPath {
+ return a.installFile(installPath, name, srcPath, Cp, deps)
+}
+
+func (a *androidModuleContext) InstallExecutable(installPath OutputPath, name string, srcPath Path,
+ deps ...Path) OutputPath {
+ return a.installFile(installPath, name, srcPath, CpExecutable, deps)
+}
+
+func (a *androidModuleContext) installFile(installPath OutputPath, name string, srcPath Path,
+ rule blueprint.Rule, deps []Path) OutputPath {
fullInstallPath := installPath.Join(a, name)
a.module.base().hooks.runInstallHooks(a, fullInstallPath, false)
@@ -737,14 +927,14 @@
orderOnlyDeps = deps
}
- a.ModuleBuild(pctx, ModuleBuildParams{
- Rule: Cp,
+ a.Build(pctx, BuildParams{
+ Rule: rule,
Description: "install " + fullInstallPath.Base(),
Output: fullInstallPath,
Input: srcPath,
Implicits: implicitDeps,
OrderOnly: orderOnlyDeps,
- Default: !a.AConfig().EmbeddedInMake(),
+ Default: !a.Config().EmbeddedInMake(),
})
a.installFiles = append(a.installFiles, fullInstallPath)
@@ -753,22 +943,18 @@
return fullInstallPath
}
-func (a *androidModuleContext) InstallFile(installPath OutputPath, srcPath Path, deps ...Path) OutputPath {
- return a.InstallFileName(installPath, filepath.Base(srcPath.String()), srcPath, deps...)
-}
-
func (a *androidModuleContext) InstallSymlink(installPath OutputPath, name string, srcPath OutputPath) OutputPath {
fullInstallPath := installPath.Join(a, name)
a.module.base().hooks.runInstallHooks(a, fullInstallPath, true)
if !a.skipInstall(fullInstallPath) {
- a.ModuleBuild(pctx, ModuleBuildParams{
+ a.Build(pctx, BuildParams{
Rule: Symlink,
Description: "install symlink " + fullInstallPath.Base(),
Output: fullInstallPath,
OrderOnly: Paths{srcPath},
- Default: !a.AConfig().EmbeddedInMake(),
+ Default: !a.Config().EmbeddedInMake(),
Args: map[string]string{
"fromPath": srcPath.String(),
},
@@ -820,8 +1006,8 @@
var SourceDepTag sourceDependencyTag
-// Returns a list of modules that must be depended on to satisfy filegroup or generated sources
-// modules listed in srcFiles using ":module" syntax
+// Adds necessary dependencies to satisfy filegroup or generated sources modules listed in srcFiles
+// using ":module" syntax, if any.
func ExtractSourcesDeps(ctx BottomUpMutatorContext, srcFiles []string) {
var deps []string
set := make(map[string]bool)
@@ -840,6 +1026,16 @@
ctx.AddDependency(ctx.Module(), SourceDepTag, deps...)
}
+// Adds necessary dependencies to satisfy filegroup or generated sources modules specified in s
+// using ":module" syntax, if any.
+func ExtractSourceDeps(ctx BottomUpMutatorContext, s *string) {
+ if s != nil {
+ if m := SrcIsModule(*s); m != "" {
+ ctx.AddDependency(ctx.Module(), SourceDepTag, m)
+ }
+ }
+}
+
type SourceFileProducer interface {
Srcs() Paths
}
@@ -850,6 +1046,18 @@
return ctx.ExpandSourcesSubDir(srcFiles, excludes, "")
}
+// Returns a single path expanded from globs and modules referenced using ":module" syntax.
+// ExtractSourceDeps must have already been called during the dependency resolution phase.
+func (ctx *androidModuleContext) ExpandSource(srcFile, prop string) Path {
+ srcFiles := ctx.ExpandSourcesSubDir([]string{srcFile}, nil, "")
+ if len(srcFiles) == 1 {
+ return srcFiles[0]
+ } else {
+ ctx.PropertyErrorf(prop, "module providing %s must produce exactly one file", prop)
+ return nil
+ }
+}
+
func (ctx *androidModuleContext) ExpandSourcesSubDir(srcFiles, excludes []string, subDir string) Paths {
prefix := PathForModuleSrc(ctx).String()
@@ -866,6 +1074,10 @@
for _, s := range srcFiles {
if m := SrcIsModule(s); m != "" {
module := ctx.GetDirectDepWithTag(m, SourceDepTag)
+ if module == nil {
+ // Error will have been handled by ExtractSourcesDeps
+ continue
+ }
if srcProducer, ok := module.(SourceFileProducer); ok {
expandedSrcFiles = append(expandedSrcFiles, srcProducer.Srcs()...)
} else {
@@ -873,10 +1085,10 @@
}
} else if pathtools.IsGlob(s) {
globbedSrcFiles := ctx.Glob(filepath.Join(prefix, s), excludes)
- expandedSrcFiles = append(expandedSrcFiles, globbedSrcFiles...)
- for i, s := range expandedSrcFiles {
- expandedSrcFiles[i] = s.(ModuleSrcPath).WithSubDir(ctx, subDir)
+ for i, s := range globbedSrcFiles {
+ globbedSrcFiles[i] = s.(ModuleSrcPath).WithSubDir(ctx, subDir)
}
+ expandedSrcFiles = append(expandedSrcFiles, globbedSrcFiles...)
} else {
s := PathForModuleSrc(ctx, s).WithSubDir(ctx, subDir)
expandedSrcFiles = append(expandedSrcFiles, s)
@@ -902,7 +1114,7 @@
RegisterSingletonType("buildtarget", BuildTargetSingleton)
}
-func BuildTargetSingleton() blueprint.Singleton {
+func BuildTargetSingleton() Singleton {
return &buildTargetSingleton{}
}
@@ -913,45 +1125,57 @@
type buildTargetSingleton struct{}
-func (c *buildTargetSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
- checkbuildDeps := []string{}
+func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) {
+ var checkbuildDeps Paths
- mmTarget := func(dir string) string {
- return filepath.Join("mm", dir)
+ mmTarget := func(dir string) WritablePath {
+ return PathForPhony(ctx,
+ "MODULES-IN-"+strings.Replace(filepath.Clean(dir), "/", "-", -1))
}
- modulesInDir := make(map[string][]string)
+ modulesInDir := make(map[string]Paths)
- ctx.VisitAllModules(func(module blueprint.Module) {
- if a, ok := module.(Module); ok {
- blueprintDir := a.base().blueprintDir
- installTarget := a.base().installTarget
- checkbuildTarget := a.base().checkbuildTarget
+ ctx.VisitAllModules(func(module Module) {
+ blueprintDir := module.base().blueprintDir
+ installTarget := module.base().installTarget
+ checkbuildTarget := module.base().checkbuildTarget
- if checkbuildTarget != "" {
- checkbuildDeps = append(checkbuildDeps, checkbuildTarget)
- modulesInDir[blueprintDir] = append(modulesInDir[blueprintDir], checkbuildTarget)
- }
+ if checkbuildTarget != nil {
+ checkbuildDeps = append(checkbuildDeps, checkbuildTarget)
+ modulesInDir[blueprintDir] = append(modulesInDir[blueprintDir], checkbuildTarget)
+ }
- if installTarget != "" {
- modulesInDir[blueprintDir] = append(modulesInDir[blueprintDir], installTarget)
- }
+ if installTarget != nil {
+ modulesInDir[blueprintDir] = append(modulesInDir[blueprintDir], installTarget)
}
})
suffix := ""
- if ctx.Config().(Config).EmbeddedInMake() {
+ if ctx.Config().EmbeddedInMake() {
suffix = "-soong"
}
// Create a top-level checkbuild target that depends on all modules
- ctx.Build(pctx, blueprint.BuildParams{
+ ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
- Outputs: []string{"checkbuild" + suffix},
+ Output: PathForPhony(ctx, "checkbuild"+suffix),
Implicits: checkbuildDeps,
- Optional: true,
})
+ // Make will generate the MODULES-IN-* targets
+ if ctx.Config().EmbeddedInMake() {
+ return
+ }
+
+ sortedKeys := func(m map[string]Paths) []string {
+ s := make([]string, 0, len(m))
+ for k := range m {
+ s = append(s, k)
+ }
+ sort.Strings(s)
+ return s
+ }
+
// Ensure ancestor directories are in modulesInDir
dirs := sortedKeys(modulesInDir)
for _, dir := range dirs {
@@ -974,16 +1198,61 @@
}
}
- // Create a mm/<directory> target that depends on all modules in a directory, and depends
- // on the mm/* targets of all of its subdirectories that contain Android.bp files.
+ // Create a MODULES-IN-<directory> target that depends on all modules in a directory, and
+ // depends on the MODULES-IN-* targets of all of its subdirectories that contain Android.bp
+ // files.
for _, dir := range dirs {
- ctx.Build(pctx, blueprint.BuildParams{
+ ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
- Outputs: []string{mmTarget(dir)},
+ Output: mmTarget(dir),
Implicits: modulesInDir[dir],
// HACK: checkbuild should be an optional build, but force it
// enabled for now in standalone builds
- Optional: ctx.Config().(Config).EmbeddedInMake(),
+ Default: !ctx.Config().EmbeddedInMake(),
+ })
+ }
+
+ // Create (host|host-cross|target)-<OS> phony rules to build a reduced checkbuild.
+ osDeps := map[OsType]Paths{}
+ ctx.VisitAllModules(func(module Module) {
+ if module.Enabled() {
+ os := module.Target().Os
+ osDeps[os] = append(osDeps[os], module.base().checkbuildFiles...)
+ }
+ })
+
+ osClass := make(map[string]Paths)
+ for os, deps := range osDeps {
+ var className string
+
+ switch os.Class {
+ case Host:
+ className = "host"
+ case HostCross:
+ className = "host-cross"
+ case Device:
+ className = "target"
+ default:
+ continue
+ }
+
+ name := PathForPhony(ctx, className+"-"+os.Name)
+ osClass[className] = append(osClass[className], name)
+
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Output: name,
+ Implicits: deps,
+ })
+ }
+
+ // Wrap those into host|host-cross|target phony rules
+ osClasses := sortedKeys(osClass)
+ for _, class := range osClasses {
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Output: PathForPhony(ctx, class),
+ Implicits: osClass[class],
})
}
}
diff --git a/android/mutator.go b/android/mutator.go
index c8f3e8f..876d161 100644
--- a/android/mutator.go
+++ b/android/mutator.go
@@ -16,14 +16,16 @@
import (
"github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
)
-// Mutator phases:
-// Pre-arch
-// Arch
-// Pre-deps
-// Deps
-// PostDeps
+// Phases:
+// run Pre-arch mutators
+// run archMutator
+// run Pre-deps mutators
+// run depsMutator
+// run PostDeps mutators
+// continue on to GenerateAndroidBuildActions
func registerMutatorsToContext(ctx *blueprint.Context, mutators []*mutator) {
for _, t := range mutators {
@@ -74,19 +76,22 @@
func(ctx RegisterMutatorsContext) {
ctx.TopDown("load_hooks", loadHookMutator).Parallel()
},
- registerPrebuiltsPreArchMutators,
+ RegisterPrebuiltsPreArchMutators,
RegisterDefaultsPreArchMutators,
}
+func registerArchMutator(ctx RegisterMutatorsContext) {
+ ctx.BottomUp("arch", archMutator).Parallel()
+ ctx.TopDown("arch_hooks", archHookMutator).Parallel()
+}
+
var preDeps = []RegisterMutatorFunc{
- func(ctx RegisterMutatorsContext) {
- ctx.BottomUp("arch", archMutator).Parallel()
- ctx.TopDown("arch_hooks", archHookMutator).Parallel()
- },
+ RegisterNamespaceMutator,
+ registerArchMutator,
}
var postDeps = []RegisterMutatorFunc{
- registerPrebuiltsPostDepsMutators,
+ RegisterPrebuiltsPostDepsMutators,
}
func PreArchMutators(f RegisterMutatorFunc) {
@@ -104,8 +109,27 @@
type AndroidTopDownMutator func(TopDownMutatorContext)
type TopDownMutatorContext interface {
- blueprint.TopDownMutatorContext
+ BaseModuleContext
androidBaseContext
+
+ OtherModuleExists(name string) bool
+ Rename(name string)
+ Module() Module
+
+ OtherModuleName(m blueprint.Module) string
+ OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{})
+ OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag
+
+ CreateModule(blueprint.ModuleFactory, ...interface{})
+
+ GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module
+ GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag)
+
+ VisitDirectDeps(visit func(Module))
+ VisitDirectDepsIf(pred func(Module) bool, visit func(Module))
+ VisitDepsDepthFirst(visit func(Module))
+ VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module))
+ WalkDeps(visit func(Module, Module) bool)
}
type androidTopDownMutatorContext struct {
@@ -116,8 +140,22 @@
type AndroidBottomUpMutator func(BottomUpMutatorContext)
type BottomUpMutatorContext interface {
- blueprint.BottomUpMutatorContext
+ BaseModuleContext
androidBaseContext
+
+ OtherModuleExists(name string) bool
+ Rename(name string)
+ Module() blueprint.Module
+
+ AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string)
+ AddReverseDependency(module blueprint.Module, tag blueprint.DependencyTag, name string)
+ CreateVariations(...string) []blueprint.Module
+ CreateLocalVariations(...string) []blueprint.Module
+ SetDependencyVariation(string)
+ AddVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string)
+ AddFarVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string)
+ AddInterVariantDependency(tag blueprint.DependencyTag, from, to blueprint.Module)
+ ReplaceDependencies(string)
}
type androidBottomUpMutatorContext struct {
@@ -169,3 +207,104 @@
m.DepsMutator(ctx)
}
}
+
+func (a *androidTopDownMutatorContext) Config() Config {
+ return a.config
+}
+
+func (a *androidBottomUpMutatorContext) Config() Config {
+ return a.config
+}
+
+func (a *androidTopDownMutatorContext) Module() Module {
+ module, _ := a.TopDownMutatorContext.Module().(Module)
+ return module
+}
+
+func (a *androidTopDownMutatorContext) VisitDirectDeps(visit func(Module)) {
+ a.TopDownMutatorContext.VisitDirectDeps(func(module blueprint.Module) {
+ if aModule, _ := module.(Module); aModule != nil {
+ visit(aModule)
+ }
+ })
+}
+
+func (a *androidTopDownMutatorContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) {
+ a.TopDownMutatorContext.VisitDirectDepsIf(
+ // pred
+ func(module blueprint.Module) bool {
+ if aModule, _ := module.(Module); aModule != nil {
+ return pred(aModule)
+ } else {
+ return false
+ }
+ },
+ // visit
+ func(module blueprint.Module) {
+ visit(module.(Module))
+ })
+}
+
+func (a *androidTopDownMutatorContext) VisitDepsDepthFirst(visit func(Module)) {
+ a.TopDownMutatorContext.VisitDepsDepthFirst(func(module blueprint.Module) {
+ if aModule, _ := module.(Module); aModule != nil {
+ visit(aModule)
+ }
+ })
+}
+
+func (a *androidTopDownMutatorContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) {
+ a.TopDownMutatorContext.VisitDepsDepthFirstIf(
+ // pred
+ func(module blueprint.Module) bool {
+ if aModule, _ := module.(Module); aModule != nil {
+ return pred(aModule)
+ } else {
+ return false
+ }
+ },
+ // visit
+ func(module blueprint.Module) {
+ visit(module.(Module))
+ })
+}
+
+func (a *androidTopDownMutatorContext) WalkDeps(visit func(Module, Module) bool) {
+ a.TopDownMutatorContext.WalkDeps(func(child, parent blueprint.Module) bool {
+ childAndroidModule, _ := child.(Module)
+ parentAndroidModule, _ := parent.(Module)
+ if childAndroidModule != nil && parentAndroidModule != nil {
+ return visit(childAndroidModule, parentAndroidModule)
+ } else {
+ return false
+ }
+ })
+}
+
+func (a *androidTopDownMutatorContext) AppendProperties(props ...interface{}) {
+ for _, p := range props {
+ err := proptools.AppendMatchingProperties(a.Module().base().customizableProperties,
+ p, nil)
+ if err != nil {
+ if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok {
+ a.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error())
+ } else {
+ panic(err)
+ }
+ }
+ }
+}
+
+func (a *androidTopDownMutatorContext) PrependProperties(props ...interface{}) {
+ for _, p := range props {
+ err := proptools.PrependMatchingProperties(a.Module().base().customizableProperties,
+ p, nil)
+ if err != nil {
+ if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok {
+ a.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error())
+ } else {
+ panic(err)
+ }
+ }
+ }
+}
diff --git a/android/namespace.go b/android/namespace.go
new file mode 100644
index 0000000..1f8ef5a
--- /dev/null
+++ b/android/namespace.go
@@ -0,0 +1,423 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "errors"
+ "fmt"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+
+ "github.com/google/blueprint"
+)
+
+// This file implements namespaces
+const (
+ namespacePrefix = "//"
+ modulePrefix = ":"
+)
+
+func init() {
+ RegisterModuleType("soong_namespace", NamespaceFactory)
+}
+
+// threadsafe sorted list
+type sortedNamespaces struct {
+ lock sync.Mutex
+ items []*Namespace
+ sorted bool
+}
+
+func (s *sortedNamespaces) add(namespace *Namespace) {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+ if s.sorted {
+ panic("It is not supported to call sortedNamespaces.add() after sortedNamespaces.sortedItems()")
+ }
+ s.items = append(s.items, namespace)
+}
+
+func (s *sortedNamespaces) sortedItems() []*Namespace {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+ if !s.sorted {
+ less := func(i int, j int) bool {
+ return s.items[i].Path < s.items[j].Path
+ }
+ sort.Slice(s.items, less)
+ s.sorted = true
+ }
+ return s.items
+}
+
+func (s *sortedNamespaces) index(namespace *Namespace) int {
+ for i, candidate := range s.sortedItems() {
+ if namespace == candidate {
+ return i
+ }
+ }
+ return -1
+}
+
+// A NameResolver implements blueprint.NameInterface, and implements the logic to
+// find a module from namespaces based on a query string.
+// A query string can be a module name or can be be "//namespace_path:module_path"
+type NameResolver struct {
+ rootNamespace *Namespace
+
+ // id counter for atomic.AddInt32
+ nextNamespaceId int32
+
+ // All namespaces, without duplicates.
+ sortedNamespaces sortedNamespaces
+
+ // Map from dir to namespace. Will have duplicates if two dirs are part of the same namespace.
+ namespacesByDir sync.Map // if generics were supported, this would be sync.Map[string]*Namespace
+
+ // func telling whether to export a namespace to Kati
+ namespaceExportFilter func(*Namespace) bool
+}
+
+func NewNameResolver(namespaceExportFilter func(*Namespace) bool) *NameResolver {
+ namespacesByDir := sync.Map{}
+
+ r := &NameResolver{
+ namespacesByDir: namespacesByDir,
+ namespaceExportFilter: namespaceExportFilter,
+ }
+ r.rootNamespace = r.newNamespace(".")
+ r.rootNamespace.visibleNamespaces = []*Namespace{r.rootNamespace}
+ r.addNamespace(r.rootNamespace)
+
+ return r
+}
+
+func (r *NameResolver) newNamespace(path string) *Namespace {
+ namespace := NewNamespace(path)
+
+ namespace.exportToKati = r.namespaceExportFilter(namespace)
+
+ return namespace
+}
+
+func (r *NameResolver) addNewNamespaceForModule(module *NamespaceModule, path string) error {
+ fileName := filepath.Base(path)
+ if fileName != "Android.bp" {
+ return errors.New("A namespace may only be declared in a file named Android.bp")
+ }
+ dir := filepath.Dir(path)
+
+ namespace := r.newNamespace(dir)
+ module.namespace = namespace
+ module.resolver = r
+ namespace.importedNamespaceNames = module.properties.Imports
+ return r.addNamespace(namespace)
+}
+
+func (r *NameResolver) addNamespace(namespace *Namespace) (err error) {
+ existingNamespace, exists := r.namespaceAt(namespace.Path)
+ if exists {
+ if existingNamespace.Path == namespace.Path {
+ return fmt.Errorf("namespace %v already exists", namespace.Path)
+ } else {
+ // It would probably confuse readers if namespaces were declared anywhere but
+ // the top of the file, so we forbid declaring namespaces after anything else.
+ return fmt.Errorf("a namespace must be the first module in the file")
+ }
+ }
+ r.sortedNamespaces.add(namespace)
+
+ r.namespacesByDir.Store(namespace.Path, namespace)
+ return nil
+}
+
+// non-recursive check for namespace
+func (r *NameResolver) namespaceAt(path string) (namespace *Namespace, found bool) {
+ mapVal, found := r.namespacesByDir.Load(path)
+ if !found {
+ return nil, false
+ }
+ return mapVal.(*Namespace), true
+}
+
+// recursive search upward for a namespace
+func (r *NameResolver) findNamespace(path string) (namespace *Namespace) {
+ namespace, found := r.namespaceAt(path)
+ if found {
+ return namespace
+ }
+ parentDir := filepath.Dir(path)
+ if parentDir == path {
+ return nil
+ }
+ namespace = r.findNamespace(parentDir)
+ r.namespacesByDir.Store(path, namespace)
+ return namespace
+}
+
+func (r *NameResolver) NewModule(ctx blueprint.NamespaceContext, moduleGroup blueprint.ModuleGroup, module blueprint.Module) (namespace blueprint.Namespace, errs []error) {
+ // if this module is a namespace, then save it to our list of namespaces
+ newNamespace, ok := module.(*NamespaceModule)
+ if ok {
+ err := r.addNewNamespaceForModule(newNamespace, ctx.ModulePath())
+ if err != nil {
+ return nil, []error{err}
+ }
+ return nil, nil
+ }
+
+ // if this module is not a namespace, then save it into the appropriate namespace
+ ns := r.findNamespaceFromCtx(ctx)
+
+ _, errs = ns.moduleContainer.NewModule(ctx, moduleGroup, module)
+ if len(errs) > 0 {
+ return nil, errs
+ }
+
+ amod, ok := module.(Module)
+ if ok {
+ // inform the module whether its namespace is one that we want to export to Make
+ amod.base().commonProperties.NamespaceExportedToMake = ns.exportToKati
+ }
+
+ return ns, nil
+}
+
+func (r *NameResolver) AllModules() []blueprint.ModuleGroup {
+ childLists := [][]blueprint.ModuleGroup{}
+ totalCount := 0
+ for _, namespace := range r.sortedNamespaces.sortedItems() {
+ newModules := namespace.moduleContainer.AllModules()
+ totalCount += len(newModules)
+ childLists = append(childLists, newModules)
+ }
+
+ allModules := make([]blueprint.ModuleGroup, 0, totalCount)
+ for _, childList := range childLists {
+ allModules = append(allModules, childList...)
+ }
+ return allModules
+}
+
+// parses a fully-qualified path (like "//namespace_path:module_name") into a namespace name and a
+// module name
+func (r *NameResolver) parseFullyQualifiedName(name string) (namespaceName string, moduleName string, ok bool) {
+ if !strings.HasPrefix(name, namespacePrefix) {
+ return "", "", false
+ }
+ name = strings.TrimPrefix(name, namespacePrefix)
+ components := strings.Split(name, modulePrefix)
+ if len(components) != 2 {
+ return "", "", false
+ }
+ return components[0], components[1], true
+
+}
+
+func (r *NameResolver) getNamespacesToSearchForModule(sourceNamespace *Namespace) (searchOrder []*Namespace) {
+ return sourceNamespace.visibleNamespaces
+}
+
+func (r *NameResolver) ModuleFromName(name string, namespace blueprint.Namespace) (group blueprint.ModuleGroup, found bool) {
+ // handle fully qualified references like "//namespace_path:module_name"
+ nsName, moduleName, isAbs := r.parseFullyQualifiedName(name)
+ if isAbs {
+ namespace, found := r.namespaceAt(nsName)
+ if !found {
+ return blueprint.ModuleGroup{}, false
+ }
+ container := namespace.moduleContainer
+ return container.ModuleFromName(moduleName, nil)
+ }
+ for _, candidate := range r.getNamespacesToSearchForModule(namespace.(*Namespace)) {
+ group, found = candidate.moduleContainer.ModuleFromName(name, nil)
+ if found {
+ return group, true
+ }
+ }
+ return blueprint.ModuleGroup{}, false
+
+}
+
+func (r *NameResolver) Rename(oldName string, newName string, namespace blueprint.Namespace) []error {
+ oldNs := r.findNamespace(oldName)
+ newNs := r.findNamespace(newName)
+ if oldNs != newNs {
+ return []error{fmt.Errorf("cannot rename %v to %v because the destination is outside namespace %v", oldName, newName, oldNs.Path)}
+ }
+
+ oldName, err := filepath.Rel(oldNs.Path, oldName)
+ if err != nil {
+ panic(err)
+ }
+ newName, err = filepath.Rel(newNs.Path, newName)
+ if err != nil {
+ panic(err)
+ }
+
+ return oldNs.moduleContainer.Rename(oldName, newName, nil)
+}
+
+// resolve each element of namespace.importedNamespaceNames and put the result in namespace.visibleNamespaces
+func (r *NameResolver) FindNamespaceImports(namespace *Namespace) (err error) {
+ namespace.visibleNamespaces = make([]*Namespace, 0, 2+len(namespace.importedNamespaceNames))
+ // search itself first
+ namespace.visibleNamespaces = append(namespace.visibleNamespaces, namespace)
+ // search its imports next
+ for _, name := range namespace.importedNamespaceNames {
+ imp, ok := r.namespaceAt(name)
+ if !ok {
+ return fmt.Errorf("namespace %v does not exist", name)
+ }
+ namespace.visibleNamespaces = append(namespace.visibleNamespaces, imp)
+ }
+ // search the root namespace last
+ namespace.visibleNamespaces = append(namespace.visibleNamespaces, r.rootNamespace)
+ return nil
+}
+
+func (r *NameResolver) chooseId(namespace *Namespace) {
+ id := r.sortedNamespaces.index(namespace)
+ if id < 0 {
+ panic(fmt.Sprintf("Namespace not found: %v\n", namespace.id))
+ }
+ namespace.id = strconv.Itoa(id)
+}
+
+func (r *NameResolver) MissingDependencyError(depender string, dependerNamespace blueprint.Namespace, depName string) (err error) {
+ text := fmt.Sprintf("%q depends on undefined module %q", depender, depName)
+
+ _, _, isAbs := r.parseFullyQualifiedName(depName)
+ if isAbs {
+ // if the user gave a fully-qualified name, we don't need to look for other
+ // modules that they might have been referring to
+ return fmt.Errorf(text)
+ }
+
+ // determine which namespaces the module can be found in
+ foundInNamespaces := []string{}
+ for _, namespace := range r.sortedNamespaces.sortedItems() {
+ _, found := namespace.moduleContainer.ModuleFromName(depName, nil)
+ if found {
+ foundInNamespaces = append(foundInNamespaces, namespace.Path)
+ }
+ }
+ if len(foundInNamespaces) > 0 {
+ // determine which namespaces are visible to dependerNamespace
+ dependerNs := dependerNamespace.(*Namespace)
+ searched := r.getNamespacesToSearchForModule(dependerNs)
+ importedNames := []string{}
+ for _, ns := range searched {
+ importedNames = append(importedNames, ns.Path)
+ }
+ text += fmt.Sprintf("\nModule %q is defined in namespace %q which can read these %v namespaces: %q", depender, dependerNs.Path, len(importedNames), importedNames)
+ text += fmt.Sprintf("\nModule %q can be found in these namespaces: %q", depName, foundInNamespaces)
+ }
+
+ return fmt.Errorf(text)
+}
+
+func (r *NameResolver) GetNamespace(ctx blueprint.NamespaceContext) blueprint.Namespace {
+ return r.findNamespaceFromCtx(ctx)
+}
+
+func (r *NameResolver) findNamespaceFromCtx(ctx blueprint.NamespaceContext) *Namespace {
+ return r.findNamespace(filepath.Dir(ctx.ModulePath()))
+}
+
+func (r *NameResolver) UniqueName(ctx blueprint.NamespaceContext, name string) (unique string) {
+ prefix := r.findNamespaceFromCtx(ctx).id
+ if prefix != "" {
+ prefix = prefix + "-"
+ }
+ return prefix + name
+}
+
+var _ blueprint.NameInterface = (*NameResolver)(nil)
+
+type Namespace struct {
+ blueprint.NamespaceMarker
+ Path string
+
+ // names of namespaces listed as imports by this namespace
+ importedNamespaceNames []string
+ // all namespaces that should be searched when a module in this namespace declares a dependency
+ visibleNamespaces []*Namespace
+
+ id string
+
+ exportToKati bool
+
+ moduleContainer blueprint.NameInterface
+}
+
+func NewNamespace(path string) *Namespace {
+ return &Namespace{Path: path, moduleContainer: blueprint.NewSimpleNameInterface()}
+}
+
+var _ blueprint.Namespace = (*Namespace)(nil)
+
+type NamespaceModule struct {
+ ModuleBase
+
+ namespace *Namespace
+ resolver *NameResolver
+
+ properties struct {
+ Imports []string
+ }
+}
+
+func (n *NamespaceModule) DepsMutator(context BottomUpMutatorContext) {
+}
+
+func (n *NamespaceModule) GenerateAndroidBuildActions(ctx ModuleContext) {
+}
+
+func (n *NamespaceModule) GenerateBuildActions(ctx blueprint.ModuleContext) {
+}
+
+func (n *NamespaceModule) Name() (name string) {
+ return *n.nameProperties.Name
+}
+
+func NamespaceFactory() Module {
+ module := &NamespaceModule{}
+
+ name := "soong_namespace"
+ module.nameProperties.Name = &name
+
+ module.AddProperties(&module.properties)
+ return module
+}
+
+func RegisterNamespaceMutator(ctx RegisterMutatorsContext) {
+ ctx.BottomUp("namespace_deps", namespaceMutator).Parallel()
+}
+
+func namespaceMutator(ctx BottomUpMutatorContext) {
+ module, ok := ctx.Module().(*NamespaceModule)
+ if ok {
+ err := module.resolver.FindNamespaceImports(module.namespace)
+ if err != nil {
+ ctx.ModuleErrorf(err.Error())
+ }
+
+ module.resolver.chooseId(module.namespace)
+ }
+}
diff --git a/android/namespace_test.go b/android/namespace_test.go
new file mode 100644
index 0000000..9ab186b
--- /dev/null
+++ b/android/namespace_test.go
@@ -0,0 +1,703 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "errors"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "github.com/google/blueprint"
+)
+
+func TestDependingOnModuleInSameNamespace(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ a := getModule(ctx, "a")
+ b := getModule(ctx, "b")
+ if !dependsOn(ctx, b, a) {
+ t.Errorf("module b does not depend on module a in the same namespace")
+ }
+}
+
+func TestDependingOnModuleInRootNamespace(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ ".": `
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ },
+ )
+
+ a := getModule(ctx, "a")
+ b := getModule(ctx, "b")
+ if !dependsOn(ctx, b, a) {
+ t.Errorf("module b in root namespace does not depend on module a in the root namespace")
+ }
+}
+
+func TestImplicitlyImportRootNamespace(t *testing.T) {
+ _ = setupTest(t,
+ map[string]string{
+ ".": `
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ // setupTest will report any errors
+}
+
+func TestDependingOnModuleInImportedNamespace(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ imports: ["dir1"],
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ a := getModule(ctx, "a")
+ b := getModule(ctx, "b")
+ if !dependsOn(ctx, b, a) {
+ t.Errorf("module b does not depend on module a in the same namespace")
+ }
+}
+
+func TestDependingOnModuleInNonImportedNamespace(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir3": `
+ soong_namespace {
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(
+ `dir3/Android.bp:4:4: "b" depends on undefined module "a"
+Module "b" is defined in namespace "dir3" which can read these 2 namespaces: ["dir3" "."]
+Module "a" can be found in these namespaces: ["dir1" "dir2"]`),
+ }
+
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestDependingOnModuleByFullyQualifiedReference(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ }
+ test_module {
+ name: "b",
+ deps: ["//dir1:a"],
+ }
+ `,
+ },
+ )
+ a := getModule(ctx, "a")
+ b := getModule(ctx, "b")
+ if !dependsOn(ctx, b, a) {
+ t.Errorf("module b does not depend on module a")
+ }
+}
+
+func TestSameNameInTwoNamespaces(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ id: "1",
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ id: "2",
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ id:"3",
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ id:"4",
+ }
+ `,
+ },
+ )
+
+ one := findModuleById(ctx, "1")
+ two := findModuleById(ctx, "2")
+ three := findModuleById(ctx, "3")
+ four := findModuleById(ctx, "4")
+ if !dependsOn(ctx, two, one) {
+ t.Fatalf("Module 2 does not depend on module 1 in its namespace")
+ }
+ if dependsOn(ctx, two, three) {
+ t.Fatalf("Module 2 depends on module 3 in another namespace")
+ }
+ if !dependsOn(ctx, four, three) {
+ t.Fatalf("Module 4 does not depend on module 3 in its namespace")
+ }
+ if dependsOn(ctx, four, one) {
+ t.Fatalf("Module 4 depends on module 1 in another namespace")
+ }
+}
+
+func TestSearchOrder(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ id: "1",
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ id:"2",
+ }
+ test_module {
+ name: "b",
+ id:"3",
+ }
+ `,
+ "dir3": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ id:"4",
+ }
+ test_module {
+ name: "b",
+ id:"5",
+ }
+ test_module {
+ name: "c",
+ id:"6",
+ }
+ `,
+ ".": `
+ test_module {
+ name: "a",
+ id: "7",
+ }
+ test_module {
+ name: "b",
+ id: "8",
+ }
+ test_module {
+ name: "c",
+ id: "9",
+ }
+ test_module {
+ name: "d",
+ id: "10",
+ }
+ `,
+ "dir4": `
+ soong_namespace {
+ imports: ["dir1", "dir2", "dir3"]
+ }
+ test_module {
+ name: "test_me",
+ id:"0",
+ deps: ["a", "b", "c", "d"],
+ }
+ `,
+ },
+ )
+
+ testMe := findModuleById(ctx, "0")
+ if !dependsOn(ctx, testMe, findModuleById(ctx, "1")) {
+ t.Errorf("test_me doesn't depend on id 1")
+ }
+ if !dependsOn(ctx, testMe, findModuleById(ctx, "3")) {
+ t.Errorf("test_me doesn't depend on id 3")
+ }
+ if !dependsOn(ctx, testMe, findModuleById(ctx, "6")) {
+ t.Errorf("test_me doesn't depend on id 6")
+ }
+ if !dependsOn(ctx, testMe, findModuleById(ctx, "10")) {
+ t.Errorf("test_me doesn't depend on id 10")
+ }
+ if numDeps(ctx, testMe) != 4 {
+ t.Errorf("num dependencies of test_me = %v, not 4\n", numDeps(ctx, testMe))
+ }
+}
+
+func TestTwoNamespacesCanImportEachOther(t *testing.T) {
+ _ = setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ imports: ["dir2"]
+ }
+ test_module {
+ name: "a",
+ }
+ test_module {
+ name: "c",
+ deps: ["b"],
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ imports: ["dir1"],
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ // setupTest will report any errors
+}
+
+func TestImportingNonexistentNamespace(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ imports: ["a_nonexistent_namespace"]
+ }
+ test_module {
+ name: "a",
+ deps: ["a_nonexistent_module"]
+ }
+ `,
+ },
+ )
+
+ // should complain about the missing namespace and not complain about the unresolvable dependency
+ expectedErrors := []error{
+ errors.New(`dir1/Android.bp:2:4: module "soong_namespace": namespace a_nonexistent_namespace does not exist`),
+ }
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestNamespacesDontInheritParentNamespaces(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir1/subdir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(`dir1/subdir1/Android.bp:4:4: "b" depends on undefined module "a"
+Module "b" is defined in namespace "dir1/subdir1" which can read these 2 namespaces: ["dir1/subdir1" "."]
+Module "a" can be found in these namespaces: ["dir1"]`),
+ }
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestModulesDoReceiveParentNamespace(t *testing.T) {
+ _ = setupTest(t,
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir1/subdir": `
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ // setupTest will report any errors
+}
+
+func TestNamespaceImportsNotTransitive(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a",
+ }
+ `,
+ "dir2": `
+ soong_namespace {
+ imports: ["dir1"],
+ }
+ test_module {
+ name: "b",
+ deps: ["a"],
+ }
+ `,
+ "dir3": `
+ soong_namespace {
+ imports: ["dir2"],
+ }
+ test_module {
+ name: "c",
+ deps: ["a"],
+ }
+ `,
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(`dir3/Android.bp:5:4: "c" depends on undefined module "a"
+Module "c" is defined in namespace "dir3" which can read these 3 namespaces: ["dir3" "dir2" "."]
+Module "a" can be found in these namespaces: ["dir1"]`),
+ }
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestTwoNamepacesInSameDir(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ soong_namespace {
+ }
+ `,
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(`dir1/Android.bp:4:4: namespace dir1 already exists`),
+ }
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestNamespaceNotAtTopOfFile(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ test_module {
+ name: "a"
+ }
+ soong_namespace {
+ }
+ `,
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(`dir1/Android.bp:5:4: a namespace must be the first module in the file`),
+ }
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestTwoModulesWithSameNameInSameNamespace(t *testing.T) {
+ _, errs := setupTestExpectErrs(
+ map[string]string{
+ "dir1": `
+ soong_namespace {
+ }
+ test_module {
+ name: "a"
+ }
+ test_module {
+ name: "a"
+ }
+ `,
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(`dir1/Android.bp:7:4: module "a" already defined
+ dir1/Android.bp:4:4 <-- previous definition here`),
+ }
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+func TestDeclaringNamespaceInNonAndroidBpFile(t *testing.T) {
+ _, errs := setupTestFromFiles(
+ map[string][]byte{
+ "Android.bp": []byte(`
+ build = ["include.bp"]
+ `),
+ "include.bp": []byte(`
+ soong_namespace {
+ }
+ `),
+ },
+ )
+
+ expectedErrors := []error{
+ errors.New(`include.bp:2:5: A namespace may only be declared in a file named Android.bp`),
+ }
+
+ if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() {
+ t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs)
+ }
+}
+
+// so that the generated .ninja file will have consistent names
+func TestConsistentNamespaceNames(t *testing.T) {
+ ctx := setupTest(t,
+ map[string]string{
+ "dir1": "soong_namespace{}",
+ "dir2": "soong_namespace{}",
+ "dir3": "soong_namespace{}",
+ })
+
+ ns1, _ := ctx.NameResolver.namespaceAt("dir1")
+ ns2, _ := ctx.NameResolver.namespaceAt("dir2")
+ ns3, _ := ctx.NameResolver.namespaceAt("dir3")
+ actualIds := []string{ns1.id, ns2.id, ns3.id}
+ expectedIds := []string{"1", "2", "3"}
+ if !reflect.DeepEqual(actualIds, expectedIds) {
+ t.Errorf("Incorrect namespace ids.\nactual: %s\nexpected: %s\n", actualIds, expectedIds)
+ }
+}
+
+// some utils to support the tests
+
+func mockFiles(bps map[string]string) (files map[string][]byte) {
+ files = make(map[string][]byte, len(bps))
+ files["Android.bp"] = []byte("")
+ for dir, text := range bps {
+ files[filepath.Join(dir, "Android.bp")] = []byte(text)
+ }
+ return files
+}
+
+func setupTestFromFiles(bps map[string][]byte) (ctx *TestContext, errs []error) {
+ buildDir, err := ioutil.TempDir("", "soong_namespace_test")
+ if err != nil {
+ return nil, []error{err}
+ }
+ defer os.RemoveAll(buildDir)
+
+ config := TestConfig(buildDir, nil)
+
+ ctx = NewTestContext()
+ ctx.MockFileSystem(bps)
+ ctx.RegisterModuleType("test_module", ModuleFactoryAdaptor(newTestModule))
+ ctx.RegisterModuleType("soong_namespace", ModuleFactoryAdaptor(NamespaceFactory))
+ ctx.PreDepsMutators(RegisterNamespaceMutator)
+ ctx.Register()
+
+ _, errs = ctx.ParseBlueprintsFiles("Android.bp")
+ if len(errs) > 0 {
+ return ctx, errs
+ }
+ _, errs = ctx.PrepareBuildActions(config)
+ return ctx, errs
+}
+
+func setupTestExpectErrs(bps map[string]string) (ctx *TestContext, errs []error) {
+ files := make(map[string][]byte, len(bps))
+ files["Android.bp"] = []byte("")
+ for dir, text := range bps {
+ files[filepath.Join(dir, "Android.bp")] = []byte(text)
+ }
+ return setupTestFromFiles(files)
+}
+
+func setupTest(t *testing.T, bps map[string]string) (ctx *TestContext) {
+ ctx, errs := setupTestExpectErrs(bps)
+ failIfErrored(t, errs)
+ return ctx
+}
+
+func dependsOn(ctx *TestContext, module TestingModule, possibleDependency TestingModule) bool {
+ depends := false
+ visit := func(dependency blueprint.Module) {
+ if dependency == possibleDependency.module {
+ depends = true
+ }
+ }
+ ctx.VisitDirectDeps(module.module, visit)
+ return depends
+}
+
+func numDeps(ctx *TestContext, module TestingModule) int {
+ count := 0
+ visit := func(dependency blueprint.Module) {
+ count++
+ }
+ ctx.VisitDirectDeps(module.module, visit)
+ return count
+}
+
+func getModule(ctx *TestContext, moduleName string) TestingModule {
+ return ctx.ModuleForTests(moduleName, "")
+}
+
+func findModuleById(ctx *TestContext, id string) (module TestingModule) {
+ visit := func(candidate blueprint.Module) {
+ testModule, ok := candidate.(*testModule)
+ if ok {
+ if testModule.properties.Id == id {
+ module = TestingModule{testModule}
+ }
+ }
+ }
+ ctx.VisitAllModules(visit)
+ return module
+}
+
+type testModule struct {
+ ModuleBase
+ properties struct {
+ Deps []string
+ Id string
+ }
+}
+
+func (m *testModule) DepsMutator(ctx BottomUpMutatorContext) {
+ for _, d := range m.properties.Deps {
+ ctx.AddDependency(ctx.Module(), nil, d)
+ }
+}
+
+func (m *testModule) GenerateAndroidBuildActions(ModuleContext) {
+}
+
+func newTestModule() Module {
+ m := &testModule{}
+ m.AddProperties(&m.properties)
+ InitAndroidModule(m)
+ return m
+}
+
+func failIfErrored(t *testing.T, errs []error) {
+ if len(errs) > 0 {
+ for _, err := range errs {
+ t.Error(err)
+ }
+ t.FailNow()
+ }
+}
diff --git a/android/onceper.go b/android/onceper.go
index 5f7a310..f19f75c 100644
--- a/android/onceper.go
+++ b/android/onceper.go
@@ -15,12 +15,12 @@
package android
import (
+ "fmt"
"sync"
- "sync/atomic"
)
type OncePer struct {
- values atomic.Value
+ values sync.Map
valuesLock sync.Mutex
}
@@ -29,33 +29,32 @@
// Once computes a value the first time it is called with a given key per OncePer, and returns the
// value without recomputing when called with the same key. key must be hashable.
func (once *OncePer) Once(key interface{}, value func() interface{}) interface{} {
- // Atomically load the map without locking. If this is the first call Load() will return nil
- // and the type assertion will fail, leaving a nil map in m, but that's OK since m is only used
- // for reads.
- m, _ := once.values.Load().(valueMap)
- if v, ok := m[key]; ok {
+ // Fast path: check if the key is already in the map
+ if v, ok := once.values.Load(key); ok {
return v
}
+ // Slow path: lock so that we don't call the value function twice concurrently
once.valuesLock.Lock()
defer once.valuesLock.Unlock()
// Check again with the lock held
- m, _ = once.values.Load().(valueMap)
- if v, ok := m[key]; ok {
+ if v, ok := once.values.Load(key); ok {
return v
}
- // Copy the existing map
- newMap := make(valueMap, len(m))
- for k, v := range m {
- newMap[k] = v
- }
-
+ // Still not in the map, call the value function and store it
v := value()
+ once.values.Store(key, v)
- newMap[key] = v
- once.values.Store(newMap)
+ return v
+}
+
+func (once *OncePer) Get(key interface{}) interface{} {
+ v, ok := once.values.Load(key)
+ if !ok {
+ panic(fmt.Errorf("Get() called before Once()"))
+ }
return v
}
diff --git a/android/package_ctx.go b/android/package_ctx.go
index 5304403..0dbcea5 100644
--- a/android/package_ctx.go
+++ b/android/package_ctx.go
@@ -16,19 +16,21 @@
import (
"fmt"
+ "runtime"
+ "strings"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
)
-// AndroidPackageContext is a wrapper for blueprint.PackageContext that adds
+// PackageContext is a wrapper for blueprint.PackageContext that adds
// some android-specific helper functions.
-type AndroidPackageContext struct {
+type PackageContext struct {
blueprint.PackageContext
}
-func NewPackageContext(pkgPath string) AndroidPackageContext {
- return AndroidPackageContext{blueprint.NewPackageContext(pkgPath)}
+func NewPackageContext(pkgPath string) PackageContext {
+ return PackageContext{blueprint.NewPackageContext(pkgPath)}
}
// configErrorWrapper can be used with Path functions when a Context is not
@@ -38,7 +40,7 @@
// The most common use here will be with VariableFunc, where only a config is
// provided, and an error should be returned.
type configErrorWrapper struct {
- pctx AndroidPackageContext
+ pctx PackageContext
config Config
errors []error
}
@@ -46,7 +48,7 @@
var _ PathContext = &configErrorWrapper{}
var _ errorfContext = &configErrorWrapper{}
-func (e *configErrorWrapper) Config() interface{} {
+func (e *configErrorWrapper) Config() Config {
return e.config
}
func (e *configErrorWrapper) Errorf(format string, args ...interface{}) {
@@ -60,13 +62,43 @@
return nil
}
+// VariableFunc wraps blueprint.PackageContext.VariableFunc, converting the interface{} config
+// argument to a Config.
+func (p PackageContext) VariableFunc(name string,
+ f func(Config) (string, error)) blueprint.Variable {
+
+ return p.PackageContext.VariableFunc(name, func(config interface{}) (string, error) {
+ return f(config.(Config))
+ })
+}
+
+// PoolFunc wraps blueprint.PackageContext.PoolFunc, converting the interface{} config
+// argument to a Config.
+func (p PackageContext) PoolFunc(name string,
+ f func(Config) (blueprint.PoolParams, error)) blueprint.Pool {
+
+ return p.PackageContext.PoolFunc(name, func(config interface{}) (blueprint.PoolParams, error) {
+ return f(config.(Config))
+ })
+}
+
+// RuleFunc wraps blueprint.PackageContext.RuleFunc, converting the interface{} config
+// argument to a Config.
+func (p PackageContext) RuleFunc(name string,
+ f func(Config) (blueprint.RuleParams, error), argNames ...string) blueprint.Rule {
+
+ return p.PackageContext.RuleFunc(name, func(config interface{}) (blueprint.RuleParams, error) {
+ return f(config.(Config))
+ }, argNames...)
+}
+
// SourcePathVariable returns a Variable whose value is the source directory
// appended with the supplied path. It may only be called during a Go package's
// initialization - either from the init() function or as part of a
// package-scoped variable's initialization.
-func (p AndroidPackageContext) SourcePathVariable(name, path string) blueprint.Variable {
- return p.VariableFunc(name, func(config interface{}) (string, error) {
- ctx := &configErrorWrapper{p, config.(Config), []error{}}
+func (p PackageContext) SourcePathVariable(name, path string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
p := safePathForSource(ctx, path)
if len(ctx.errors) > 0 {
return "", ctx.errors[0]
@@ -75,28 +107,98 @@
})
}
-// HostBinVariable returns a Variable whose value is the path to a host tool
-// in the bin directory for host targets. It may only be called during a Go
-// package's initialization - either from the init() function or as part of a
-// package-scoped variable's initialization.
-func (p AndroidPackageContext) HostBinToolVariable(name, path string) blueprint.Variable {
- return p.VariableFunc(name, func(config interface{}) (string, error) {
- ctx := &configErrorWrapper{p, config.(Config), []error{}}
- p := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "bin", path)
+// SourcePathsVariable returns a Variable whose value is the source directory
+// appended with the supplied paths, joined with separator. It may only be
+// called during a Go package's initialization - either from the init()
+// function or as part of a package-scoped variable's initialization.
+func (p PackageContext) SourcePathsVariable(name, separator string, paths ...string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
+ var ret []string
+ for _, path := range paths {
+ p := safePathForSource(ctx, path)
+ if len(ctx.errors) > 0 {
+ return "", ctx.errors[0]
+ }
+ ret = append(ret, p.String())
+ }
+ return strings.Join(ret, separator), nil
+ })
+}
+
+// SourcePathVariableWithEnvOverride returns a Variable whose value is the source directory
+// appended with the supplied path, or the value of the given environment variable if it is set.
+// The environment variable is not required to point to a path inside the source tree.
+// It may only be called during a Go package's initialization - either from the init() function or
+// as part of a package-scoped variable's initialization.
+func (p PackageContext) SourcePathVariableWithEnvOverride(name, path, env string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
+ p := safePathForSource(ctx, path)
if len(ctx.errors) > 0 {
return "", ctx.errors[0]
}
- return p.String(), nil
+ return config.GetenvWithDefault(env, p.String()), nil
})
}
+// HostBinToolVariable returns a Variable whose value is the path to a host tool
+// in the bin directory for host targets. It may only be called during a Go
+// package's initialization - either from the init() function or as part of a
+// package-scoped variable's initialization.
+func (p PackageContext) HostBinToolVariable(name, path string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ po, err := p.HostBinToolPath(config, path)
+ if err != nil {
+ return "", err
+ }
+ return po.String(), nil
+ })
+}
+
+func (p PackageContext) HostBinToolPath(config Config, path string) (Path, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
+ pa := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "bin", path)
+ if len(ctx.errors) > 0 {
+ return nil, ctx.errors[0]
+ }
+ return pa, nil
+}
+
+// HostJNIToolVariable returns a Variable whose value is the path to a host tool
+// in the lib directory for host targets. It may only be called during a Go
+// package's initialization - either from the init() function or as part of a
+// package-scoped variable's initialization.
+func (p PackageContext) HostJNIToolVariable(name, path string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ po, err := p.HostJNIToolPath(config, path)
+ if err != nil {
+ return "", err
+ }
+ return po.String(), nil
+ })
+}
+
+func (p PackageContext) HostJNIToolPath(config Config, path string) (Path, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
+ ext := ".so"
+ if runtime.GOOS == "darwin" {
+ ext = ".dylib"
+ }
+ pa := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "lib64", path+ext)
+ if len(ctx.errors) > 0 {
+ return nil, ctx.errors[0]
+ }
+ return pa, nil
+}
+
// HostJavaToolVariable returns a Variable whose value is the path to a host
// tool in the frameworks directory for host targets. It may only be called
// during a Go package's initialization - either from the init() function or as
// part of a package-scoped variable's initialization.
-func (p AndroidPackageContext) HostJavaToolVariable(name, path string) blueprint.Variable {
- return p.VariableFunc(name, func(config interface{}) (string, error) {
- ctx := &configErrorWrapper{p, config.(Config), []error{}}
+func (p PackageContext) HostJavaToolVariable(name, path string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
p := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "framework", path)
if len(ctx.errors) > 0 {
return "", ctx.errors[0]
@@ -105,13 +207,22 @@
})
}
+func (p PackageContext) HostJavaToolPath(config Config, path string) (Path, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
+ pa := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "framework", path)
+ if len(ctx.errors) > 0 {
+ return nil, ctx.errors[0]
+ }
+ return pa, nil
+}
+
// IntermediatesPathVariable returns a Variable whose value is the intermediate
// directory appended with the supplied path. It may only be called during a Go
// package's initialization - either from the init() function or as part of a
// package-scoped variable's initialization.
-func (p AndroidPackageContext) IntermediatesPathVariable(name, path string) blueprint.Variable {
- return p.VariableFunc(name, func(config interface{}) (string, error) {
- ctx := &configErrorWrapper{p, config.(Config), []error{}}
+func (p PackageContext) IntermediatesPathVariable(name, path string) blueprint.Variable {
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
p := PathForIntermediates(ctx, path)
if len(ctx.errors) > 0 {
return "", ctx.errors[0]
@@ -124,11 +235,11 @@
// list of present source paths prefixed with the supplied prefix. It may only
// be called during a Go package's initialization - either from the init()
// function or as part of a package-scoped variable's initialization.
-func (p AndroidPackageContext) PrefixedExistentPathsForSourcesVariable(
+func (p PackageContext) PrefixedExistentPathsForSourcesVariable(
name, prefix string, paths []string) blueprint.Variable {
- return p.VariableFunc(name, func(config interface{}) (string, error) {
- ctx := &configErrorWrapper{p, config.(Config), []error{}}
+ return p.VariableFunc(name, func(config Config) (string, error) {
+ ctx := &configErrorWrapper{p, config, []error{}}
paths := ExistentPathsForSources(ctx, "", paths)
if len(ctx.errors) > 0 {
return "", ctx.errors[0]
@@ -137,30 +248,25 @@
})
}
-type RuleParams struct {
- blueprint.RuleParams
- GomaSupported bool
-}
-
// AndroidStaticRule wraps blueprint.StaticRule and provides a default Pool if none is specified
-func (p AndroidPackageContext) AndroidStaticRule(name string, params blueprint.RuleParams,
+func (p PackageContext) AndroidStaticRule(name string, params blueprint.RuleParams,
argNames ...string) blueprint.Rule {
- return p.AndroidRuleFunc(name, func(interface{}) (blueprint.RuleParams, error) {
+ return p.AndroidRuleFunc(name, func(Config) (blueprint.RuleParams, error) {
return params, nil
}, argNames...)
}
// AndroidGomaStaticRule wraps blueprint.StaticRule but uses goma's parallelism if goma is enabled
-func (p AndroidPackageContext) AndroidGomaStaticRule(name string, params blueprint.RuleParams,
+func (p PackageContext) AndroidGomaStaticRule(name string, params blueprint.RuleParams,
argNames ...string) blueprint.Rule {
return p.StaticRule(name, params, argNames...)
}
-func (p AndroidPackageContext) AndroidRuleFunc(name string,
- f func(interface{}) (blueprint.RuleParams, error), argNames ...string) blueprint.Rule {
- return p.PackageContext.RuleFunc(name, func(config interface{}) (blueprint.RuleParams, error) {
+func (p PackageContext) AndroidRuleFunc(name string,
+ f func(Config) (blueprint.RuleParams, error), argNames ...string) blueprint.Rule {
+ return p.RuleFunc(name, func(config Config) (blueprint.RuleParams, error) {
params, err := f(config)
- if config.(Config).UseGoma() && params.Pool == nil {
+ if config.UseGoma() && params.Pool == nil {
// When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the
// local parallelism value
params.Pool = localPool
diff --git a/android/paths.go b/android/paths.go
index b5b4730..e47b9e4 100644
--- a/android/paths.go
+++ b/android/paths.go
@@ -18,6 +18,7 @@
"fmt"
"path/filepath"
"reflect"
+ "sort"
"strings"
"github.com/google/blueprint"
@@ -28,7 +29,7 @@
// Path methods.
type PathContext interface {
Fs() pathtools.FileSystem
- Config() interface{}
+ Config() Config
AddNinjaFileDeps(deps ...string)
}
@@ -36,8 +37,8 @@
GlobWithDeps(globPattern string, excludes []string) ([]string, error)
}
-var _ PathContext = blueprint.SingletonContext(nil)
-var _ PathContext = blueprint.ModuleContext(nil)
+var _ PathContext = SingletonContext(nil)
+var _ PathContext = ModuleContext(nil)
type ModuleInstallPathContext interface {
PathContext
@@ -66,15 +67,6 @@
var _ moduleErrorf = blueprint.ModuleContext(nil)
-// pathConfig returns the android Config interface associated to the context.
-// Panics if the context isn't affiliated with an android build.
-func pathConfig(ctx PathContext) Config {
- if ret, ok := ctx.Config().(Config); ok {
- return ret
- }
- panic("Paths may only be used on Soong builds")
-}
-
// reportPathError will register an error with the attached context. It
// attempts ctx.ModuleErrorf for a better error message first, then falls
// back to ctx.Errorf.
@@ -100,7 +92,7 @@
// Rel returns the portion of the path relative to the directory it was created from. For
// example, Rel on a PathsForModuleSrc would return the path relative to the module source
- // directory.
+ // directory, and OutputPath.Join("foo").Rel() would return "foo".
Rel() string
}
@@ -196,12 +188,12 @@
// PathsForSource returns Paths rooted from SrcDir
func PathsForSource(ctx PathContext, paths []string) Paths {
- if pathConfig(ctx).AllowMissingDependencies() {
+ if ctx.Config().AllowMissingDependencies() {
if modCtx, ok := ctx.(ModuleContext); ok {
ret := make(Paths, 0, len(paths))
- intermediates := filepath.Join(modCtx.ModuleDir(), modCtx.ModuleName(), modCtx.ModuleSubDir(), "missing")
+ intermediates := pathForModule(modCtx).withRel("missing")
for _, path := range paths {
- p := ExistentPathForSource(ctx, intermediates, path)
+ p := ExistentPathForSource(ctx, intermediates.String(), path)
if p.Valid() {
ret = append(ret, p.Path())
} else {
@@ -246,7 +238,10 @@
// source directory, but strip the local source directory from the beginning of
// each string.
func pathsForModuleSrcFromFullPath(ctx ModuleContext, paths []string) Paths {
- prefix := filepath.Join(ctx.AConfig().srcDir, ctx.ModuleDir()) + "/"
+ prefix := filepath.Join(ctx.Config().srcDir, ctx.ModuleDir()) + "/"
+ if prefix == "./" {
+ prefix = ""
+ }
ret := make(Paths, 0, len(paths))
for _, p := range paths {
path := filepath.Clean(p)
@@ -267,7 +262,7 @@
}
// Use Glob so that if the default doesn't exist, a dependency is added so that when it
// is created, we're run again.
- path := filepath.Join(ctx.AConfig().srcDir, ctx.ModuleDir(), def)
+ path := filepath.Join(ctx.Config().srcDir, ctx.ModuleDir(), def)
return ctx.Glob(path, []string{})
}
@@ -283,6 +278,132 @@
return ret
}
+// FirstUniquePaths returns all unique elements of a Paths, keeping the first copy of each. It
+// modifies the Paths slice contents in place, and returns a subslice of the original slice.
+func FirstUniquePaths(list Paths) Paths {
+ k := 0
+outer:
+ for i := 0; i < len(list); i++ {
+ for j := 0; j < k; j++ {
+ if list[i] == list[j] {
+ continue outer
+ }
+ }
+ list[k] = list[i]
+ k++
+ }
+ return list[:k]
+}
+
+// LastUniquePaths returns all unique elements of a Paths, keeping the last copy of each. It
+// modifies the Paths slice contents in place, and returns a subslice of the original slice.
+func LastUniquePaths(list Paths) Paths {
+ totalSkip := 0
+ for i := len(list) - 1; i >= totalSkip; i-- {
+ skip := 0
+ for j := i - 1; j >= totalSkip; j-- {
+ if list[i] == list[j] {
+ skip++
+ } else {
+ list[j+skip] = list[j]
+ }
+ }
+ totalSkip += skip
+ }
+ return list[totalSkip:]
+}
+
+func indexPathList(s Path, list []Path) int {
+ for i, l := range list {
+ if l == s {
+ return i
+ }
+ }
+
+ return -1
+}
+
+func inPathList(p Path, list []Path) bool {
+ return indexPathList(p, list) != -1
+}
+
+func FilterPathList(list []Path, filter []Path) (remainder []Path, filtered []Path) {
+ for _, l := range list {
+ if inPathList(l, filter) {
+ filtered = append(filtered, l)
+ } else {
+ remainder = append(remainder, l)
+ }
+ }
+
+ return
+}
+
+// HasExt returns true of any of the paths have extension ext, otherwise false
+func (p Paths) HasExt(ext string) bool {
+ for _, path := range p {
+ if path.Ext() == ext {
+ return true
+ }
+ }
+
+ return false
+}
+
+// FilterByExt returns the subset of the paths that have extension ext
+func (p Paths) FilterByExt(ext string) Paths {
+ ret := make(Paths, 0, len(p))
+ for _, path := range p {
+ if path.Ext() == ext {
+ ret = append(ret, path)
+ }
+ }
+ return ret
+}
+
+// FilterOutByExt returns the subset of the paths that do not have extension ext
+func (p Paths) FilterOutByExt(ext string) Paths {
+ ret := make(Paths, 0, len(p))
+ for _, path := range p {
+ if path.Ext() != ext {
+ ret = append(ret, path)
+ }
+ }
+ return ret
+}
+
+// DirectorySortedPaths is a slice of paths that are sorted such that all files in a directory
+// (including subdirectories) are in a contiguous subslice of the list, and can be found in
+// O(log(N)) time using a binary search on the directory prefix.
+type DirectorySortedPaths Paths
+
+func PathsToDirectorySortedPaths(paths Paths) DirectorySortedPaths {
+ ret := append(DirectorySortedPaths(nil), paths...)
+ sort.Slice(ret, func(i, j int) bool {
+ return ret[i].String() < ret[j].String()
+ })
+ return ret
+}
+
+// PathsInDirectory returns a subslice of the DirectorySortedPaths as a Paths that contains all entries
+// that are in the specified directory and its subdirectories.
+func (p DirectorySortedPaths) PathsInDirectory(dir string) Paths {
+ prefix := filepath.Clean(dir) + "/"
+ start := sort.Search(len(p), func(i int) bool {
+ return prefix < p[i].String()
+ })
+
+ ret := p[start:]
+
+ end := sort.Search(len(ret), func(i int) bool {
+ return !strings.HasPrefix(ret[i].String(), prefix)
+ })
+
+ ret = ret[:end]
+
+ return Paths(ret)
+}
+
// WritablePaths is a slice of WritablePaths, used for multiple outputs.
type WritablePaths []WritablePath
@@ -298,6 +419,18 @@
return ret
}
+// Paths returns the WritablePaths as a Paths
+func (p WritablePaths) Paths() Paths {
+ if p == nil {
+ return nil
+ }
+ ret := make(Paths, len(p))
+ for i, path := range p {
+ ret[i] = path
+ }
+ return ret
+}
+
type basePath struct {
path string
config Config
@@ -319,6 +452,16 @@
return p.path
}
+func (p basePath) String() string {
+ return p.path
+}
+
+func (p basePath) withRel(rel string) basePath {
+ p.path = filepath.Join(p.path, rel)
+ p.rel = rel
+ return p
+}
+
// SourcePath is a Path representing a file path rooted from SrcDir
type SourcePath struct {
basePath
@@ -326,18 +469,23 @@
var _ Path = SourcePath{}
+func (p SourcePath) withRel(rel string) SourcePath {
+ p.basePath = p.basePath.withRel(rel)
+ return p
+}
+
// safePathForSource is for paths that we expect are safe -- only for use by go
// code that is embedding ninja variables in paths
func safePathForSource(ctx PathContext, path string) SourcePath {
p := validateSafePath(ctx, path)
- ret := SourcePath{basePath{p, pathConfig(ctx), ""}}
+ ret := SourcePath{basePath{p, ctx.Config(), ""}}
abs, err := filepath.Abs(ret.String())
if err != nil {
reportPathError(ctx, "%s", err.Error())
return ret
}
- buildroot, err := filepath.Abs(pathConfig(ctx).buildDir)
+ buildroot, err := filepath.Abs(ctx.Config().buildDir)
if err != nil {
reportPathError(ctx, "%s", err.Error())
return ret
@@ -355,14 +503,14 @@
// On error, it will return a usable, but invalid SourcePath, and report a ModuleError.
func PathForSource(ctx PathContext, pathComponents ...string) SourcePath {
p := validatePath(ctx, pathComponents...)
- ret := SourcePath{basePath{p, pathConfig(ctx), ""}}
+ ret := SourcePath{basePath{p, ctx.Config(), ""}}
abs, err := filepath.Abs(ret.String())
if err != nil {
reportPathError(ctx, "%s", err.Error())
return ret
}
- buildroot, err := filepath.Abs(pathConfig(ctx).buildDir)
+ buildroot, err := filepath.Abs(ctx.Config().buildDir)
if err != nil {
reportPathError(ctx, "%s", err.Error())
return ret
@@ -390,14 +538,14 @@
}
p := validatePath(ctx, pathComponents...)
- path := SourcePath{basePath{p, pathConfig(ctx), ""}}
+ path := SourcePath{basePath{p, ctx.Config(), ""}}
abs, err := filepath.Abs(path.String())
if err != nil {
reportPathError(ctx, "%s", err.Error())
return OptionalPath{}
}
- buildroot, err := filepath.Abs(pathConfig(ctx).buildDir)
+ buildroot, err := filepath.Abs(ctx.Config().buildDir)
if err != nil {
reportPathError(ctx, "%s", err.Error())
return OptionalPath{}
@@ -452,7 +600,7 @@
// provided paths... may not use '..' to escape from the current path.
func (p SourcePath) Join(ctx PathContext, paths ...string) SourcePath {
path := validatePath(ctx, paths...)
- return PathForSource(ctx, p.path, path)
+ return p.withRel(path)
}
// OverlayPath returns the overlay for `path' if it exists. This assumes that the
@@ -493,6 +641,11 @@
basePath
}
+func (p OutputPath) withRel(rel string) OutputPath {
+ p.basePath = p.basePath.withRel(rel)
+ return p
+}
+
var _ Path = OutputPath{}
// PathForOutput joins the provided paths and returns an OutputPath that is
@@ -500,7 +653,7 @@
// On error, it will return a usable, but invalid OutputPath, and report a ModuleError.
func PathForOutput(ctx PathContext, pathComponents ...string) OutputPath {
path := validatePath(ctx, pathComponents...)
- return OutputPath{basePath{path, pathConfig(ctx), ""}}
+ return OutputPath{basePath{path, ctx.Config(), ""}}
}
func (p OutputPath) writablePath() {}
@@ -517,7 +670,7 @@
// provided paths... may not use '..' to escape from the current path.
func (p OutputPath) Join(ctx PathContext, paths ...string) OutputPath {
path := validatePath(ctx, paths...)
- return PathForOutput(ctx, p.path, path)
+ return p.withRel(path)
}
// PathForIntermediates returns an OutputPath representing the top-level
@@ -587,6 +740,10 @@
var _ Path = ModuleOutPath{}
+func pathForModule(ctx ModuleContext) OutputPath {
+ return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir())
+}
+
// PathForVndkRefDump returns an OptionalPath representing the path of the reference
// abi dump for the given module. This is not guaranteed to be valid.
func PathForVndkRefAbiDump(ctx ModuleContext, version, fileName string, vndkOrNdk, isSourceDump bool) OptionalPath {
@@ -615,14 +772,15 @@
// output directory.
func PathForModuleOut(ctx ModuleContext, paths ...string) ModuleOutPath {
p := validatePath(ctx, paths...)
- return ModuleOutPath{PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), p)}
+ return ModuleOutPath{
+ OutputPath: pathForModule(ctx).withRel(p),
+ }
}
// ModuleGenPath is a Path representing the 'gen' directory in a module's output
// directory. Mainly used for generated sources.
type ModuleGenPath struct {
ModuleOutPath
- path string
}
var _ Path = ModuleGenPath{}
@@ -634,8 +792,9 @@
func PathForModuleGen(ctx ModuleContext, paths ...string) ModuleGenPath {
p := validatePath(ctx, paths...)
return ModuleGenPath{
- PathForModuleOut(ctx, "gen", p),
- p,
+ ModuleOutPath: ModuleOutPath{
+ OutputPath: pathForModule(ctx).withRel("gen").withRel(p),
+ },
}
}
@@ -686,7 +845,7 @@
var partition string
if ctx.InstallInData() {
partition = "data"
- } else if ctx.Vendor() {
+ } else if ctx.InstallOnVendorPartition() {
partition = ctx.DeviceConfig().VendorPath()
} else {
partition = "system"
@@ -695,9 +854,17 @@
if ctx.InstallInSanitizerDir() {
partition = "data/asan/" + partition
}
- outPaths = []string{"target", "product", ctx.AConfig().DeviceName(), partition}
+ outPaths = []string{"target", "product", ctx.Config().DeviceName(), partition}
} else {
- outPaths = []string{"host", ctx.Os().String() + "-x86"}
+ switch ctx.Os() {
+ case Linux:
+ outPaths = []string{"host", "linux-x86"}
+ case LinuxBionic:
+ // TODO: should this be a separate top level, or shared with linux-x86?
+ outPaths = []string{"host", "linux_bionic-x86"}
+ default:
+ outPaths = []string{"host", ctx.Os().String() + "-x86"}
+ }
}
if ctx.Debug() {
outPaths = append([]string{"debug"}, outPaths...)
@@ -735,6 +902,22 @@
return validateSafePath(ctx, pathComponents...)
}
+func PathForPhony(ctx PathContext, phony string) WritablePath {
+ if strings.ContainsAny(phony, "$/") {
+ reportPathError(ctx, "Phony target contains invalid character ($ or /): %s", phony)
+ }
+ return PhonyPath{basePath{phony, ctx.Config(), ""}}
+}
+
+type PhonyPath struct {
+ basePath
+}
+
+func (p PhonyPath) writablePath() {}
+
+var _ Path = PhonyPath{}
+var _ WritablePath = PhonyPath{}
+
type testPath struct {
basePath
}
diff --git a/android/paths_test.go b/android/paths_test.go
index 3986b71..1e4ba53 100644
--- a/android/paths_test.go
+++ b/android/paths_test.go
@@ -194,7 +194,7 @@
return pathtools.MockFs(nil)
}
-func (m moduleInstallPathContextImpl) Config() interface{} {
+func (m moduleInstallPathContextImpl) Config() Config {
return m.androidBaseContextImpl.config
}
@@ -209,7 +209,7 @@
}
func TestPathForModuleInstall(t *testing.T) {
- testConfig := TestConfig("")
+ testConfig := TestConfig("", nil)
hostTarget := Target{Os: Linux}
deviceTarget := Target{Os: Android}
@@ -340,3 +340,75 @@
})
}
}
+
+func TestDirectorySortedPaths(t *testing.T) {
+ makePaths := func() Paths {
+ return Paths{
+ PathForTesting("a.txt"),
+ PathForTesting("a/txt"),
+ PathForTesting("a/b/c"),
+ PathForTesting("a/b/d"),
+ PathForTesting("b"),
+ PathForTesting("b/b.txt"),
+ PathForTesting("a/a.txt"),
+ }
+ }
+
+ expected := []string{
+ "a.txt",
+ "a/a.txt",
+ "a/b/c",
+ "a/b/d",
+ "a/txt",
+ "b",
+ "b/b.txt",
+ }
+
+ paths := makePaths()
+ reversePaths := make(Paths, len(paths))
+ for i, v := range paths {
+ reversePaths[len(paths)-i-1] = v
+ }
+
+ sortedPaths := PathsToDirectorySortedPaths(paths)
+ reverseSortedPaths := PathsToDirectorySortedPaths(reversePaths)
+
+ if !reflect.DeepEqual(Paths(sortedPaths).Strings(), expected) {
+ t.Fatalf("sorted paths:\n %#v\n != \n %#v", paths.Strings(), expected)
+ }
+
+ if !reflect.DeepEqual(Paths(reverseSortedPaths).Strings(), expected) {
+ t.Fatalf("sorted reversed paths:\n %#v\n !=\n %#v", reversePaths.Strings(), expected)
+ }
+
+ expectedA := []string{
+ "a/a.txt",
+ "a/b/c",
+ "a/b/d",
+ "a/txt",
+ }
+
+ inA := sortedPaths.PathsInDirectory("a")
+ if !reflect.DeepEqual(inA.Strings(), expectedA) {
+ t.Errorf("FilesInDirectory(a):\n %#v\n != \n %#v", inA.Strings(), expectedA)
+ }
+
+ expectedA_B := []string{
+ "a/b/c",
+ "a/b/d",
+ }
+
+ inA_B := sortedPaths.PathsInDirectory("a/b")
+ if !reflect.DeepEqual(inA_B.Strings(), expectedA_B) {
+ t.Errorf("FilesInDirectory(a/b):\n %#v\n != \n %#v", inA_B.Strings(), expectedA_B)
+ }
+
+ expectedB := []string{
+ "b/b.txt",
+ }
+
+ inB := sortedPaths.PathsInDirectory("b")
+ if !reflect.DeepEqual(inB.Strings(), expectedB) {
+ t.Errorf("FilesInDirectory(b):\n %#v\n != \n %#v", inA.Strings(), expectedA)
+ }
+}
diff --git a/android/prebuilt.go b/android/prebuilt.go
index 080df91..d3f9704 100644
--- a/android/prebuilt.go
+++ b/android/prebuilt.go
@@ -14,7 +14,11 @@
package android
-import "github.com/google/blueprint"
+import (
+ "fmt"
+
+ "github.com/google/blueprint"
+)
// This file implements common functionality for handling modules that may exist as prebuilts,
// source, or both.
@@ -25,35 +29,43 @@
var prebuiltDepTag prebuiltDependencyTag
-type Prebuilt struct {
- Properties struct {
- Srcs []string `android:"arch_variant"`
- // When prefer is set to true the prebuilt will be used instead of any source module with
- // a matching name.
- Prefer bool `android:"arch_variant"`
+type PrebuiltProperties struct {
+ // When prefer is set to true the prebuilt will be used instead of any source module with
+ // a matching name.
+ Prefer *bool `android:"arch_variant"`
- SourceExists bool `blueprint:"mutated"`
- UsePrebuilt bool `blueprint:"mutated"`
- }
- module Module
+ SourceExists bool `blueprint:"mutated"`
+ UsePrebuilt bool `blueprint:"mutated"`
+}
+
+type Prebuilt struct {
+ properties PrebuiltProperties
+ module Module
+ srcs *[]string
}
func (p *Prebuilt) Name(name string) string {
return "prebuilt_" + name
}
-func (p *Prebuilt) Path(ctx ModuleContext) Path {
- if len(p.Properties.Srcs) == 0 {
+func (p *Prebuilt) SingleSourcePath(ctx ModuleContext) Path {
+ if len(*p.srcs) == 0 {
ctx.PropertyErrorf("srcs", "missing prebuilt source file")
return nil
}
- if len(p.Properties.Srcs) > 1 {
+ if len(*p.srcs) > 1 {
ctx.PropertyErrorf("srcs", "multiple prebuilt source files")
return nil
}
- return PathForModuleSrc(ctx, p.Properties.Srcs[0])
+ return PathForModuleSrc(ctx, (*p.srcs)[0])
+}
+
+func InitPrebuiltModule(module PrebuiltInterface, srcs *[]string) {
+ p := module.Prebuilt()
+ module.AddProperties(&p.properties)
+ p.srcs = srcs
}
type PrebuiltInterface interface {
@@ -61,11 +73,11 @@
Prebuilt() *Prebuilt
}
-func registerPrebuiltsPreArchMutators(ctx RegisterMutatorsContext) {
+func RegisterPrebuiltsPreArchMutators(ctx RegisterMutatorsContext) {
ctx.BottomUp("prebuilts", prebuiltMutator).Parallel()
}
-func registerPrebuiltsPostDepsMutators(ctx RegisterMutatorsContext) {
+func RegisterPrebuiltsPostDepsMutators(ctx RegisterMutatorsContext) {
ctx.TopDown("prebuilt_select", PrebuiltSelectModuleMutator).Parallel()
ctx.BottomUp("prebuilt_replace", PrebuiltReplaceMutator).Parallel()
}
@@ -78,7 +90,7 @@
name := m.base().BaseModuleName()
if ctx.OtherModuleExists(name) {
ctx.AddReverseDependency(ctx.Module(), prebuiltDepTag, name)
- p.Properties.SourceExists = true
+ p.properties.SourceExists = true
} else {
ctx.Rename(name)
}
@@ -90,15 +102,18 @@
func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) {
if m, ok := ctx.Module().(PrebuiltInterface); ok && m.Prebuilt() != nil {
p := m.Prebuilt()
- if !p.Properties.SourceExists {
- p.Properties.UsePrebuilt = p.usePrebuilt(ctx, nil)
+ if p.srcs == nil {
+ panic(fmt.Errorf("prebuilt module did not have InitPrebuiltModule called on it"))
+ }
+ if !p.properties.SourceExists {
+ p.properties.UsePrebuilt = p.usePrebuilt(ctx, nil)
}
} else if s, ok := ctx.Module().(Module); ok {
- ctx.VisitDirectDeps(func(m blueprint.Module) {
+ ctx.VisitDirectDeps(func(m Module) {
if ctx.OtherModuleDependencyTag(m) == prebuiltDepTag {
p := m.(PrebuiltInterface).Prebuilt()
if p.usePrebuilt(ctx, s) {
- p.Properties.UsePrebuilt = true
+ p.properties.UsePrebuilt = true
s.SkipInstall()
}
}
@@ -113,8 +128,8 @@
if m, ok := ctx.Module().(PrebuiltInterface); ok && m.Prebuilt() != nil {
p := m.Prebuilt()
name := m.base().BaseModuleName()
- if p.Properties.UsePrebuilt {
- if p.Properties.SourceExists {
+ if p.properties.UsePrebuilt {
+ if p.properties.SourceExists {
ctx.ReplaceDependencies(name)
}
} else {
@@ -126,12 +141,12 @@
// usePrebuilt returns true if a prebuilt should be used instead of the source module. The prebuilt
// will be used if it is marked "prefer" or if the source module is disabled.
func (p *Prebuilt) usePrebuilt(ctx TopDownMutatorContext, source Module) bool {
- if len(p.Properties.Srcs) == 0 {
+ if len(*p.srcs) == 0 {
return false
}
// TODO: use p.Properties.Name and ctx.ModuleDir to override preference
- if p.Properties.Prefer {
+ if Bool(p.properties.Prefer) {
return true
}
diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go
index fe763ed..93f5805 100644
--- a/android/prebuilt_test.go
+++ b/android/prebuilt_test.go
@@ -118,13 +118,13 @@
}
defer os.RemoveAll(buildDir)
- config := TestConfig(buildDir)
+ config := TestConfig(buildDir, nil)
for _, test := range prebuiltsTests {
t.Run(test.name, func(t *testing.T) {
ctx := NewTestContext()
- ctx.PreArchMutators(registerPrebuiltsPreArchMutators)
- ctx.PostDepsMutators(registerPrebuiltsPostDepsMutators)
+ ctx.PreArchMutators(RegisterPrebuiltsPreArchMutators)
+ ctx.PostDepsMutators(RegisterPrebuiltsPostDepsMutators)
ctx.RegisterModuleType("prebuilt", ModuleFactoryAdaptor(newPrebuiltModule))
ctx.RegisterModuleType("source", ModuleFactoryAdaptor(newSourceModule))
ctx.Register()
@@ -151,7 +151,7 @@
}
if p, ok := m.(*prebuiltModule); ok {
dependsOnPrebuiltModule = true
- if !p.Prebuilt().Properties.UsePrebuilt {
+ if !p.Prebuilt().properties.UsePrebuilt {
t.Errorf("dependency on prebuilt module not marked used")
}
}
@@ -180,12 +180,16 @@
type prebuiltModule struct {
ModuleBase
- prebuilt Prebuilt
+ prebuilt Prebuilt
+ properties struct {
+ Srcs []string
+ }
}
func newPrebuiltModule() Module {
m := &prebuiltModule{}
- m.AddProperties(&m.prebuilt.Properties)
+ m.AddProperties(&m.properties)
+ InitPrebuiltModule(m, &m.properties.Srcs)
InitAndroidModule(m)
return m
}
diff --git a/android/proto.go b/android/proto.go
new file mode 100644
index 0000000..1c70656
--- /dev/null
+++ b/android/proto.go
@@ -0,0 +1,63 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+// TODO(ccross): protos are often used to communicate between multiple modules. If the only
+// way to convert a proto to source is to reference it as a source file, and external modules cannot
+// reference source files in other modules, then every module that owns a proto file will need to
+// export a library for every type of external user (lite vs. full, c vs. c++ vs. java). It would
+// be better to support a proto module type that exported a proto file along with some include dirs,
+// and then external modules could depend on the proto module but use their own settings to
+// generate the source.
+
+func ProtoFlags(ctx ModuleContext, p *ProtoProperties) []string {
+ var protoFlags []string
+ if len(p.Proto.Local_include_dirs) > 0 {
+ localProtoIncludeDirs := PathsForModuleSrc(ctx, p.Proto.Local_include_dirs)
+ protoFlags = append(protoFlags, JoinWithPrefix(localProtoIncludeDirs.Strings(), "-I"))
+ }
+ if len(p.Proto.Include_dirs) > 0 {
+ rootProtoIncludeDirs := PathsForSource(ctx, p.Proto.Include_dirs)
+ protoFlags = append(protoFlags, JoinWithPrefix(rootProtoIncludeDirs.Strings(), "-I"))
+ }
+
+ protoFlags = append(protoFlags, "-I .")
+
+ return protoFlags
+}
+
+// ProtoDir returns the module's "gen/proto" directory
+func ProtoDir(ctx ModuleContext) ModuleGenPath {
+ return PathForModuleGen(ctx, "proto")
+}
+
+// ProtoSubDir returns the module's "gen/proto/path/to/module" directory
+func ProtoSubDir(ctx ModuleContext) ModuleGenPath {
+ return PathForModuleGen(ctx, "proto", ctx.ModuleDir())
+}
+
+type ProtoProperties struct {
+ Proto struct {
+ // Proto generator type. C++: full or lite. Java: micro, nano, stream, or lite.
+ Type *string `android:"arch_variant"`
+
+ // list of directories that will be added to the protoc include paths.
+ Include_dirs []string
+
+ // list of directories relative to the bp file that will
+ // be added to the protoc include paths.
+ Local_include_dirs []string
+ } `android:"arch_variant"`
+}
diff --git a/android/register.go b/android/register.go
index 226e790..6c88af1 100644
--- a/android/register.go
+++ b/android/register.go
@@ -31,6 +31,7 @@
}
var singletons []singleton
+var preSingletons []singleton
type mutator struct {
name string
@@ -43,7 +44,7 @@
type ModuleFactory func() Module
-// ModuleFactoryAdapter Wraps a ModuleFactory into a blueprint.ModuleFactory by converting an Module
+// ModuleFactoryAdaptor wraps a ModuleFactory into a blueprint.ModuleFactory by converting a Module
// into a blueprint.Module and a list of property structs
func ModuleFactoryAdaptor(factory ModuleFactory) blueprint.ModuleFactory {
return func() (blueprint.Module, []interface{}) {
@@ -52,12 +53,27 @@
}
}
+type SingletonFactory func() Singleton
+
+// SingletonFactoryAdaptor wraps a SingletonFactory into a blueprint.SingletonFactory by converting
+// a Singleton into a blueprint.Singleton
+func SingletonFactoryAdaptor(factory SingletonFactory) blueprint.SingletonFactory {
+ return func() blueprint.Singleton {
+ singleton := factory()
+ return singletonAdaptor{singleton}
+ }
+}
+
func RegisterModuleType(name string, factory ModuleFactory) {
moduleTypes = append(moduleTypes, moduleType{name, ModuleFactoryAdaptor(factory)})
}
-func RegisterSingletonType(name string, factory blueprint.SingletonFactory) {
- singletons = append(singletons, singleton{name, factory})
+func RegisterSingletonType(name string, factory SingletonFactory) {
+ singletons = append(singletons, singleton{name, SingletonFactoryAdaptor(factory)})
+}
+
+func RegisterPreSingletonType(name string, factory SingletonFactory) {
+ preSingletons = append(preSingletons, singleton{name, SingletonFactoryAdaptor(factory)})
}
type Context struct {
@@ -69,6 +85,10 @@
}
func (ctx *Context) Register() {
+ for _, t := range preSingletons {
+ ctx.RegisterPreSingletonType(t.name, t.factory)
+ }
+
for _, t := range moduleTypes {
ctx.RegisterModuleType(t.name, t.factory)
}
@@ -79,5 +99,5 @@
registerMutators(ctx.Context, preArch, preDeps, postDeps)
- ctx.RegisterSingletonType("env", EnvSingleton)
+ ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(EnvSingleton))
}
diff --git a/android/singleton.go b/android/singleton.go
new file mode 100644
index 0000000..f577b0a
--- /dev/null
+++ b/android/singleton.go
@@ -0,0 +1,165 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "github.com/google/blueprint"
+ "github.com/google/blueprint/pathtools"
+)
+
+// SingletonContext
+type SingletonContext interface {
+ Config() Config
+
+ ModuleName(module blueprint.Module) string
+ ModuleDir(module blueprint.Module) string
+ ModuleSubDir(module blueprint.Module) string
+ ModuleType(module blueprint.Module) string
+ BlueprintFile(module blueprint.Module) string
+
+ ModuleErrorf(module blueprint.Module, format string, args ...interface{})
+ Errorf(format string, args ...interface{})
+ Failed() bool
+
+ Variable(pctx PackageContext, name, value string)
+ Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule
+ Build(pctx PackageContext, params BuildParams)
+ RequireNinjaVersion(major, minor, micro int)
+
+ // SetNinjaBuildDir sets the value of the top-level "builddir" Ninja variable
+ // that controls where Ninja stores its build log files. This value can be
+ // set at most one time for a single build, later calls are ignored.
+ SetNinjaBuildDir(pctx PackageContext, value string)
+
+ // Eval takes a string with embedded ninja variables, and returns a string
+ // with all of the variables recursively expanded. Any variables references
+ // are expanded in the scope of the PackageContext.
+ Eval(pctx PackageContext, ninjaStr string) (string, error)
+
+ VisitAllModules(visit func(Module))
+ VisitAllModulesIf(pred func(Module) bool, visit func(Module))
+ VisitDepsDepthFirst(module Module, visit func(Module))
+ VisitDepsDepthFirstIf(module Module, pred func(Module) bool,
+ visit func(Module))
+
+ VisitAllModuleVariants(module Module, visit func(Module))
+
+ PrimaryModule(module Module) Module
+ FinalModule(module Module) Module
+
+ AddNinjaFileDeps(deps ...string)
+
+ // GlobWithDeps returns a list of files that match the specified pattern but do not match any
+ // of the patterns in excludes. It also adds efficient dependencies to rerun the primary
+ // builder whenever a file matching the pattern as added or removed, without rerunning if a
+ // file that does not match the pattern is added to a searched directory.
+ GlobWithDeps(pattern string, excludes []string) ([]string, error)
+
+ Fs() pathtools.FileSystem
+}
+
+type singletonAdaptor struct {
+ Singleton
+}
+
+func (s singletonAdaptor) GenerateBuildActions(ctx blueprint.SingletonContext) {
+ s.Singleton.GenerateBuildActions(singletonContextAdaptor{ctx})
+}
+
+type Singleton interface {
+ GenerateBuildActions(SingletonContext)
+}
+
+type singletonContextAdaptor struct {
+ blueprint.SingletonContext
+}
+
+func (s singletonContextAdaptor) Config() Config {
+ return s.SingletonContext.Config().(Config)
+}
+
+func (s singletonContextAdaptor) Variable(pctx PackageContext, name, value string) {
+ s.SingletonContext.Variable(pctx.PackageContext, name, value)
+}
+
+func (s singletonContextAdaptor) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule {
+ return s.SingletonContext.Rule(pctx.PackageContext, name, params, argNames...)
+}
+
+func (s singletonContextAdaptor) Build(pctx PackageContext, params BuildParams) {
+ bparams := convertBuildParams(params)
+ s.SingletonContext.Build(pctx.PackageContext, bparams)
+
+}
+
+func (s singletonContextAdaptor) SetNinjaBuildDir(pctx PackageContext, value string) {
+ s.SingletonContext.SetNinjaBuildDir(pctx.PackageContext, value)
+}
+
+func (s singletonContextAdaptor) Eval(pctx PackageContext, ninjaStr string) (string, error) {
+ return s.SingletonContext.Eval(pctx.PackageContext, ninjaStr)
+}
+
+// visitAdaptor wraps a visit function that takes an android.Module parameter into
+// a function that takes an blueprint.Module parameter and only calls the visit function if the
+// blueprint.Module is an android.Module.
+func visitAdaptor(visit func(Module)) func(blueprint.Module) {
+ return func(module blueprint.Module) {
+ if aModule, ok := module.(Module); ok {
+ visit(aModule)
+ }
+ }
+}
+
+// predAdaptor wraps a pred function that takes an android.Module parameter
+// into a function that takes an blueprint.Module parameter and only calls the visit function if the
+// blueprint.Module is an android.Module, otherwise returns false.
+func predAdaptor(pred func(Module) bool) func(blueprint.Module) bool {
+ return func(module blueprint.Module) bool {
+ if aModule, ok := module.(Module); ok {
+ return pred(aModule)
+ } else {
+ return false
+ }
+ }
+}
+
+func (s singletonContextAdaptor) VisitAllModules(visit func(Module)) {
+ s.SingletonContext.VisitAllModules(visitAdaptor(visit))
+}
+
+func (s singletonContextAdaptor) VisitAllModulesIf(pred func(Module) bool, visit func(Module)) {
+ s.SingletonContext.VisitAllModulesIf(predAdaptor(pred), visitAdaptor(visit))
+}
+
+func (s singletonContextAdaptor) VisitDepsDepthFirst(module Module, visit func(Module)) {
+ s.SingletonContext.VisitDepsDepthFirst(module, visitAdaptor(visit))
+}
+
+func (s singletonContextAdaptor) VisitDepsDepthFirstIf(module Module, pred func(Module) bool, visit func(Module)) {
+ s.SingletonContext.VisitDepsDepthFirstIf(module, predAdaptor(pred), visitAdaptor(visit))
+}
+
+func (s singletonContextAdaptor) VisitAllModuleVariants(module Module, visit func(Module)) {
+ s.SingletonContext.VisitAllModuleVariants(module, visitAdaptor(visit))
+}
+
+func (s singletonContextAdaptor) PrimaryModule(module Module) Module {
+ return s.SingletonContext.PrimaryModule(module).(Module)
+}
+
+func (s singletonContextAdaptor) FinalModule(module Module) Module {
+ return s.SingletonContext.FinalModule(module).(Module)
+}
diff --git a/android/testing.go b/android/testing.go
index 4144775..ae012b0 100644
--- a/android/testing.go
+++ b/android/testing.go
@@ -16,20 +16,38 @@
import (
"fmt"
+ "path/filepath"
"strings"
"github.com/google/blueprint"
)
func NewTestContext() *TestContext {
- return &TestContext{
- Context: blueprint.NewContext(),
+ namespaceExportFilter := func(namespace *Namespace) bool {
+ return true
}
+
+ nameResolver := NewNameResolver(namespaceExportFilter)
+ ctx := &TestContext{
+ Context: blueprint.NewContext(),
+ NameResolver: nameResolver,
+ }
+
+ ctx.SetNameInterface(nameResolver)
+
+ return ctx
+}
+
+func NewTestArchContext() *TestContext {
+ ctx := NewTestContext()
+ ctx.preDeps = append(ctx.preDeps, registerArchMutator)
+ return ctx
}
type TestContext struct {
*blueprint.Context
preArch, preDeps, postDeps []RegisterMutatorFunc
+ NameResolver *NameResolver
}
func (ctx *TestContext) PreArchMutators(f RegisterMutatorFunc) {
@@ -47,7 +65,7 @@
func (ctx *TestContext) Register() {
registerMutators(ctx.Context, ctx.preArch, ctx.preDeps, ctx.postDeps)
- ctx.RegisterSingletonType("env", EnvSingleton)
+ ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(EnvSingleton))
}
func (ctx *TestContext) ModuleForTests(name, variant string) TestingModule {
@@ -59,12 +77,38 @@
})
if module == nil {
- panic(fmt.Errorf("failed to find module %q variant %q", name, variant))
+ // find all the modules that do exist
+ allModuleNames := []string{}
+ ctx.VisitAllModules(func(m blueprint.Module) {
+ allModuleNames = append(allModuleNames, m.(Module).Name()+"("+ctx.ModuleSubDir(m)+")")
+ })
+
+ panic(fmt.Errorf("failed to find module %q variant %q."+
+ "\nall modules: %v", name, variant, allModuleNames))
}
return TestingModule{module}
}
+// MockFileSystem causes the Context to replace all reads with accesses to the provided map of
+// filenames to contents stored as a byte slice.
+func (ctx *TestContext) MockFileSystem(files map[string][]byte) {
+ // no module list file specified; find every file named Blueprints or Android.bp
+ pathsToParse := []string{}
+ for candidate := range files {
+ base := filepath.Base(candidate)
+ if base == "Blueprints" || base == "Android.bp" {
+ pathsToParse = append(pathsToParse, candidate)
+ }
+ }
+ if len(pathsToParse) < 1 {
+ panic(fmt.Sprintf("No Blueprint or Android.bp files found in mock filesystem: %v\n", files))
+ }
+ files[blueprint.MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n"))
+
+ ctx.Context.MockFileSystem(files)
+}
+
type TestingModule struct {
module Module
}
@@ -73,7 +117,7 @@
return m.module
}
-func (m TestingModule) Rule(rule string) ModuleBuildParams {
+func (m TestingModule) Rule(rule string) BuildParams {
for _, p := range m.module.BuildParamsForTests() {
if strings.Contains(p.Rule.String(), rule) {
return p
@@ -82,17 +126,29 @@
panic(fmt.Errorf("couldn't find rule %q", rule))
}
-func (m TestingModule) Output(file string) ModuleBuildParams {
+func (m TestingModule) Description(desc string) BuildParams {
+ for _, p := range m.module.BuildParamsForTests() {
+ if p.Description == desc {
+ return p
+ }
+ }
+ panic(fmt.Errorf("couldn't find description %q", desc))
+}
+
+func (m TestingModule) Output(file string) BuildParams {
+ var searchedOutputs []string
for _, p := range m.module.BuildParamsForTests() {
outputs := append(WritablePaths(nil), p.Outputs...)
if p.Output != nil {
outputs = append(outputs, p.Output)
}
for _, f := range outputs {
- if f.Base() == file {
+ if f.String() == file || f.Rel() == file {
return p
}
+ searchedOutputs = append(searchedOutputs, f.Rel())
}
}
- panic(fmt.Errorf("couldn't find output %q", file))
+ panic(fmt.Errorf("couldn't find output %q.\nall outputs: %v",
+ file, searchedOutputs))
}
diff --git a/android/util.go b/android/util.go
index 80c7870..29bb9b1 100644
--- a/android/util.go
+++ b/android/util.go
@@ -77,6 +77,41 @@
return false
}
+// FirstUniqueStrings returns all unique elements of a slice of strings, keeping the first copy of
+// each. It modifies the slice contents in place, and returns a subslice of the original slice.
+func FirstUniqueStrings(list []string) []string {
+ k := 0
+outer:
+ for i := 0; i < len(list); i++ {
+ for j := 0; j < k; j++ {
+ if list[i] == list[j] {
+ continue outer
+ }
+ }
+ list[k] = list[i]
+ k++
+ }
+ return list[:k]
+}
+
+// LastUniqueStrings returns all unique elements of a slice of strings, keeping the last copy of
+// each. It modifies the slice contents in place, and returns a subslice of the original slice.
+func LastUniqueStrings(list []string) []string {
+ totalSkip := 0
+ for i := len(list) - 1; i >= totalSkip; i-- {
+ skip := 0
+ for j := i - 1; j >= totalSkip; j-- {
+ if list[i] == list[j] {
+ skip++
+ } else {
+ list[j+skip] = list[j]
+ }
+ }
+ totalSkip += skip
+ }
+ return list[totalSkip:]
+}
+
// checkCalledFromInit panics if a Go package's init function is not on the
// call stack.
func checkCalledFromInit() {
@@ -122,3 +157,10 @@
ok = true
return
}
+
+func GetNumericSdkVersion(v string) string {
+ if strings.Contains(v, "system_") {
+ return strings.Replace(v, "system_", "", 1)
+ }
+ return v
+}
diff --git a/android/util_test.go b/android/util_test.go
new file mode 100644
index 0000000..32f92b4
--- /dev/null
+++ b/android/util_test.go
@@ -0,0 +1,120 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "reflect"
+ "testing"
+)
+
+var firstUniqueStringsTestCases = []struct {
+ in []string
+ out []string
+}{
+ {
+ in: []string{"a"},
+ out: []string{"a"},
+ },
+ {
+ in: []string{"a", "b"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"a", "a"},
+ out: []string{"a"},
+ },
+ {
+ in: []string{"a", "b", "a"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"b", "a", "a"},
+ out: []string{"b", "a"},
+ },
+ {
+ in: []string{"a", "a", "b"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"a", "b", "a", "b"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"liblog", "libdl", "libc++", "libdl", "libc", "libm"},
+ out: []string{"liblog", "libdl", "libc++", "libc", "libm"},
+ },
+}
+
+func TestFirstUniqueStrings(t *testing.T) {
+ for _, testCase := range firstUniqueStringsTestCases {
+ out := FirstUniqueStrings(testCase.in)
+ if !reflect.DeepEqual(out, testCase.out) {
+ t.Errorf("incorrect output:")
+ t.Errorf(" input: %#v", testCase.in)
+ t.Errorf(" expected: %#v", testCase.out)
+ t.Errorf(" got: %#v", out)
+ }
+ }
+}
+
+var lastUniqueStringsTestCases = []struct {
+ in []string
+ out []string
+}{
+ {
+ in: []string{"a"},
+ out: []string{"a"},
+ },
+ {
+ in: []string{"a", "b"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"a", "a"},
+ out: []string{"a"},
+ },
+ {
+ in: []string{"a", "b", "a"},
+ out: []string{"b", "a"},
+ },
+ {
+ in: []string{"b", "a", "a"},
+ out: []string{"b", "a"},
+ },
+ {
+ in: []string{"a", "a", "b"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"a", "b", "a", "b"},
+ out: []string{"a", "b"},
+ },
+ {
+ in: []string{"liblog", "libdl", "libc++", "libdl", "libc", "libm"},
+ out: []string{"liblog", "libc++", "libdl", "libc", "libm"},
+ },
+}
+
+func TestLastUniqueStrings(t *testing.T) {
+ for _, testCase := range lastUniqueStringsTestCases {
+ out := LastUniqueStrings(testCase.in)
+ if !reflect.DeepEqual(out, testCase.out) {
+ t.Errorf("incorrect output:")
+ t.Errorf(" input: %#v", testCase.in)
+ t.Errorf(" expected: %#v", testCase.out)
+ t.Errorf(" got: %#v", out)
+ }
+ }
+}
diff --git a/android/variable.go b/android/variable.go
index 8462d0d..ab8103a 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -81,6 +81,7 @@
Debuggable struct {
Cflags []string
Cppflags []string
+ Init_rc []string
}
// eng is true for -eng builds, and can be used to turn on additionaly heavyweight debugging
@@ -91,7 +92,11 @@
}
Pdk struct {
- Enabled *bool
+ Enabled *bool `android:"arch_variant"`
+ } `android:"arch_variant"`
+
+ Uml struct {
+ Cppflags []string
}
} `android:"arch_variant"`
}
@@ -102,8 +107,10 @@
// Suffix to add to generated Makefiles
Make_suffix *string `json:",omitempty"`
- Platform_sdk_version *int `json:",omitempty"`
- Platform_version_all_codenames []string `json:",omitempty"`
+ Platform_sdk_version *int `json:",omitempty"`
+ Platform_sdk_final *bool `json:",omitempty"`
+ Platform_version_active_codenames []string `json:",omitempty"`
+ Platform_version_future_codenames []string `json:",omitempty"`
DeviceName *string `json:",omitempty"`
DeviceArch *string `json:",omitempty"`
@@ -125,6 +132,19 @@
CrossHostArch *string `json:",omitempty"`
CrossHostSecondaryArch *string `json:",omitempty"`
+ ResourceOverlays *[]string `json:",omitempty"`
+ EnforceRROTargets *[]string `json:",omitempty"`
+ EnforceRROExcludedOverlays *[]string `json:",omitempty"`
+
+ AAPTCharacteristics *string `json:",omitempty"`
+ AAPTConfig *[]string `json:",omitempty"`
+ AAPTPreferredConfig *string `json:",omitempty"`
+ AAPTPrebuiltDPI *[]string `json:",omitempty"`
+
+ DefaultAppCertificate *string `json:",omitempty"`
+
+ AppsDefaultVersionName *string `json:",omitempty"`
+
Allow_missing_dependencies *bool `json:",omitempty"`
Unbundled_build *bool `json:",omitempty"`
Brillo *bool `json:",omitempty"`
@@ -135,13 +155,18 @@
UseGoma *bool `json:",omitempty"`
Debuggable *bool `json:",omitempty"`
Eng *bool `json:",omitempty"`
- EnableCFI *bool `json:",omitempty"`
Device_uses_hwc2 *bool `json:",omitempty"`
Treble *bool `json:",omitempty"`
Pdk *bool `json:",omitempty"`
+ Uml *bool `json:",omitempty"`
+ MinimizeJavaDebugInfo *bool `json:",omitempty"`
IntegerOverflowExcludePaths *[]string `json:",omitempty"`
+ EnableCFI *bool `json:",omitempty"`
+ CFIExcludePaths *[]string `json:",omitempty"`
+ CFIIncludePaths *[]string `json:",omitempty"`
+
VendorPath *string `json:",omitempty"`
ClangTidy *bool `json:",omitempty"`
@@ -166,6 +191,11 @@
Override_rs_driver *string `json:",omitempty"`
DeviceKernelHeaders []string `json:",omitempty"`
+ DistDir *string `json:",omitempty"`
+
+ ExtraVndkVersions []string `json:",omitempty"`
+
+ NamespacesToExport []string `json:",omitempty"`
}
func boolPtr(v bool) *bool {
@@ -182,21 +212,30 @@
func (v *productVariables) SetDefaultConfig() {
*v = productVariables{
- Platform_sdk_version: intPtr(24),
+ Platform_sdk_version: intPtr(26),
+ Platform_version_active_codenames: []string{"P"},
+ Platform_version_future_codenames: []string{"P"},
+
HostArch: stringPtr("x86_64"),
HostSecondaryArch: stringPtr("x86"),
- DeviceName: stringPtr("flounder"),
+ DeviceName: stringPtr("generic_arm64"),
DeviceArch: stringPtr("arm64"),
DeviceArchVariant: stringPtr("armv8-a"),
- DeviceCpuVariant: stringPtr("denver64"),
+ DeviceCpuVariant: stringPtr("generic"),
DeviceAbi: &[]string{"arm64-v8a"},
DeviceUsesClang: boolPtr(true),
DeviceSecondaryArch: stringPtr("arm"),
- DeviceSecondaryArchVariant: stringPtr("armv7-a-neon"),
- DeviceSecondaryCpuVariant: stringPtr("denver"),
- DeviceSecondaryAbi: &[]string{"armeabi-v7a"},
- Malloc_not_svelte: boolPtr(false),
- Safestack: boolPtr(false),
+ DeviceSecondaryArchVariant: stringPtr("armv8-a"),
+ DeviceSecondaryCpuVariant: stringPtr("generic"),
+ DeviceSecondaryAbi: &[]string{"armeabi-v7a", "armeabi"},
+
+ AAPTConfig: &[]string{"normal", "large", "xlarge", "hdpi", "xhdpi", "xxhdpi"},
+ AAPTPreferredConfig: stringPtr("xhdpi"),
+ AAPTCharacteristics: stringPtr("nosdcard"),
+ AAPTPrebuiltDPI: &[]string{"xhdpi", "xxhdpi"},
+
+ Malloc_not_svelte: boolPtr(true),
+ Safestack: boolPtr(false),
}
if runtime.GOOS == "linux" {
@@ -225,7 +264,7 @@
property := "product_variables." + proptools.PropertyNameForField(name)
// Check that the variable was set for the product
- val := reflect.ValueOf(mctx.Config().(Config).ProductVariables).FieldByName(name)
+ val := reflect.ValueOf(mctx.Config().ProductVariables).FieldByName(name)
if !val.IsValid() || val.Kind() != reflect.Ptr || val.IsNil() {
continue
}
diff --git a/android/writedocs.go b/android/writedocs.go
new file mode 100644
index 0000000..9737030
--- /dev/null
+++ b/android/writedocs.go
@@ -0,0 +1,73 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/google/blueprint"
+)
+
+func init() {
+ RegisterSingletonType("writedocs", DocsSingleton)
+}
+
+func DocsSingleton() Singleton {
+ return &docsSingleton{}
+}
+
+type docsSingleton struct{}
+
+func primaryBuilderPath(ctx SingletonContext) Path {
+ primaryBuilder, err := filepath.Rel(ctx.Config().BuildDir(), os.Args[0])
+ if err != nil {
+ ctx.Errorf("path to primary builder %q is not in build dir %q",
+ os.Args[0], ctx.Config().BuildDir())
+ }
+
+ return PathForOutput(ctx, primaryBuilder)
+}
+
+func (c *docsSingleton) GenerateBuildActions(ctx SingletonContext) {
+ // Generate build system docs for the primary builder. Generating docs reads the source
+ // files used to build the primary builder, but that dependency will be picked up through
+ // the dependency on the primary builder itself. There are no dependencies on the
+ // Blueprints files, as any relevant changes to the Blueprints files would have caused
+ // a rebuild of the primary builder.
+ docsFile := PathForOutput(ctx, "docs", "soong_build.html")
+ primaryBuilder := primaryBuilderPath(ctx)
+ soongDocs := ctx.Rule(pctx, "soongDocs",
+ blueprint.RuleParams{
+ Command: fmt.Sprintf("%s --soong_docs %s %s",
+ primaryBuilder.String(), docsFile.String(), strings.Join(os.Args[1:], " ")),
+ CommandDeps: []string{primaryBuilder.String()},
+ Description: fmt.Sprintf("%s docs $out", primaryBuilder.Base()),
+ })
+
+ ctx.Build(pctx, BuildParams{
+ Rule: soongDocs,
+ Output: docsFile,
+ })
+
+ // Add a phony target for building the documentation
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Output: PathForPhony(ctx, "soong_docs"),
+ Input: docsFile,
+ })
+}
diff --git a/androidmk/cmd/androidmk/android.go b/androidmk/cmd/androidmk/android.go
index 194b2c9..4022a5e 100644
--- a/androidmk/cmd/androidmk/android.go
+++ b/androidmk/cmd/androidmk/android.go
@@ -1,15 +1,31 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package main
import (
mkparser "android/soong/androidmk/parser"
"fmt"
+ "sort"
"strings"
bpparser "github.com/google/blueprint/parser"
)
const (
- clear_vars = "__android_mk_clear_vars"
+ clear_vars = "__android_mk_clear_vars"
+ include_ignored = "__android_mk_include_ignored"
)
type bpVariable struct {
@@ -26,15 +42,17 @@
var rewriteProperties = map[string](func(variableAssignmentContext) error){
// custom functions
+ "LOCAL_AIDL_INCLUDES": localAidlIncludes,
"LOCAL_C_INCLUDES": localIncludeDirs,
"LOCAL_EXPORT_C_INCLUDE_DIRS": exportIncludeDirs,
"LOCAL_LDFLAGS": ldflags,
"LOCAL_MODULE_CLASS": prebuiltClass,
"LOCAL_MODULE_STEM": stem,
"LOCAL_MODULE_HOST_OS": hostOs,
- "LOCAL_SRC_FILES": srcFiles,
"LOCAL_SANITIZE": sanitize(""),
"LOCAL_SANITIZE_DIAG": sanitize("diag."),
+ "LOCAL_CFLAGS": cflags,
+ "LOCAL_UNINSTALLABLE_MODULE": invert("installable"),
// composite functions
"LOCAL_MODULE_TAGS": includeVariableIf(bpVariable{"tags", bpparser.ListType}, not(valueDumpEquals("optional"))),
@@ -72,9 +90,15 @@
"LOCAL_PROTOC_OPTIMIZE_TYPE": "proto.type",
"LOCAL_MODULE_OWNER": "owner",
"LOCAL_RENDERSCRIPT_TARGET_API": "renderscript.target_api",
+ "LOCAL_NOTICE_FILE": "notice",
+ "LOCAL_JAVA_LANGUAGE_VERSION": "java_version",
+ "LOCAL_INSTRUMENTATION_FOR": "instrumentation_for",
+
+ "LOCAL_DEX_PREOPT_PROFILE_CLASS_LISTING": "dex_preopt.profile",
})
addStandardProperties(bpparser.ListType,
map[string]string{
+ "LOCAL_SRC_FILES": "srcs",
"LOCAL_SRC_FILES_EXCLUDE": "exclude_srcs",
"LOCAL_HEADER_LIBRARIES": "header_libs",
"LOCAL_SHARED_LIBRARIES": "shared_libs",
@@ -83,7 +107,6 @@
"LOCAL_SYSTEM_SHARED_LIBRARIES": "system_shared_libs",
"LOCAL_ASFLAGS": "asflags",
"LOCAL_CLANG_ASFLAGS": "clang_asflags",
- "LOCAL_CFLAGS": "cflags",
"LOCAL_CONLYFLAGS": "conlyflags",
"LOCAL_CPPFLAGS": "cppflags",
"LOCAL_REQUIRED_MODULES": "required",
@@ -102,33 +125,38 @@
"LOCAL_RENDERSCRIPT_INCLUDES": "renderscript.include_dirs",
"LOCAL_RENDERSCRIPT_FLAGS": "renderscript.flags",
- "LOCAL_JAVA_RESOURCE_DIRS": "resource_dirs",
+ "LOCAL_JAVA_RESOURCE_DIRS": "java_resource_dirs",
"LOCAL_JAVACFLAGS": "javacflags",
"LOCAL_DX_FLAGS": "dxflags",
"LOCAL_JAVA_LIBRARIES": "libs",
"LOCAL_STATIC_JAVA_LIBRARIES": "static_libs",
- "LOCAL_AIDL_INCLUDES": "aidl.include_dirs",
"LOCAL_AAPT_FLAGS": "aaptflags",
"LOCAL_PACKAGE_SPLITS": "package_splits",
"LOCAL_COMPATIBILITY_SUITE": "test_suites",
+
+ "LOCAL_ANNOTATION_PROCESSORS": "annotation_processors",
+ "LOCAL_ANNOTATION_PROCESSOR_CLASSES": "annotation_processor_classes",
})
addStandardProperties(bpparser.BoolType,
map[string]string{
// Bool properties
- "LOCAL_IS_HOST_MODULE": "host",
- "LOCAL_CLANG": "clang",
- "LOCAL_FORCE_STATIC_EXECUTABLE": "static_executable",
- "LOCAL_NATIVE_COVERAGE": "native_coverage",
- "LOCAL_NO_CRT": "nocrt",
- "LOCAL_ALLOW_UNDEFINED_SYMBOLS": "allow_undefined_symbols",
- "LOCAL_RTTI_FLAG": "rtti",
- "LOCAL_NO_STANDARD_LIBRARIES": "no_standard_libraries",
- "LOCAL_PACK_MODULE_RELOCATIONS": "pack_relocations",
- "LOCAL_TIDY": "tidy",
- "LOCAL_PROPRIETARY_MODULE": "proprietary",
- "LOCAL_VENDOR_MODULE": "vendor",
-
+ "LOCAL_IS_HOST_MODULE": "host",
+ "LOCAL_CLANG": "clang",
+ "LOCAL_FORCE_STATIC_EXECUTABLE": "static_executable",
+ "LOCAL_NATIVE_COVERAGE": "native_coverage",
+ "LOCAL_NO_CRT": "nocrt",
+ "LOCAL_ALLOW_UNDEFINED_SYMBOLS": "allow_undefined_symbols",
+ "LOCAL_RTTI_FLAG": "rtti",
+ "LOCAL_NO_STANDARD_LIBRARIES": "no_standard_libs",
+ "LOCAL_PACK_MODULE_RELOCATIONS": "pack_relocations",
+ "LOCAL_TIDY": "tidy",
+ "LOCAL_PROPRIETARY_MODULE": "proprietary",
+ "LOCAL_VENDOR_MODULE": "vendor",
"LOCAL_EXPORT_PACKAGE_RESOURCES": "export_package_resources",
+
+ "LOCAL_DEX_PREOPT": "dex_preopt.enabled",
+ "LOCAL_DEX_PREOPT_APP_IMAGE": "dex_preopt.app_image",
+ "LOCAL_DEX_PREOPT_GENERATE_PROFILE": "dex_preopt.profile_guided",
})
}
@@ -207,7 +235,9 @@
return lists, nil
}
-func splitLocalGlobalPath(value bpparser.Expression) (string, bpparser.Expression, error) {
+// classifyLocalOrGlobalPath tells whether a file path should be interpreted relative to the current module (local)
+// or relative to the root of the source checkout (global)
+func classifyLocalOrGlobalPath(value bpparser.Expression) (string, bpparser.Expression, error) {
switch v := value.(type) {
case *bpparser.Variable:
if v.Name == "LOCAL_PATH" {
@@ -220,7 +250,7 @@
}
case *bpparser.Operator:
if v.Type() != bpparser.StringType {
- return "", nil, fmt.Errorf("splitLocalGlobalPath expected a string, got %s", value.Type)
+ return "", nil, fmt.Errorf("classifyLocalOrGlobalPath expected a string, got %s", value.Type)
}
if v.Operator != '+' {
@@ -251,68 +281,57 @@
case *bpparser.String:
return "global", value, nil
default:
- return "", nil, fmt.Errorf("splitLocalGlobalPath expected a string, got %s", value.Type)
+ return "", nil, fmt.Errorf("classifyLocalOrGlobalPath expected a string, got %s", value.Type)
}
}
+func sortedMapKeys(inputMap map[string]string) (sortedKeys []string) {
+ keys := make([]string, 0, len(inputMap))
+ for key := range inputMap {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+ return keys
+}
+
+// splitAndAssign splits a Make list into components and then
+// creates the corresponding variable assignments.
+func splitAndAssign(ctx variableAssignmentContext, splitFunc listSplitFunc, namesByClassification map[string]string) error {
+ val, err := makeVariableToBlueprint(ctx.file, ctx.mkvalue, bpparser.ListType)
+ if err != nil {
+ return err
+ }
+
+ lists, err := splitBpList(val, splitFunc)
+ if err != nil {
+ return err
+ }
+
+ for _, nameClassification := range sortedMapKeys(namesByClassification) {
+ name := namesByClassification[nameClassification]
+ if component, ok := lists[nameClassification]; ok && !emptyList(component) {
+ err = setVariable(ctx.file, ctx.append, ctx.prefix, name, component, true)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
func localIncludeDirs(ctx variableAssignmentContext) error {
- val, err := makeVariableToBlueprint(ctx.file, ctx.mkvalue, bpparser.ListType)
- if err != nil {
- return err
- }
-
- lists, err := splitBpList(val, splitLocalGlobalPath)
- if err != nil {
- return err
- }
-
- if global, ok := lists["global"]; ok && !emptyList(global) {
- err = setVariable(ctx.file, ctx.append, ctx.prefix, "include_dirs", global, true)
- if err != nil {
- return err
- }
- }
-
- if local, ok := lists["local"]; ok && !emptyList(local) {
- err = setVariable(ctx.file, ctx.append, ctx.prefix, "local_include_dirs", local, true)
- if err != nil {
- return err
- }
- }
-
- return nil
+ return splitAndAssign(ctx, classifyLocalOrGlobalPath, map[string]string{"global": "include_dirs", "local": "local_include_dirs"})
}
func exportIncludeDirs(ctx variableAssignmentContext) error {
- val, err := makeVariableToBlueprint(ctx.file, ctx.mkvalue, bpparser.ListType)
- if err != nil {
- return err
- }
-
- lists, err := splitBpList(val, splitLocalGlobalPath)
- if err != nil {
- return err
- }
-
- if local, ok := lists["local"]; ok && !emptyList(local) {
- err = setVariable(ctx.file, ctx.append, ctx.prefix, "export_include_dirs", local, true)
- if err != nil {
- return err
- }
- ctx.append = true
- }
-
// Add any paths that could not be converted to local relative paths to export_include_dirs
// anyways, they will cause an error if they don't exist and can be fixed manually.
- if global, ok := lists["global"]; ok && !emptyList(global) {
- err = setVariable(ctx.file, ctx.append, ctx.prefix, "export_include_dirs", global, true)
- if err != nil {
- return err
- }
- }
+ return splitAndAssign(ctx, classifyLocalOrGlobalPath, map[string]string{"global": "export_include_dirs", "local": "export_include_dirs"})
+}
- return nil
+func localAidlIncludes(ctx variableAssignmentContext) error {
+ return splitAndAssign(ctx, classifyLocalOrGlobalPath, map[string]string{"global": "aidl.include_dirs", "local": "aidl.local_include_dirs"})
}
func stem(ctx variableAssignmentContext) error {
@@ -360,7 +379,7 @@
}
if !inList("linux") && err == nil {
- err = setVariable(ctx.file, ctx.append, "target.linux", "enabled", falseValue, true)
+ err = setVariable(ctx.file, ctx.append, "target.linux_glibc", "enabled", falseValue, true)
}
if !inList("darwin") && err == nil {
@@ -370,50 +389,6 @@
return err
}
-func splitSrcsLogtags(value bpparser.Expression) (string, bpparser.Expression, error) {
- switch v := value.(type) {
- case *bpparser.Variable:
- // TODO: attempt to split variables?
- return "srcs", value, nil
- case *bpparser.Operator:
- // TODO: attempt to handle expressions?
- return "srcs", value, nil
- case *bpparser.String:
- if strings.HasSuffix(v.Value, ".logtags") {
- return "logtags", value, nil
- }
- return "srcs", value, nil
- default:
- return "", nil, fmt.Errorf("splitSrcsLogtags expected a string, got %s", value.Type())
- }
-
-}
-
-func srcFiles(ctx variableAssignmentContext) error {
- val, err := makeVariableToBlueprint(ctx.file, ctx.mkvalue, bpparser.ListType)
- if err != nil {
- return err
- }
-
- lists, err := splitBpList(val, splitSrcsLogtags)
-
- if srcs, ok := lists["srcs"]; ok && !emptyList(srcs) {
- err = setVariable(ctx.file, ctx.append, ctx.prefix, "srcs", srcs, true)
- if err != nil {
- return err
- }
- }
-
- if logtags, ok := lists["logtags"]; ok && !emptyList(logtags) {
- err = setVariable(ctx.file, true, ctx.prefix, "logtags", logtags, true)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
func sanitize(sub string) func(ctx variableAssignmentContext) error {
return func(ctx variableAssignmentContext) error {
val, err := makeVariableToBlueprint(ctx.file, ctx.mkvalue, bpparser.ListType)
@@ -461,7 +436,7 @@
}
func prebuiltClass(ctx variableAssignmentContext) error {
- class := ctx.mkvalue.Value(nil)
+ class := ctx.mkvalue.Value(ctx.file.scope)
if v, ok := prebuiltTypes[class]; ok {
ctx.file.scope.Set("BUILD_PREBUILT", v)
} else {
@@ -532,6 +507,26 @@
return nil
}
+func cflags(ctx variableAssignmentContext) error {
+ // The Soong replacement for CFLAGS doesn't need the same extra escaped quotes that were present in Make
+ ctx.mkvalue = ctx.mkvalue.Clone()
+ ctx.mkvalue.ReplaceLiteral(`\"`, `"`)
+ return includeVariableNow(bpVariable{"cflags", bpparser.ListType}, ctx)
+}
+
+func invert(name string) func(ctx variableAssignmentContext) error {
+ return func(ctx variableAssignmentContext) error {
+ val, err := makeVariableToBlueprint(ctx.file, ctx.mkvalue, bpparser.BoolType)
+ if err != nil {
+ return err
+ }
+
+ val.(*bpparser.Bool).Value = !val.(*bpparser.Bool).Value
+
+ return setVariable(ctx.file, ctx.append, ctx.prefix, name, val, true)
+ }
+}
+
// given a conditional, returns a function that will insert a variable assignment or not, based on the conditional
func includeVariableIf(bpVar bpVariable, conditional func(ctx variableAssignmentContext) bool) func(ctx variableAssignmentContext) error {
return func(ctx variableAssignmentContext) error {
@@ -592,7 +587,7 @@
// 64 must be after x86_64
{"64", "multilib.lib64"},
{"darwin", "target.darwin"},
- {"linux", "target.linux"},
+ {"linux", "target.linux_glibc"},
{"windows", "target.windows"},
}
@@ -610,11 +605,11 @@
true: "target.windows",
false: "target.not_windows"},
"($(HOST_OS),linux)": {
- true: "target.linux",
- false: "target.not_linux"},
+ true: "target.linux_glibc",
+ false: "target.not_linux_glibc"},
"($(HOST_OS), linux)": {
- true: "target.linux",
- false: "target.not_linux"},
+ true: "target.linux_glibc",
+ false: "target.not_linux_glibc"},
"($(BUILD_OS),darwin)": {
true: "target.darwin",
false: "target.not_darwin"},
@@ -622,13 +617,15 @@
true: "target.darwin",
false: "target.not_darwin"},
"($(BUILD_OS),linux)": {
- true: "target.linux",
- false: "target.not_linux"},
+ true: "target.linux_glibc",
+ false: "target.not_linux_glibc"},
"($(BUILD_OS), linux)": {
- true: "target.linux",
- false: "target.not_linux"},
+ true: "target.linux_glibc",
+ false: "target.not_linux_glibc"},
"(,$(TARGET_BUILD_APPS))": {
false: "product_variables.unbundled_build"},
+ "($(TARGET_BUILD_APPS),)": {
+ false: "product_variables.unbundled_build"},
"($(TARGET_BUILD_PDK),true)": {
true: "product_variables.pdk"},
"($(TARGET_BUILD_PDK), true)": {
@@ -648,10 +645,23 @@
return fmt.Sprintf("%s/**/*.java", dir)
}
+func allProtoFilesUnder(args []string) string {
+ dir := ""
+ if len(args) > 0 {
+ dir = strings.TrimSpace(args[0])
+ }
+
+ return fmt.Sprintf("%s/**/*.proto", dir)
+}
+
func allSubdirJavaFiles(args []string) string {
return "**/*.java"
}
+func includeIgnored(args []string) string {
+ return include_ignored
+}
+
var moduleTypes = map[string]string{
"BUILD_SHARED_LIBRARY": "cc_library_shared",
"BUILD_STATIC_LIBRARY": "cc_library_static",
@@ -676,7 +686,7 @@
"SHARED_LIBRARIES": "cc_prebuilt_library_shared",
"STATIC_LIBRARIES": "cc_prebuilt_library_static",
"EXECUTABLES": "cc_prebuilt_binary",
- "JAVA_LIBRARIES": "prebuilt_java_library",
+ "JAVA_LIBRARIES": "java_import",
}
var soongModuleTypes = map[string]bool{}
@@ -686,7 +696,12 @@
globalScope.Set("CLEAR_VARS", clear_vars)
globalScope.SetFunc("my-dir", mydir)
globalScope.SetFunc("all-java-files-under", allJavaFilesUnder)
+ globalScope.SetFunc("all-proto-files-under", allProtoFilesUnder)
globalScope.SetFunc("all-subdir-java-files", allSubdirJavaFiles)
+ globalScope.SetFunc("all-makefiles-under", includeIgnored)
+ globalScope.SetFunc("first-makefiles-under", includeIgnored)
+ globalScope.SetFunc("all-named-subdir-makefiles", includeIgnored)
+ globalScope.SetFunc("all-subdir-makefiles", includeIgnored)
for k, v := range moduleTypes {
globalScope.Set(k, v)
diff --git a/androidmk/cmd/androidmk/androidmk.go b/androidmk/cmd/androidmk/androidmk.go
index d26643a..385690c 100644
--- a/androidmk/cmd/androidmk/androidmk.go
+++ b/androidmk/cmd/androidmk/androidmk.go
@@ -1,7 +1,22 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package main
import (
"bytes"
+ "flag"
"fmt"
"io/ioutil"
"os"
@@ -15,6 +30,13 @@
bpparser "github.com/google/blueprint/parser"
)
+var usage = func() {
+ fmt.Fprintf(os.Stderr, "usage: androidmk [flags] <inputFile>\n"+
+ "\nandroidmk parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n")
+ flag.PrintDefaults()
+ os.Exit(1)
+}
+
// TODO: non-expanded variables with expressions
type bpFile struct {
@@ -48,10 +70,11 @@
f.bpPos.Line++
}
-func (f *bpFile) errorf(node mkparser.Node, s string, args ...interface{}) {
- orig := node.Dump()
- s = fmt.Sprintf(s, args...)
- f.insertExtraComment(fmt.Sprintf("// ANDROIDMK TRANSLATION ERROR: %s", s))
+// records that the given node failed to be converted and includes an explanatory message
+func (f *bpFile) errorf(failedNode mkparser.Node, message string, args ...interface{}) {
+ orig := failedNode.Dump()
+ message = fmt.Sprintf(message, args...)
+ f.addErrorText(fmt.Sprintf("// ANDROIDMK TRANSLATION ERROR: %s", message))
lines := strings.Split(orig, "\n")
for _, l := range lines {
@@ -59,12 +82,35 @@
}
}
+// records that something unexpected occurred
+func (f *bpFile) warnf(message string, args ...interface{}) {
+ message = fmt.Sprintf(message, args...)
+ f.addErrorText(fmt.Sprintf("// ANDROIDMK TRANSLATION WARNING: %s", message))
+}
+
+// adds the given error message as-is to the bottom of the (in-progress) file
+func (f *bpFile) addErrorText(message string) {
+ f.insertExtraComment(message)
+}
+
func (f *bpFile) setMkPos(pos, end scanner.Position) {
- if pos.Line < f.mkPos.Line {
- panic(fmt.Errorf("out of order lines, %q after %q", pos, f.mkPos))
+ // It is unusual but not forbidden for pos.Line to be smaller than f.mkPos.Line
+ // For example:
+ //
+ // if true # this line is emitted 1st
+ // if true # this line is emitted 2nd
+ // some-target: some-file # this line is emitted 3rd
+ // echo doing something # this recipe is emitted 6th
+ // endif #some comment # this endif is emitted 4th; this comment is part of the recipe
+ // echo doing more stuff # this is part of the recipe
+ // endif # this endif is emitted 5th
+ //
+ // However, if pos.Line < f.mkPos.Line, we treat it as though it were equal
+ if pos.Line >= f.mkPos.Line {
+ f.bpPos.Line += (pos.Line - f.mkPos.Line)
+ f.mkPos = end
}
- f.bpPos.Line += (pos.Line - f.mkPos.Line)
- f.mkPos = end
+
}
type conditional struct {
@@ -73,7 +119,13 @@
}
func main() {
- b, err := ioutil.ReadFile(os.Args[1])
+ flag.Usage = usage
+ flag.Parse()
+ if len(flag.Args()) != 1 {
+ usage()
+ }
+ filePathToRead := flag.Arg(0)
+ b, err := ioutil.ReadFile(filePathToRead)
if err != nil {
fmt.Println(err.Error())
return
@@ -125,6 +177,9 @@
makeModule(file, val)
case val == clear_vars:
resetModule(file)
+ case val == include_ignored:
+ // subdirs are already automatically included in Soong
+ continue
default:
file.errorf(x, "unsupported include")
continue
@@ -358,6 +413,10 @@
*oldValue = val
} else {
names := strings.Split(name, ".")
+ if file.module == nil {
+ file.warnf("No 'include $(CLEAR_VARS)' detected before first assignment; clearing vars now")
+ resetModule(file)
+ }
container := &file.module.Properties
for i, n := range names[:len(names)-1] {
diff --git a/androidmk/cmd/androidmk/androidmk_test.go b/androidmk/cmd/androidmk/androidmk_test.go
index 5fbc951..22a52d4 100644
--- a/androidmk/cmd/androidmk/androidmk_test.go
+++ b/androidmk/cmd/androidmk/androidmk_test.go
@@ -213,35 +213,6 @@
`,
},
{
- desc: "*.logtags in LOCAL_SRC_FILES",
- in: `
-include $(CLEAR_VARS)
-LOCAL_SRC_FILES := events.logtags
-LOCAL_SRC_FILES += a.c events2.logtags
-include $(BUILD_SHARED_LIBRARY)
-`,
- expected: `
-cc_library_shared {
- logtags: ["events.logtags"] + ["events2.logtags"],
- srcs: ["a.c"],
-}
-`,
- },
- {
- desc: "LOCAL_LOGTAGS_FILES and *.logtags in LOCAL_SRC_FILES",
- in: `
-include $(CLEAR_VARS)
-LOCAL_LOGTAGS_FILES := events.logtags
-LOCAL_SRC_FILES := events2.logtags
-include $(BUILD_SHARED_LIBRARY)
-`,
- expected: `
-cc_library_shared {
- logtags: ["events.logtags"] + ["events2.logtags"],
-}
-`,
- },
- {
desc: "_<OS> suffixes",
in: `
include $(CLEAR_VARS)
@@ -256,7 +227,7 @@
darwin: {
srcs: ["darwin.c"],
},
- linux: {
+ linux_glibc: {
srcs: ["linux.c"],
},
windows: {
@@ -374,6 +345,99 @@
}
`,
},
+
+ {
+ desc: "Input containing escaped quotes",
+ in: `
+include $(CLEAR_VARS)
+LOCAL_MODULE:= libsensorservice
+LOCAL_CFLAGS:= -DLOG_TAG=\"-DDontEscapeMe\"
+LOCAL_SRC_FILES := \"EscapeMe.cc\"
+include $(BUILD_SHARED_LIBRARY)
+`,
+
+ expected: `
+cc_library_shared {
+ name: "libsensorservice",
+ cflags: ["-DLOG_TAG=\"-DDontEscapeMe\""],
+ srcs: ["\\\"EscapeMe.cc\\\""],
+}
+`,
+ },
+ {
+
+ desc: "Don't fail on missing CLEAR_VARS",
+ in: `
+LOCAL_MODULE := iAmAModule
+include $(BUILD_SHARED_LIBRARY)`,
+
+ expected: `
+// ANDROIDMK TRANSLATION WARNING: No 'include $(CLEAR_VARS)' detected before first assignment; clearing vars now
+cc_library_shared {
+ name: "iAmAModule",
+
+}`,
+ },
+ {
+
+ desc: "LOCAL_AIDL_INCLUDES",
+ in: `
+include $(CLEAR_VARS)
+LOCAL_MODULE := iAmAModule
+LOCAL_AIDL_INCLUDES := $(LOCAL_PATH)/src/main/java system/core
+include $(BUILD_SHARED_LIBRARY)`,
+
+ expected: `
+cc_library_shared {
+ name: "iAmAModule",
+ aidl: {
+ include_dirs: ["system/core"],
+ local_include_dirs: ["src/main/java"],
+ }
+}`,
+ },
+ {
+ // the important part of this test case is that it confirms that androidmk doesn't
+ // panic in this case
+ desc: "multiple directives inside recipe",
+ in: `
+ifeq ($(a),true)
+ifeq ($(b),false)
+imABuildStatement: somefile
+ echo begin
+endif # a==true
+ echo middle
+endif # b==false
+ echo end
+`,
+ expected: `
+// ANDROIDMK TRANSLATION ERROR: unsupported conditional
+// ifeq ($(a),true)
+
+// ANDROIDMK TRANSLATION ERROR: unsupported conditional
+// ifeq ($(b),false)
+
+// ANDROIDMK TRANSLATION ERROR: unsupported line
+// rule: imABuildStatement: somefile
+// echo begin
+// # a==true
+// echo middle
+// # b==false
+// echo end
+//
+// ANDROIDMK TRANSLATION ERROR: endif from unsupported contitional
+// endif
+// ANDROIDMK TRANSLATION ERROR: endif from unsupported contitional
+// endif
+ `,
+ },
+ {
+ desc: "ignore all-makefiles-under",
+ in: `
+include $(call all-makefiles-under,$(LOCAL_PATH))
+`,
+ expected: ``,
+ },
}
func reformatBlueprint(input string) string {
diff --git a/androidmk/cmd/androidmk/values.go b/androidmk/cmd/androidmk/values.go
index 0074bb9..d240a01 100644
--- a/androidmk/cmd/androidmk/values.go
+++ b/androidmk/cmd/androidmk/values.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package main
import (
diff --git a/androidmk/parser/ast.go b/androidmk/parser/ast.go
index 07e1842..d5d1354 100644
--- a/androidmk/parser/ast.go
+++ b/androidmk/parser/ast.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package parser
type Pos int
diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go
index 00d331b..e6885a8 100644
--- a/androidmk/parser/make_strings.go
+++ b/androidmk/parser/make_strings.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package parser
import (
@@ -29,6 +43,11 @@
}
}
+func (ms *MakeString) Clone() (result *MakeString) {
+ clone := *ms
+ return &clone
+}
+
func (ms *MakeString) Pos() Pos {
return ms.StringPos
}
@@ -164,6 +183,12 @@
return s[len(s)-1] == uint8(ch)
}
+func (ms *MakeString) ReplaceLiteral(input string, output string) {
+ for i := range ms.Strings {
+ ms.Strings[i] = strings.Replace(ms.Strings[i], input, output, -1)
+ }
+}
+
func splitAnyN(s, sep string, n int) []string {
ret := []string{}
for n == -1 || n > 1 {
diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go
index 5636b79..8ad3d74 100644
--- a/androidmk/parser/make_strings_test.go
+++ b/androidmk/parser/make_strings_test.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package parser
import (
diff --git a/androidmk/parser/parser.go b/androidmk/parser/parser.go
index ef5b492..89ee308 100644
--- a/androidmk/parser/parser.go
+++ b/androidmk/parser/parser.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package parser
import (
diff --git a/androidmk/parser/scope.go b/androidmk/parser/scope.go
index 5e94ea5..7a514fa 100644
--- a/androidmk/parser/scope.go
+++ b/androidmk/parser/scope.go
@@ -1,6 +1,22 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package parser
-import "strings"
+import (
+ "strings"
+)
type Scope interface {
Get(name string) string
@@ -84,6 +100,9 @@
if ret, ok := v.EvalFunction(scope); ok {
return ret
}
+ if scope == nil {
+ panic("Cannot take the value of a variable in a nil scope")
+ }
return scope.Get(v.Name.Value(scope))
}
diff --git a/bootstrap.bash b/bootstrap.bash
index dbc6eb2..4db8539 100755
--- a/bootstrap.bash
+++ b/bootstrap.bash
@@ -1,53 +1,23 @@
#!/bin/bash
-set -e
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-ORIG_SRCDIR=$(dirname "${BASH_SOURCE[0]}")
-if [[ "$ORIG_SRCDIR" != "." ]]; then
- if [[ ! -z "$BUILDDIR" ]]; then
- echo "error: To use BUILDDIR, run from the source directory"
- exit 1
- fi
- export BUILDDIR=$("${ORIG_SRCDIR}/build/soong/scripts/reverse_path.py" "$ORIG_SRCDIR")
- cd $ORIG_SRCDIR
-fi
-if [[ -z "$BUILDDIR" ]]; then
- echo "error: Run ${BASH_SOURCE[0]} from the build output directory"
- exit 1
-fi
-export SRCDIR="."
-export BOOTSTRAP="${SRCDIR}/bootstrap.bash"
-export BLUEPRINTDIR="${SRCDIR}/build/blueprint"
+echo '==== ERROR: bootstrap.bash & ./soong are obsolete ====' >&2
+echo 'Use `m --skip-make` with a standalone OUT_DIR instead.' >&2
+echo 'Without envsetup.sh, use:' >&2
+echo ' build/soong/soong_ui.bash --make-mode --skip-make' >&2
+echo '======================================================' >&2
+exit 1
-export TOPNAME="Android.bp"
-export RUN_TESTS="-t"
-
-case $(uname) in
- Linux)
- export PREBUILTOS="linux-x86"
- ;;
- Darwin)
- export PREBUILTOS="darwin-x86"
- ;;
- *) echo "unknown OS:" $(uname) && exit 1;;
-esac
-export GOROOT="${SRCDIR}/prebuilts/go/$PREBUILTOS"
-
-if [[ $# -eq 0 ]]; then
- mkdir -p $BUILDDIR
-
- if [[ $(find $BUILDDIR -maxdepth 1 -name Android.bp) ]]; then
- echo "FAILED: The build directory must not be a source directory"
- exit 1
- fi
-
- export SRCDIR_FROM_BUILDDIR=$(build/soong/scripts/reverse_path.py "$BUILDDIR")
-
- sed -e "s|@@BuildDir@@|${BUILDDIR}|" \
- -e "s|@@SrcDirFromBuildDir@@|${SRCDIR_FROM_BUILDDIR}|" \
- -e "s|@@PrebuiltOS@@|${PREBUILTOS}|" \
- "$SRCDIR/build/soong/soong.bootstrap.in" > $BUILDDIR/.soong.bootstrap
- ln -sf "${SRCDIR_FROM_BUILDDIR}/build/soong/soong.bash" $BUILDDIR/soong
-fi
-
-"$SRCDIR/build/blueprint/bootstrap.bash" "$@"
diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go
index 1249494..fcd4fec 100644
--- a/bpfix/bpfix/bpfix.go
+++ b/bpfix/bpfix/bpfix.go
@@ -19,6 +19,7 @@
import (
"bytes"
"fmt"
+
"github.com/google/blueprint/parser"
)
@@ -26,7 +27,6 @@
// A FixRequest doesn't specify whether to do a dry run or where to write the results; that's in cmd/bpfix.go
type FixRequest struct {
simplifyKnownRedundantVariables bool
- removeEmptyLists bool
}
func NewFixRequest() FixRequest {
@@ -36,7 +36,6 @@
func (r FixRequest) AddAll() (result FixRequest) {
result = r
result.simplifyKnownRedundantVariables = true
- result.removeEmptyLists = true
return result
}
@@ -89,12 +88,6 @@
return nil, err
}
}
- if config.removeEmptyLists {
- tree, err = removePropertiesHavingTheirDefaultValues(tree)
- if err != nil {
- return nil, err
- }
- }
return tree, err
}
@@ -154,32 +147,3 @@
}
return tree, nil
}
-
-func removePropertiesHavingTheirDefaultValues(tree *parser.File) (fixed *parser.File, err error) {
- for _, def := range tree.Defs {
- mod, ok := def.(*parser.Module)
- if !ok {
- continue
- }
- writeIndex := 0
- for _, prop := range mod.Properties {
- val := prop.Value
- keep := true
- switch val := val.(type) {
- case *parser.List:
- if len(val.Values) == 0 {
- keep = false
- }
- break
- default:
- keep = true
- }
- if keep {
- mod.Properties[writeIndex] = prop
- writeIndex++
- }
- }
- mod.Properties = mod.Properties[:writeIndex]
- }
- return tree, nil
-}
diff --git a/build_test.bash b/build_test.bash
index 065d7f6..4c43224 100755
--- a/build_test.bash
+++ b/build_test.bash
@@ -28,11 +28,14 @@
export TRACE_BEGIN_SOONG=$(date +%s%N)
export TOP=$(cd $(dirname ${BASH_SOURCE[0]})/../..; PWD= /bin/pwd)
+cd "${TOP}"
source "${TOP}/build/soong/scripts/microfactory.bash"
case $(uname) in
Linux)
export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
+ export SEGFAULT_USE_ALTSTACK=1
+ ulimit -a
;;
esac
diff --git a/cc/androidmk.go b/cc/androidmk.go
index 940e7c7..1db5373 100644
--- a/cc/androidmk.go
+++ b/cc/androidmk.go
@@ -30,7 +30,7 @@
type AndroidMkContext interface {
Target() android.Target
subAndroidMk(*android.AndroidMkData, interface{})
- vndk() bool
+ useVndk() bool
}
type subAndroidMkProvider interface {
@@ -49,30 +49,38 @@
}
}
-func (c *Module) AndroidMk() (ret android.AndroidMkData, err error) {
+func (c *Module) AndroidMk() android.AndroidMkData {
if c.Properties.HideFromMake {
- ret.Disabled = true
- return ret, nil
+ return android.AndroidMkData{
+ Disabled: true,
+ }
}
- ret.OutputFile = c.outputFile
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) (err error) {
- fmt.Fprintln(w, "LOCAL_SANITIZE := never")
- if len(c.Properties.AndroidMkSharedLibs) > 0 {
- fmt.Fprintln(w, "LOCAL_SHARED_LIBRARIES := "+strings.Join(c.Properties.AndroidMkSharedLibs, " "))
- }
- if c.Target().Os == android.Android && c.Properties.Sdk_version != "" && !c.vndk() {
- fmt.Fprintln(w, "LOCAL_SDK_VERSION := "+c.Properties.Sdk_version)
- fmt.Fprintln(w, "LOCAL_NDK_STL_VARIANT := none")
- } else {
- // These are already included in LOCAL_SHARED_LIBRARIES
- fmt.Fprintln(w, "LOCAL_CXX_STL := none")
- }
- if c.vndk() {
- fmt.Fprintln(w, "LOCAL_USE_VNDK := true")
- }
- return nil
- })
+ ret := android.AndroidMkData{
+ OutputFile: c.outputFile,
+ Extra: []android.AndroidMkExtraFunc{
+ func(w io.Writer, outputFile android.Path) {
+ if len(c.Properties.Logtags) > 0 {
+ fmt.Fprintln(w, "LOCAL_LOGTAGS_FILES :=", strings.Join(c.Properties.Logtags, " "))
+ }
+ fmt.Fprintln(w, "LOCAL_SANITIZE := never")
+ if len(c.Properties.AndroidMkSharedLibs) > 0 {
+ fmt.Fprintln(w, "LOCAL_SHARED_LIBRARIES := "+strings.Join(c.Properties.AndroidMkSharedLibs, " "))
+ }
+ if c.Target().Os == android.Android &&
+ String(c.Properties.Sdk_version) != "" && !c.useVndk() {
+ fmt.Fprintln(w, "LOCAL_SDK_VERSION := "+String(c.Properties.Sdk_version))
+ fmt.Fprintln(w, "LOCAL_NDK_STL_VARIANT := none")
+ } else {
+ // These are already included in LOCAL_SHARED_LIBRARIES
+ fmt.Fprintln(w, "LOCAL_CXX_STL := none")
+ }
+ if c.useVndk() {
+ fmt.Fprintln(w, "LOCAL_USE_VNDK := true")
+ }
+ },
+ },
+ }
for _, feature := range c.features {
c.subAndroidMk(&ret, feature)
@@ -85,13 +93,13 @@
}
c.subAndroidMk(&ret, c.installer)
- if c.vndk() && Bool(c.Properties.Vendor_available) {
+ if c.useVndk() && c.hasVendorVariant() {
// .vendor suffix is added only when we will have two variants: core and vendor.
// The suffix is not added for vendor-only module.
ret.SubName += vendorSuffix
}
- return ret, nil
+ return ret
}
func androidMkWriteTestData(data android.Paths, ctx AndroidMkContext, ret *android.AndroidMkData) {
@@ -106,9 +114,8 @@
testFiles = append(testFiles, path+":"+rel)
}
if len(testFiles) > 0 {
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
fmt.Fprintln(w, "LOCAL_TEST_DATA := "+strings.Join(testFiles, " "))
- return nil
})
}
}
@@ -133,10 +140,10 @@
ret.Class = "SHARED_LIBRARIES"
} else if library.header() {
- ret.Custom = func(w io.Writer, name, prefix, moduleDir string) error {
+ ret.Custom = func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)")
fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
- fmt.Fprintln(w, "LOCAL_MODULE :=", name)
+ fmt.Fprintln(w, "LOCAL_MODULE :=", name+data.SubName)
archStr := ctx.Target().Arch.ArchType.String()
var host bool
@@ -152,22 +159,24 @@
}
if host {
- fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", ctx.Target().Os.String())
+ makeOs := ctx.Target().Os.String()
+ if ctx.Target().Os == android.Linux || ctx.Target().Os == android.LinuxBionic {
+ makeOs = "linux"
+ }
+ fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", makeOs)
fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true")
- } else if ctx.vndk() {
+ } else if ctx.useVndk() {
fmt.Fprintln(w, "LOCAL_USE_VNDK := true")
}
library.androidMkWriteExportedFlags(w)
fmt.Fprintln(w, "include $(BUILD_HEADER_LIBRARY)")
-
- return nil
}
return
}
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
library.androidMkWriteExportedFlags(w)
fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES := ")
if library.sAbiOutputFile.Valid() {
@@ -185,8 +194,6 @@
if library.coverageOutputFile.Valid() {
fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", library.coverageOutputFile.String())
}
-
- return nil
})
if library.shared() {
@@ -195,13 +202,11 @@
}
func (object *objectLinker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
- ret.Custom = func(w io.Writer, name, prefix, moduleDir string) error {
+ ret.Custom = func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
out := ret.OutputFile.Path()
- fmt.Fprintln(w, "\n$("+prefix+"OUT_INTERMEDIATE_LIBRARIES)/"+name+objectExtension+":", out.String())
+ fmt.Fprintln(w, "\n$("+prefix+"OUT_INTERMEDIATE_LIBRARIES)/"+name+data.SubName+objectExtension+":", out.String())
fmt.Fprintln(w, "\t$(copy-file-to-target)")
-
- return nil
}
}
@@ -210,7 +215,7 @@
ctx.subAndroidMk(ret, &binary.stripper)
ret.Class = "EXECUTABLES"
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
fmt.Fprintln(w, "LOCAL_SYSTEM_SHARED_LIBRARIES :=")
if Bool(binary.Properties.Static_executable) {
fmt.Fprintln(w, "LOCAL_FORCE_STATIC_EXECUTABLE := true")
@@ -223,19 +228,17 @@
if binary.coverageOutputFile.Valid() {
fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", binary.coverageOutputFile.String())
}
- return nil
})
}
func (benchmark *benchmarkDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
ctx.subAndroidMk(ret, benchmark.binaryDecorator)
ret.Class = "NATIVE_TESTS"
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
if len(benchmark.Properties.Test_suites) > 0 {
fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=",
strings.Join(benchmark.Properties.Test_suites, " "))
}
- return nil
})
androidMkWriteTestData(benchmark.data, ctx, ret)
@@ -245,15 +248,14 @@
ctx.subAndroidMk(ret, test.binaryDecorator)
ret.Class = "NATIVE_TESTS"
if Bool(test.Properties.Test_per_src) {
- ret.SubName = "_" + test.binaryDecorator.Properties.Stem
+ ret.SubName = "_" + String(test.binaryDecorator.Properties.Stem)
}
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
if len(test.Properties.Test_suites) > 0 {
fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=",
strings.Join(test.Properties.Test_suites, " "))
}
- return nil
})
androidMkWriteTestData(test.data, ctx, ret)
@@ -265,11 +267,9 @@
func (library *toolchainLibraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
ret.Class = "STATIC_LIBRARIES"
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+outputFile.Ext())
fmt.Fprintln(w, "LOCAL_SYSTEM_SHARED_LIBRARIES :=")
-
- return nil
})
}
@@ -279,25 +279,23 @@
return
}
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
- if stripper.StripProperties.Strip.None {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
+ if Bool(stripper.StripProperties.Strip.None) {
+
fmt.Fprintln(w, "LOCAL_STRIP_MODULE := false")
- } else if stripper.StripProperties.Strip.Keep_symbols {
+ } else if Bool(stripper.StripProperties.Strip.Keep_symbols) {
fmt.Fprintln(w, "LOCAL_STRIP_MODULE := keep_symbols")
} else {
fmt.Fprintln(w, "LOCAL_STRIP_MODULE := mini-debug-info")
}
-
- return nil
})
}
func (packer *relocationPacker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
if packer.Properties.PackingRelocations {
fmt.Fprintln(w, "LOCAL_PACK_MODULE_RELOCATIONS := true")
}
- return nil
})
}
@@ -308,14 +306,13 @@
ret.OutputFile = android.OptionalPathForPath(installer.path)
}
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
path := installer.path.RelPathString()
dir, file := filepath.Split(path)
stem := strings.TrimSuffix(file, filepath.Ext(file))
fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+filepath.Ext(file))
fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir))
fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem)
- return nil
})
}
@@ -323,8 +320,8 @@
ret.SubName = ndkLibrarySuffix + "." + c.properties.ApiLevel
ret.Class = "SHARED_LIBRARIES"
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
- path, file := filepath.Split(c.installPath)
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
+ path, file := filepath.Split(c.installPath.String())
stem := strings.TrimSuffix(file, filepath.Ext(file))
fmt.Fprintln(w, "LOCAL_SYSTEM_SHARED_LIBRARIES :=")
fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+outputFile.Ext())
@@ -336,7 +333,6 @@
// dozens of libraries with the same name, they'll clobber each other
// and the real versions of the libraries from the platform).
fmt.Fprintln(w, "LOCAL_COPY_TO_INTERMEDIATE_LIBRARIES := false")
- return nil
})
}
@@ -344,7 +340,7 @@
ret.Class = "SHARED_LIBRARIES"
ret.SubName = ".vendor"
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
c.libraryDecorator.androidMkWriteExportedFlags(w)
fmt.Fprintln(w, "LOCAL_BUILT_MODULE_STEM := $(LOCAL_MODULE)"+outputFile.Ext())
@@ -353,7 +349,26 @@
fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true")
fmt.Fprintln(w, "LOCAL_USE_VNDK := true")
+ })
+}
- return nil
+func (c *vndkPrebuiltLibraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
+ ret.Class = "SHARED_LIBRARIES"
+
+ ret.SubName = vndkSuffix + c.version()
+
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
+ c.libraryDecorator.androidMkWriteExportedFlags(w)
+
+ path := c.path.RelPathString()
+ dir, file := filepath.Split(path)
+ stem := strings.TrimSuffix(file, filepath.Ext(file))
+ fmt.Fprintln(w, "LOCAL_STRIP_MODULE := false")
+ fmt.Fprintln(w, "LOCAL_SYSTEM_SHARED_LIBRARIES :=")
+ fmt.Fprintln(w, "LOCAL_USE_VNDK := true")
+ fmt.Fprintln(w, "LOCAL_BUILT_MODULE_STEM := $(LOCAL_MODULE)"+outputFile.Ext())
+ fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+filepath.Ext(file))
+ fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir))
+ fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem)
})
}
diff --git a/cc/binary.go b/cc/binary.go
index f6e62b7..206237a 100644
--- a/cc/binary.go
+++ b/cc/binary.go
@@ -15,10 +15,6 @@
package cc
import (
- "path/filepath"
-
- "github.com/google/blueprint/proptools"
-
"android/soong/android"
)
@@ -27,16 +23,19 @@
Static_executable *bool `android:"arch_variant"`
// set the name of the output
- Stem string `android:"arch_variant"`
+ Stem *string `android:"arch_variant"`
// append to the name of the output
- Suffix string `android:"arch_variant"`
+ Suffix *string `android:"arch_variant"`
// if set, add an extra objcopy --prefix-symbols= step
- Prefix_symbols string
+ Prefix_symbols *string
+
+ // local file name to pass to the linker as --version_script
+ Version_script *string `android:"arch_variant"`
// if set, install a symlink to the preferred architecture
- Symlink_preferred_arch bool
+ Symlink_preferred_arch *bool
// install symlinks to the binary. Symlink names will have the suffix and the binary
// extension (if any) appended
@@ -96,18 +95,18 @@
func (binary *binaryDecorator) getStem(ctx BaseModuleContext) string {
stem := ctx.baseModuleName()
- if binary.Properties.Stem != "" {
- stem = binary.Properties.Stem
+ if String(binary.Properties.Stem) != "" {
+ stem = String(binary.Properties.Stem)
}
- return stem + binary.Properties.Suffix
+ return stem + String(binary.Properties.Suffix)
}
func (binary *binaryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps {
deps = binary.baseLinker.linkerDeps(ctx, deps)
if ctx.toolchain().Bionic() {
if !Bool(binary.baseLinker.Properties.Nocrt) {
- if !ctx.sdk() {
+ if !ctx.useSdk() {
if binary.static() {
deps.CrtBegin = "crtbegin_static"
} else {
@@ -149,6 +148,10 @@
[]string{"libc", "libc_nomalloc", "libcompiler_rt"})
deps.LateStaticLibs = append(groupLibs, deps.LateStaticLibs...)
}
+
+ if ctx.Os() == android.LinuxBionic && !binary.static() {
+ deps.LinkerScript = "host_bionic_linker_script"
+ }
}
if !binary.static() && inList("libc", deps.StaticLibs) {
@@ -179,8 +182,8 @@
if !ctx.toolchain().Bionic() {
if ctx.Os() == android.Linux {
- if binary.Properties.Static_executable == nil && Bool(ctx.AConfig().ProductVariables.HostStaticBinaries) {
- binary.Properties.Static_executable = proptools.BoolPtr(true)
+ if binary.Properties.Static_executable == nil && Bool(ctx.Config().ProductVariables.HostStaticBinaries) {
+ binary.Properties.Static_executable = BoolPtr(true)
}
} else {
// Static executables are not supported on Darwin or Windows
@@ -201,7 +204,7 @@
flags = binary.baseLinker.linkerFlags(ctx, flags)
if ctx.Host() && !binary.static() {
- if !ctx.AConfig().IsEnvTrue("DISABLE_HOST_PIE") {
+ if !ctx.Config().IsEnvTrue("DISABLE_HOST_PIE") {
flags.LdFlags = append(flags.LdFlags, "-pie")
if ctx.Windows() {
flags.LdFlags = append(flags.LdFlags, "-Wl,-e_mainCRTStartup")
@@ -231,7 +234,6 @@
"-Bstatic",
"-Wl,--gc-sections",
)
-
} else {
if flags.DynamicLinker == "" {
if binary.Properties.DynamicLinker != "" {
@@ -241,15 +243,7 @@
case android.Android:
flags.DynamicLinker = "/system/bin/linker"
case android.LinuxBionic:
- // The linux kernel expects the linker to be an
- // absolute path
- path := android.PathForOutput(ctx,
- "host", "linux_bionic-x86", "bin", "linker")
- if p, err := filepath.Abs(path.String()); err == nil {
- flags.DynamicLinker = p
- } else {
- ctx.ModuleErrorf("can't find path to dynamic linker: %q", err)
- }
+ flags.DynamicLinker = ""
default:
ctx.ModuleErrorf("unknown dynamic linker")
}
@@ -266,6 +260,7 @@
"-Wl,--gc-sections",
"-Wl,-z,nocopyreloc",
)
+
}
} else {
if binary.static() {
@@ -282,6 +277,7 @@
func (binary *binaryDecorator) link(ctx ModuleContext,
flags Flags, deps PathDeps, objs Objects) android.Path {
+ versionScript := android.OptionalPathForModuleSrc(ctx, binary.Properties.Version_script)
fileName := binary.getStem(ctx) + flags.Toolchain.ExecutableSuffix()
outputFile := android.PathForModuleOut(ctx, fileName)
ret := outputFile
@@ -291,6 +287,20 @@
sharedLibs := deps.SharedLibs
sharedLibs = append(sharedLibs, deps.LateSharedLibs...)
+ if versionScript.Valid() {
+ if ctx.Darwin() {
+ ctx.PropertyErrorf("version_script", "Not supported on Darwin")
+ } else {
+ flags.LdFlags = append(flags.LdFlags, "-Wl,--version-script,"+versionScript.String())
+ linkerDeps = append(linkerDeps, versionScript.Path())
+ }
+ }
+
+ if deps.LinkerScript.Valid() {
+ flags.LdFlags = append(flags.LdFlags, "-Wl,-T,"+deps.LinkerScript.String())
+ linkerDeps = append(linkerDeps, deps.LinkerScript.Path())
+ }
+
if flags.DynamicLinker != "" {
flags.LdFlags = append(flags.LdFlags, " -Wl,-dynamic-linker,"+flags.DynamicLinker)
}
@@ -303,16 +313,17 @@
binary.stripper.strip(ctx, outputFile, strippedOutputFile, builderFlags)
}
- if binary.Properties.Prefix_symbols != "" {
+ if String(binary.Properties.Prefix_symbols) != "" {
afterPrefixSymbols := outputFile
outputFile = android.PathForModuleOut(ctx, "unprefixed", fileName)
- TransformBinaryPrefixSymbols(ctx, binary.Properties.Prefix_symbols, outputFile,
+ TransformBinaryPrefixSymbols(ctx, String(binary.Properties.Prefix_symbols), outputFile,
flagsToBuilderFlags(flags), afterPrefixSymbols)
}
linkerDeps = append(linkerDeps, deps.SharedLibsDeps...)
linkerDeps = append(linkerDeps, deps.LateSharedLibsDeps...)
linkerDeps = append(linkerDeps, objs.tidyFiles...)
+ linkerDeps = append(linkerDeps, flags.LdFlagsDeps...)
TransformObjToDynamicBinary(ctx, objs.objFiles, sharedLibs, deps.StaticLibs,
deps.LateStaticLibs, deps.WholeStaticLibs, linkerDeps, deps.CrtBegin, deps.CrtEnd, true,
@@ -329,11 +340,11 @@
binary.baseInstaller.install(ctx, file)
for _, symlink := range binary.Properties.Symlinks {
binary.symlinks = append(binary.symlinks,
- symlink+binary.Properties.Suffix+ctx.toolchain().ExecutableSuffix())
+ symlink+String(binary.Properties.Suffix)+ctx.toolchain().ExecutableSuffix())
}
- if binary.Properties.Symlink_preferred_arch {
- if binary.Properties.Stem == "" && binary.Properties.Suffix == "" {
+ if Bool(binary.Properties.Symlink_preferred_arch) {
+ if String(binary.Properties.Stem) == "" && String(binary.Properties.Suffix) == "" {
ctx.PropertyErrorf("symlink_preferred_arch", "must also specify stem or suffix")
}
if ctx.TargetPrimary() {
diff --git a/cc/builder.go b/cc/builder.go
index 873b117..b5f4c5c 100644
--- a/cc/builder.go
+++ b/cc/builder.go
@@ -37,6 +37,14 @@
)
var (
+ abiCheckAllowFlags = []string{
+ "-allow-extensions",
+ "-allow-unreferenced-changes",
+ "-allow-unreferenced-elf-symbol-changes",
+ }
+)
+
+var (
pctx = android.NewPackageContext("android/soong/cc")
cc = pctx.AndroidGomaStaticRule("cc",
@@ -151,11 +159,20 @@
yasm = pctx.AndroidStaticRule("yasm",
blueprint.RuleParams{
- Command: "$yasmCmd $asFlags -o $out $in",
+ Command: "$yasmCmd $asFlags -o $out $in && $yasmCmd $asFlags -M $in >$out.d",
CommandDeps: []string{"$yasmCmd"},
+ Depfile: "$out.d",
+ Deps: blueprint.DepsGCC,
},
"asFlags")
+ windres = pctx.AndroidStaticRule("windres",
+ blueprint.RuleParams{
+ Command: "$windresCmd $flags -I$$(dirname $in) -i $in -o $out",
+ CommandDeps: []string{"$windresCmd"},
+ },
+ "windresCmd", "flags")
+
_ = pctx.SourcePathVariable("sAbiDumper", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/header-abi-dumper")
// -w has been added since header-abi-dumper does not need to produce any sort of diagnostic information.
@@ -179,13 +196,21 @@
_ = pctx.SourcePathVariable("sAbiDiffer", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/header-abi-diff")
- // Abidiff check turned on in advice-only mode. Builds will not fail on abi incompatibilties / extensions.
- sAbiDiff = pctx.AndroidStaticRule("sAbiDiff",
- blueprint.RuleParams{
- Command: "$sAbiDiffer -lib $libName -arch $arch -advice-only -o ${out} -new $in -old $referenceDump",
- CommandDeps: []string{"$sAbiDiffer"},
+ sAbiDiff = pctx.AndroidRuleFunc("sAbiDiff",
+ func(config android.Config) (blueprint.RuleParams, error) {
+
+ commandStr := "($sAbiDiffer $allowFlags -lib $libName -arch $arch -check-all-apis -o ${out} -new $in -old $referenceDump)"
+ distDir := config.ProductVariables.DistDir
+ if distDir != nil && *distDir != "" {
+ distAbiDiffDir := *distDir + "/abidiffs/"
+ commandStr += " || (mkdir -p " + distAbiDiffDir + " && cp ${out} " + distAbiDiffDir + " && exit 1)"
+ }
+ return blueprint.RuleParams{
+ Command: commandStr,
+ CommandDeps: []string{"$sAbiDiffer"},
+ }, nil
},
- "referenceDump", "libName", "arch")
+ "allowFlags", "referenceDump", "libName", "arch")
unzipRefSAbiDump = pctx.AndroidStaticRule("unzipRefSAbiDump",
blueprint.RuleParams{
@@ -210,7 +235,7 @@
arFlags string
asFlags string
cFlags string
- toolingCFlags string // Seperate set of Cflags for clang LibTooling tools
+ toolingCFlags string // A separate set of Cflags for clang LibTooling tools
conlyFlags string
cppFlags string
ldFlags string
@@ -330,8 +355,9 @@
objFiles[i] = objFile
- if srcFile.Ext() == ".asm" {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ switch srcFile.Ext() {
+ case ".asm":
+ ctx.Build(pctx, android.BuildParams{
Rule: yasm,
Description: "yasm " + srcFile.Rel(),
Output: objFile,
@@ -342,6 +368,19 @@
},
})
continue
+ case ".rc":
+ ctx.Build(pctx, android.BuildParams{
+ Rule: windres,
+ Description: "windres " + srcFile.Rel(),
+ Output: objFile,
+ Input: srcFile,
+ OrderOnly: deps,
+ Args: map[string]string{
+ "windresCmd": gccCmd(flags.toolchain, "windres"),
+ "flags": flags.toolchain.WindresFlags(),
+ },
+ })
+ continue
}
var moduleCflags string
@@ -397,7 +436,7 @@
coverageFiles = append(coverageFiles, gcnoFile)
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: cc,
Description: ccDesc + " " + srcFile.Rel(),
Output: objFile,
@@ -414,7 +453,7 @@
tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy")
tidyFiles = append(tidyFiles, tidyFile)
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: clangTidy,
Description: "clang-tidy " + srcFile.Rel(),
Output: tidyFile,
@@ -433,7 +472,7 @@
sAbiDumpFile := android.ObjPathWithExt(ctx, subdir, srcFile, "sdump")
sAbiDumpFiles = append(sAbiDumpFiles, sAbiDumpFile)
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: sAbiDump,
Description: "header-abi-dumper " + srcFile.Rel(),
Output: sAbiDumpFile,
@@ -471,7 +510,7 @@
arFlags += " " + flags.arFlags
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: ar,
Description: "static link " + outputFile.Base(),
Output: outputFile,
@@ -497,14 +536,14 @@
dummy := android.PathForModuleOut(ctx, "dummy"+objectExtension)
dummyAr := android.PathForModuleOut(ctx, "dummy"+staticLibraryExtension)
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: emptyFile,
Description: "empty object file",
Output: dummy,
Implicits: deps,
})
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: darwinAr,
Description: "empty static archive",
Output: dummyAr,
@@ -514,7 +553,7 @@
},
})
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: darwinAppendAr,
Description: "static link " + outputFile.Base(),
Output: outputFile,
@@ -542,7 +581,7 @@
out = android.PathForModuleOut(ctx, outputFile.Base()+strconv.Itoa(i))
}
- build := android.ModuleBuildParams{
+ build := android.BuildParams{
Rule: darwinAr,
Description: "static link " + out.Base(),
Output: out,
@@ -556,12 +595,12 @@
build.Rule = darwinAppendAr
build.Args["inAr"] = in.String()
}
- ctx.ModuleBuild(pctx, build)
+ ctx.Build(pctx, build)
}
}
// Generate a rule for compiling multiple .o files, plus static libraries, whole static libraries,
-// and shared libraires, to a shared library (.so) or dynamic executable
+// and shared libraries, to a shared library (.so) or dynamic executable
func TransformObjToDynamicBinary(ctx android.ModuleContext,
objFiles, sharedLibs, staticLibs, lateStaticLibs, wholeStaticLibs, deps android.Paths,
crtBegin, crtEnd android.OptionalPath, groupLate bool, flags builderFlags, outputFile android.WritablePath) {
@@ -616,7 +655,7 @@
deps = append(deps, crtBegin.Path(), crtEnd.Path())
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: ld,
Description: "link " + outputFile.Base(),
Output: outputFile,
@@ -646,7 +685,7 @@
linkedDumpDep = soFile
symbolFilterStr = "-so " + soFile.String()
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: sAbiLink,
Description: "header-abi-linker " + outputFile.Base(),
Output: outputFile,
@@ -664,7 +703,7 @@
func UnzipRefDump(ctx android.ModuleContext, zippedRefDump android.Path, baseName string) android.Path {
outputFile := android.PathForModuleOut(ctx, baseName+"_ref.lsdump")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: unzipRefSAbiDump,
Description: "gunzip" + outputFile.Base(),
Output: outputFile,
@@ -676,7 +715,7 @@
func SourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceDump android.Path,
baseName string) android.OptionalPath {
outputFile := android.PathForModuleOut(ctx, baseName+".abidiff")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: sAbiDiff,
Description: "header-abi-diff " + outputFile.Base(),
Output: outputFile,
@@ -686,18 +725,19 @@
"referenceDump": referenceDump.String(),
"libName": baseName,
"arch": ctx.Arch().ArchType.Name,
+ "allowFlags": strings.Join(abiCheckAllowFlags, " "),
},
})
return android.OptionalPathForPath(outputFile)
}
-// Generate a rule for extract a table of contents from a shared library (.so)
-func TransformSharedObjectToToc(ctx android.ModuleContext, inputFile android.WritablePath,
+// Generate a rule for extracting a table of contents from a shared library (.so)
+func TransformSharedObjectToToc(ctx android.ModuleContext, inputFile android.Path,
outputFile android.WritablePath, flags builderFlags) {
crossCompile := gccCmd(flags.toolchain, "")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: toc,
Description: "generate toc " + inputFile.Base(),
Output: outputFile,
@@ -719,7 +759,7 @@
ldCmd = gccCmd(flags.toolchain, "g++")
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: partialLd,
Description: "link " + outputFile.Base(),
Output: outputFile,
@@ -737,7 +777,7 @@
objcopyCmd := gccCmd(flags.toolchain, "objcopy")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: prefixSymbols,
Description: "prefix symbols " + outputFile.Base(),
Output: outputFile,
@@ -764,7 +804,7 @@
args += " --keep-symbols"
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: strip,
Description: "strip " + outputFile.Base(),
Output: outputFile,
@@ -779,7 +819,7 @@
func TransformDarwinStrip(ctx android.ModuleContext, inputFile android.Path,
outputFile android.WritablePath) {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: darwinStrip,
Description: "strip " + outputFile.Base(),
Output: outputFile,
@@ -804,7 +844,7 @@
func CopyGccLib(ctx android.ModuleContext, libName string,
flags builderFlags, outputFile android.WritablePath) {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: copyGccLib,
Description: "copy gcc library " + libName,
Output: outputFile,
diff --git a/cc/cc.go b/cc/cc.go
index 0f754a6..fa33bc6 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -34,8 +34,9 @@
android.RegisterModuleType("cc_defaults", defaultsFactory)
android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) {
- ctx.BottomUp("link", linkageMutator).Parallel()
ctx.BottomUp("image", vendorMutator).Parallel()
+ ctx.BottomUp("link", linkageMutator).Parallel()
+ ctx.BottomUp("vndk", vndkMutator).Parallel()
ctx.BottomUp("ndk_api", ndkApiMutator).Parallel()
ctx.BottomUp("test_per_src", testPerSrcMutator).Parallel()
ctx.BottomUp("begin", beginMutator).Parallel()
@@ -45,11 +46,17 @@
ctx.TopDown("asan_deps", sanitizerDepsMutator(asan))
ctx.BottomUp("asan", sanitizerMutator(asan)).Parallel()
+ ctx.TopDown("cfi_deps", sanitizerDepsMutator(cfi))
+ ctx.BottomUp("cfi", sanitizerMutator(cfi)).Parallel()
+
ctx.TopDown("tsan_deps", sanitizerDepsMutator(tsan))
ctx.BottomUp("tsan", sanitizerMutator(tsan)).Parallel()
ctx.BottomUp("coverage", coverageLinkingMutator).Parallel()
ctx.TopDown("vndk_deps", sabiDepsMutator)
+
+ ctx.TopDown("lto_deps", ltoDepsMutator)
+ ctx.BottomUp("lto", ltoMutator).Parallel()
})
pctx.Import("android/soong/cc/config")
@@ -70,6 +77,7 @@
ReexportGeneratedHeaders []string
CrtBegin, CrtEnd string
+ LinkerScript string
}
type PathDeps struct {
@@ -94,6 +102,7 @@
// Paths to crt*.o files
CrtBegin, CrtEnd android.OptionalPath
+ LinkerScript android.OptionalPath
}
type Flags struct {
@@ -128,7 +137,8 @@
RequiredInstructionSet string
DynamicLinker string
- CFlagsDeps android.Paths // Files depended on by compiler flags
+ CFlagsDeps android.Paths // Files depended on by compiler flags
+ LdFlagsDeps android.Paths // Files depended on by linker flags
GroupStaticLibs bool
}
@@ -136,6 +146,9 @@
type ObjectLinkerProperties struct {
// names of other cc_object modules to link into this module using partial linking
Objs []string `android:"arch_variant"`
+
+ // if set, add an extra objcopy --prefix-symbols= step
+ Prefix_symbols *string
}
// Properties used to compile all C or C++ modules
@@ -144,30 +157,37 @@
Clang *bool `android:"arch_variant"`
// Minimum sdk version supported when compiling against the ndk
- Sdk_version string
-
- // don't insert default compiler flags into asflags, cflags,
- // cppflags, conlyflags, ldflags, or include_dirs
- No_default_compiler_flags *bool
-
- // whether this module should be allowed to install onto /vendor as
- // well as /system. The two variants will be built separately, one
- // like normal, and the other limited to the set of libraries and
- // headers that are exposed to /vendor modules.
- //
- // The vendor variant may be used with a different (newer) /system,
- // so it shouldn't have any unversioned runtime dependencies, or
- // make assumptions about the system that may not be true in the
- // future.
- //
- // Nothing happens if BOARD_VNDK_VERSION isn't set in the BoardConfig.mk
- Vendor_available *bool
+ Sdk_version *string
AndroidMkSharedLibs []string `blueprint:"mutated"`
HideFromMake bool `blueprint:"mutated"`
PreventInstall bool `blueprint:"mutated"`
UseVndk bool `blueprint:"mutated"`
+
+ // *.logtags files, to combine together in order to generate the /system/etc/event-log-tags
+ // file
+ Logtags []string
+}
+
+type VendorProperties struct {
+ // whether this module should be allowed to be directly depended by other
+ // modules with `vendor: true`, `proprietary: true`, or `vendor_available:true`.
+ // If set to true, two variants will be built separately, one like
+ // normal, and the other limited to the set of libraries and headers
+ // that are exposed to /vendor modules.
+ //
+ // The vendor variant may be used with a different (newer) /system,
+ // so it shouldn't have any unversioned runtime dependencies, or
+ // make assumptions about the system that may not be true in the
+ // future.
+ //
+ // If set to false, this module becomes inaccessible from /vendor modules.
+ //
+ // Default value is true when vndk: {enabled: true} or vendor: true.
+ //
+ // Nothing happens if BOARD_VNDK_VERSION isn't set in the BoardConfig.mk
+ Vendor_available *bool
}
type UnusedProperties struct {
@@ -179,10 +199,9 @@
staticBinary() bool
clang() bool
toolchain() config.Toolchain
- noDefaultCompilerFlags() bool
- sdk() bool
+ useSdk() bool
sdkVersion() string
- vndk() bool
+ useVndk() bool
isVndk() bool
isVndkSp() bool
createVndkSourceAbiDump() bool
@@ -215,7 +234,7 @@
type compiler interface {
compilerInit(ctx BaseModuleContext)
compilerDeps(ctx DepsContext, deps Deps) Deps
- compilerFlags(ctx ModuleContext, flags Flags) Flags
+ compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags
compilerProps() []interface{}
appendCflags([]string)
@@ -265,6 +284,7 @@
objDepTag = dependencyTag{name: "obj"}
crtBeginDepTag = dependencyTag{name: "crtbegin"}
crtEndDepTag = dependencyTag{name: "crtend"}
+ linkerScriptDepTag = dependencyTag{name: "linker script"}
reuseObjTag = dependencyTag{name: "reuse objects"}
ndkStubDepTag = dependencyTag{name: "ndk stub", library: true}
ndkLateStubDepTag = dependencyTag{name: "ndk late stub", library: true}
@@ -277,8 +297,9 @@
android.ModuleBase
android.DefaultableModuleBase
- Properties BaseProperties
- unused UnusedProperties
+ Properties BaseProperties
+ VendorProperties VendorProperties
+ unused UnusedProperties
// initialize before calling Init
hod android.HostOrDeviceSupported
@@ -294,6 +315,8 @@
coverage *coverage
sabi *sabi
vndkdep *vndkdep
+ lto *lto
+ pgo *pgo
androidMkSharedLibDeps []string
@@ -305,10 +328,18 @@
// Flags used to compile this module
flags Flags
+
+ // When calling a linker, if module A depends on module B, then A must precede B in its command
+ // line invocation. depsInLinkOrder stores the proper ordering of all of the transitive
+ // deps of this module
+ depsInLinkOrder android.Paths
+
+ // only non-nil when this is a shared library that reuses the objects of a static library
+ staticVariant *Module
}
func (c *Module) Init() android.Module {
- c.AddProperties(&c.Properties, &c.unused)
+ c.AddProperties(&c.Properties, &c.VendorProperties, &c.unused)
if c.compiler != nil {
c.AddProperties(c.compiler.compilerProps()...)
}
@@ -333,6 +364,12 @@
if c.vndkdep != nil {
c.AddProperties(c.vndkdep.props()...)
}
+ if c.lto != nil {
+ c.AddProperties(c.lto.props()...)
+ }
+ if c.pgo != nil {
+ c.AddProperties(c.pgo.props()...)
+ }
for _, feature := range c.features {
c.AddProperties(feature.props()...)
}
@@ -355,7 +392,7 @@
return false
}
-func (c *Module) vndk() bool {
+func (c *Module) useVndk() bool {
return c.Properties.UseVndk
}
@@ -366,6 +403,12 @@
return false
}
+// Returns true only when this module is configured to have core and vendor
+// variants.
+func (c *Module) hasVendorVariant() bool {
+ return c.isVndk() || Bool(c.VendorProperties.Vendor_available)
+}
+
type baseModuleContext struct {
android.BaseContext
moduleContextImpl
@@ -381,10 +424,8 @@
moduleContextImpl
}
-// Vendor returns true for vendor modules excluding VNDK libraries so that
-// they get installed onto the correct partition
-func (ctx *moduleContext) Vendor() bool {
- return ctx.ModuleContext.Vendor() || (ctx.mod.vndk() && !ctx.mod.isVndk())
+func (ctx *moduleContext) InstallOnVendorPartition() bool {
+ return ctx.ModuleContext.InstallOnVendorPartition() || (ctx.mod.useVndk() && !ctx.mod.isVndk())
}
type moduleContextImpl struct {
@@ -401,12 +442,7 @@
}
func (ctx *moduleContextImpl) static() bool {
- if static, ok := ctx.mod.linker.(interface {
- static() bool
- }); ok {
- return static.static()
- }
- return false
+ return ctx.mod.static()
}
func (ctx *moduleContextImpl) staticBinary() bool {
@@ -418,35 +454,30 @@
return false
}
-func (ctx *moduleContextImpl) noDefaultCompilerFlags() bool {
- return Bool(ctx.mod.Properties.No_default_compiler_flags)
-}
-
-func (ctx *moduleContextImpl) sdk() bool {
- if ctx.ctx.Device() && !ctx.vndk() {
- return ctx.mod.Properties.Sdk_version != ""
+func (ctx *moduleContextImpl) useSdk() bool {
+ if ctx.ctx.Device() && !ctx.useVndk() {
+ return String(ctx.mod.Properties.Sdk_version) != ""
}
return false
}
func (ctx *moduleContextImpl) sdkVersion() string {
if ctx.ctx.Device() {
- if ctx.vndk() {
+ if ctx.useVndk() {
return "current"
} else {
- return ctx.mod.Properties.Sdk_version
+ return String(ctx.mod.Properties.Sdk_version)
}
}
return ""
}
-func (ctx *moduleContextImpl) vndk() bool {
- return ctx.mod.vndk()
-}
-
func (ctx *moduleContextImpl) isVndk() bool {
return ctx.mod.isVndk()
}
+func (ctx *moduleContextImpl) useVndk() bool {
+ return ctx.mod.useVndk()
+}
func (ctx *moduleContextImpl) isVndkSp() bool {
if vndk := ctx.mod.vndkdep; vndk != nil {
@@ -457,7 +488,7 @@
// Create source abi dumps if the module belongs to the list of VndkLibraries.
func (ctx *moduleContextImpl) createVndkSourceAbiDump() bool {
- return ctx.ctx.Device() && (ctx.mod.isVndk() || inList(ctx.baseModuleName(), config.LLndkLibraries()))
+ return ctx.ctx.Device() && ((ctx.useVndk() && ctx.isVndk()) || inList(ctx.baseModuleName(), llndkLibraries))
}
func (ctx *moduleContextImpl) selectedStl() string {
@@ -488,6 +519,8 @@
module.coverage = &coverage{}
module.sabi = &sabi{}
module.vndkdep = &vndkdep{}
+ module.lto = <o{}
+ module.pgo = &pgo{}
return module
}
@@ -508,7 +541,62 @@
return name
}
+// orderDeps reorders dependencies into a list such that if module A depends on B, then
+// A will precede B in the resultant list.
+// This is convenient for passing into a linker.
+// Note that directSharedDeps should be the analogous static library for each shared lib dep
+func orderDeps(directStaticDeps []android.Path, directSharedDeps []android.Path, allTransitiveDeps map[android.Path][]android.Path) (orderedAllDeps []android.Path, orderedDeclaredDeps []android.Path) {
+ // If A depends on B, then
+ // Every list containing A will also contain B later in the list
+ // So, after concatenating all lists, the final instance of B will have come from the same
+ // original list as the final instance of A
+ // So, the final instance of B will be later in the concatenation than the final A
+ // So, keeping only the final instance of A and of B ensures that A is earlier in the output
+ // list than B
+ for _, dep := range directStaticDeps {
+ orderedAllDeps = append(orderedAllDeps, dep)
+ orderedAllDeps = append(orderedAllDeps, allTransitiveDeps[dep]...)
+ }
+ for _, dep := range directSharedDeps {
+ orderedAllDeps = append(orderedAllDeps, dep)
+ orderedAllDeps = append(orderedAllDeps, allTransitiveDeps[dep]...)
+ }
+
+ orderedAllDeps = android.LastUniquePaths(orderedAllDeps)
+
+ // We don't want to add any new dependencies into directStaticDeps (to allow the caller to
+ // intentionally exclude or replace any unwanted transitive dependencies), so we limit the
+ // resultant list to only what the caller has chosen to include in directStaticDeps
+ _, orderedDeclaredDeps = android.FilterPathList(orderedAllDeps, directStaticDeps)
+
+ return orderedAllDeps, orderedDeclaredDeps
+}
+
+func orderStaticModuleDeps(module *Module, staticDeps []*Module, sharedDeps []*Module) (results []android.Path) {
+ // convert Module to Path
+ allTransitiveDeps := make(map[android.Path][]android.Path, len(staticDeps))
+ staticDepFiles := []android.Path{}
+ for _, dep := range staticDeps {
+ allTransitiveDeps[dep.outputFile.Path()] = dep.depsInLinkOrder
+ staticDepFiles = append(staticDepFiles, dep.outputFile.Path())
+ }
+ sharedDepFiles := []android.Path{}
+ for _, sharedDep := range sharedDeps {
+ staticAnalogue := sharedDep.staticVariant
+ if staticAnalogue != nil {
+ allTransitiveDeps[staticAnalogue.outputFile.Path()] = staticAnalogue.depsInLinkOrder
+ sharedDepFiles = append(sharedDepFiles, staticAnalogue.outputFile.Path())
+ }
+ }
+
+ // reorder the dependencies based on transitive dependencies
+ module.depsInLinkOrder, results = orderDeps(staticDepFiles, sharedDepFiles, allTransitiveDeps)
+
+ return results
+}
+
func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
+
ctx := &moduleContext{
ModuleContext: actx,
moduleContextImpl: moduleContextImpl{
@@ -517,12 +605,17 @@
}
ctx.ctx = ctx
+ deps := c.depsToPaths(ctx)
+ if ctx.Failed() {
+ return
+ }
+
flags := Flags{
Toolchain: c.toolchain(ctx),
Clang: c.clang(ctx),
}
if c.compiler != nil {
- flags = c.compiler.compilerFlags(ctx, flags)
+ flags = c.compiler.compilerFlags(ctx, flags, deps)
}
if c.linker != nil {
flags = c.linker.linkerFlags(ctx, flags)
@@ -536,6 +629,12 @@
if c.coverage != nil {
flags = c.coverage.flags(ctx, flags)
}
+ if c.lto != nil {
+ flags = c.lto.flags(ctx, flags)
+ }
+ if c.pgo != nil {
+ flags = c.pgo.flags(ctx, flags)
+ }
for _, feature := range c.features {
flags = feature.flags(ctx, flags)
}
@@ -547,10 +646,6 @@
flags.CppFlags, _ = filterList(flags.CppFlags, config.IllegalFlags)
flags.ConlyFlags, _ = filterList(flags.ConlyFlags, config.IllegalFlags)
- deps := c.depsToPaths(ctx)
- if ctx.Failed() {
- return
- }
flags.GlobalFlags = append(flags.GlobalFlags, deps.Flags...)
c.flags = flags
// We need access to all the flags seen by a source file.
@@ -619,15 +714,21 @@
if c.vndkdep != nil {
c.vndkdep.begin(ctx)
}
+ if c.lto != nil {
+ c.lto.begin(ctx)
+ }
+ if c.pgo != nil {
+ c.pgo.begin(ctx)
+ }
for _, feature := range c.features {
feature.begin(ctx)
}
- if ctx.sdk() {
- version, err := normalizeNdkApiLevel(ctx.sdkVersion(), ctx.Arch())
+ if ctx.useSdk() {
+ version, err := normalizeNdkApiLevel(ctx, ctx.sdkVersion(), ctx.Arch())
if err != nil {
ctx.PropertyErrorf("sdk_version", err.Error())
}
- c.Properties.Sdk_version = version
+ c.Properties.Sdk_version = StringPtr(version)
}
}
@@ -637,6 +738,12 @@
if c.compiler != nil {
deps = c.compiler.compilerDeps(ctx, deps)
}
+ // Add the PGO dependency (the clang_rt.profile runtime library), which
+ // sometimes depends on symbols from libgcc, before libgcc gets added
+ // in linkerDeps().
+ if c.pgo != nil {
+ deps = c.pgo.deps(ctx, deps)
+ }
if c.linker != nil {
deps = c.linker.linkerDeps(ctx, deps)
}
@@ -655,16 +762,19 @@
if c.vndkdep != nil {
deps = c.vndkdep.deps(ctx, deps)
}
+ if c.lto != nil {
+ deps = c.lto.deps(ctx, deps)
+ }
for _, feature := range c.features {
deps = feature.deps(ctx, deps)
}
- deps.WholeStaticLibs = lastUniqueElements(deps.WholeStaticLibs)
- deps.StaticLibs = lastUniqueElements(deps.StaticLibs)
- deps.LateStaticLibs = lastUniqueElements(deps.LateStaticLibs)
- deps.SharedLibs = lastUniqueElements(deps.SharedLibs)
- deps.LateSharedLibs = lastUniqueElements(deps.LateSharedLibs)
- deps.HeaderLibs = lastUniqueElements(deps.HeaderLibs)
+ deps.WholeStaticLibs = android.LastUniqueStrings(deps.WholeStaticLibs)
+ deps.StaticLibs = android.LastUniqueStrings(deps.StaticLibs)
+ deps.LateStaticLibs = android.LastUniqueStrings(deps.LateStaticLibs)
+ deps.SharedLibs = android.LastUniqueStrings(deps.SharedLibs)
+ deps.LateSharedLibs = android.LastUniqueStrings(deps.LateSharedLibs)
+ deps.HeaderLibs = android.LastUniqueStrings(deps.HeaderLibs)
for _, lib := range deps.ReexportSharedLibHeaders {
if !inList(lib, deps.SharedLibs) {
@@ -725,27 +835,31 @@
if ctx.Os() == android.Android {
version := ctx.sdkVersion()
- // Rewrites the names of shared libraries into the names of the NDK
- // libraries where appropriate. This returns two slices.
+ // rewriteNdkLibs takes a list of names of shared libraries and scans it for three types
+ // of names:
//
- // The first is a list of non-variant shared libraries (either rewritten
- // NDK libraries to the modules in prebuilts/ndk, or not rewritten
- // because they are not NDK libraries).
+ // 1. Name of an NDK library that refers to a prebuilt module.
+ // For each of these, it adds the name of the prebuilt module (which will be in
+ // prebuilts/ndk) to the list of nonvariant libs.
+ // 2. Name of an NDK library that refers to an ndk_library module.
+ // For each of these, it adds the name of the ndk_library module to the list of
+ // variant libs.
+ // 3. Anything else (so anything that isn't an NDK library).
+ // It adds these to the nonvariantLibs list.
//
- // The second is a list of ndk_library modules. These need to be
- // separated because they are a variation dependency and must be added
- // in a different manner.
- rewriteNdkLibs := func(list []string) ([]string, []string) {
- variantLibs := []string{}
- nonvariantLibs := []string{}
+ // The caller can then know to add the variantLibs dependencies differently from the
+ // nonvariantLibs
+ rewriteNdkLibs := func(list []string) (nonvariantLibs []string, variantLibs []string) {
+ variantLibs = []string{}
+ nonvariantLibs = []string{}
for _, entry := range list {
- if ctx.sdk() && inList(entry, ndkPrebuiltSharedLibraries) {
+ if ctx.useSdk() && inList(entry, ndkPrebuiltSharedLibraries) {
if !inList(entry, ndkMigratedLibs) {
nonvariantLibs = append(nonvariantLibs, entry+".ndk."+version)
} else {
variantLibs = append(variantLibs, entry+ndkLibrarySuffix)
}
- } else if ctx.vndk() && inList(entry, config.LLndkLibraries()) {
+ } else if ctx.useVndk() && inList(entry, llndkLibraries) {
nonvariantLibs = append(nonvariantLibs, entry+llndkLibrarySuffix)
} else {
nonvariantLibs = append(nonvariantLibs, entry)
@@ -810,6 +924,9 @@
if deps.CrtEnd != "" {
actx.AddDependency(c, crtEndDepTag, deps.CrtEnd)
}
+ if deps.LinkerScript != "" {
+ actx.AddDependency(c, linkerScriptDepTag, deps.LinkerScript)
+ }
version := ctx.sdkVersion()
actx.AddVariationDependencies([]blueprint.Variation{
@@ -832,7 +949,7 @@
clang = true
}
- if ctx.Device() && ctx.AConfig().DeviceUsesClang() {
+ if ctx.Device() && ctx.Config().DeviceUsesClang() {
clang = true
}
}
@@ -844,116 +961,115 @@
return clang
}
+// Whether a module can link to another module, taking into
+// account NDK linking.
+func checkLinkType(ctx android.ModuleContext, from *Module, to *Module) {
+ if from.Target().Os != android.Android {
+ // Host code is not restricted
+ return
+ }
+ if from.Properties.UseVndk {
+ // Though vendor code is limited by the vendor mutator,
+ // each vendor-available module needs to check
+ // link-type for VNDK.
+ if from.vndkdep != nil {
+ from.vndkdep.vndkCheckLinkType(ctx, to)
+ }
+ return
+ }
+ if String(from.Properties.Sdk_version) == "" {
+ // Platform code can link to anything
+ return
+ }
+ if _, ok := to.linker.(*toolchainLibraryDecorator); ok {
+ // These are always allowed
+ return
+ }
+ if _, ok := to.linker.(*ndkPrebuiltLibraryLinker); ok {
+ // These are allowed, but they don't set sdk_version
+ return
+ }
+ if _, ok := to.linker.(*ndkPrebuiltStlLinker); ok {
+ // These are allowed, but they don't set sdk_version
+ return
+ }
+ if _, ok := to.linker.(*stubDecorator); ok {
+ // These aren't real libraries, but are the stub shared libraries that are included in
+ // the NDK.
+ return
+ }
+ if String(to.Properties.Sdk_version) == "" {
+ // NDK code linking to platform code is never okay.
+ ctx.ModuleErrorf("depends on non-NDK-built library %q",
+ ctx.OtherModuleName(to))
+ }
+
+ // At this point we know we have two NDK libraries, but we need to
+ // check that we're not linking against anything built against a higher
+ // API level, as it is only valid to link against older or equivalent
+ // APIs.
+
+ if String(from.Properties.Sdk_version) == "current" {
+ // Current can link against anything.
+ return
+ } else if String(to.Properties.Sdk_version) == "current" {
+ // Current can't be linked against by anything else.
+ ctx.ModuleErrorf("links %q built against newer API version %q",
+ ctx.OtherModuleName(to), "current")
+ }
+
+ fromApi, err := strconv.Atoi(String(from.Properties.Sdk_version))
+ if err != nil {
+ ctx.PropertyErrorf("sdk_version",
+ "Invalid sdk_version value (must be int): %q",
+ String(from.Properties.Sdk_version))
+ }
+ toApi, err := strconv.Atoi(String(to.Properties.Sdk_version))
+ if err != nil {
+ ctx.PropertyErrorf("sdk_version",
+ "Invalid sdk_version value (must be int): %q",
+ String(to.Properties.Sdk_version))
+ }
+
+ if toApi > fromApi {
+ ctx.ModuleErrorf("links %q built against newer API version %q",
+ ctx.OtherModuleName(to), String(to.Properties.Sdk_version))
+ }
+}
+
// Convert dependencies to paths. Returns a PathDeps containing paths
func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps {
var depPaths PathDeps
- // Whether a module can link to another module, taking into
- // account NDK linking.
- checkLinkType := func(from, to *Module) {
- if from.Target().Os != android.Android {
- // Host code is not restricted
- return
- }
- if from.Properties.UseVndk {
- // Though vendor code is limited by the vendor mutator,
- // each vendor-available module needs to check
- // link-type for VNDK.
- if from.vndkdep != nil {
- from.vndkdep.vndkCheckLinkType(ctx, to)
- }
- return
- }
- if from.Properties.Sdk_version == "" {
- // Platform code can link to anything
- return
- }
- if _, ok := to.linker.(*toolchainLibraryDecorator); ok {
- // These are always allowed
- return
- }
- if _, ok := to.linker.(*ndkPrebuiltLibraryLinker); ok {
- // These are allowed, but don't set sdk_version
- return
- }
- if _, ok := to.linker.(*ndkPrebuiltStlLinker); ok {
- // These are allowed, but don't set sdk_version
- return
- }
- if _, ok := to.linker.(*stubDecorator); ok {
- // These aren't real libraries, but are the stub shared libraries that are included in
- // the NDK.
- return
- }
- if to.Properties.Sdk_version == "" {
- // NDK code linking to platform code is never okay.
- ctx.ModuleErrorf("depends on non-NDK-built library %q",
- ctx.OtherModuleName(to))
- }
+ directStaticDeps := []*Module{}
+ directSharedDeps := []*Module{}
- // All this point we know we have two NDK libraries, but we need to
- // check that we're not linking against anything built against a higher
- // API level, as it is only valid to link against older or equivalent
- // APIs.
+ ctx.VisitDirectDeps(func(dep android.Module) {
+ depName := ctx.OtherModuleName(dep)
+ depTag := ctx.OtherModuleDependencyTag(dep)
- if from.Properties.Sdk_version == "current" {
- // Current can link against anything.
- return
- } else if to.Properties.Sdk_version == "current" {
- // Current can't be linked against by anything else.
- ctx.ModuleErrorf("links %q built against newer API version %q",
- ctx.OtherModuleName(to), "current")
- }
-
- fromApi, err := strconv.Atoi(from.Properties.Sdk_version)
- if err != nil {
- ctx.PropertyErrorf("sdk_version",
- "Invalid sdk_version value (must be int): %q",
- from.Properties.Sdk_version)
- }
- toApi, err := strconv.Atoi(to.Properties.Sdk_version)
- if err != nil {
- ctx.PropertyErrorf("sdk_version",
- "Invalid sdk_version value (must be int): %q",
- to.Properties.Sdk_version)
- }
-
- if toApi > fromApi {
- ctx.ModuleErrorf("links %q built against newer API version %q",
- ctx.OtherModuleName(to), to.Properties.Sdk_version)
- }
- }
-
- ctx.VisitDirectDeps(func(m blueprint.Module) {
- name := ctx.OtherModuleName(m)
- tag := ctx.OtherModuleDependencyTag(m)
-
- a, _ := m.(android.Module)
- if a == nil {
- ctx.ModuleErrorf("module %q not an android module", name)
- return
- }
-
- cc, _ := m.(*Module)
- if cc == nil {
- switch tag {
+ ccDep, _ := dep.(*Module)
+ if ccDep == nil {
+ // handling for a few module types that aren't cc Module but that are also supported
+ switch depTag {
case android.DefaultsDepTag, android.SourceDepTag:
+ // Nothing to do
case genSourceDepTag:
- if genRule, ok := m.(genrule.SourceFileGenerator); ok {
+ if genRule, ok := dep.(genrule.SourceFileGenerator); ok {
depPaths.GeneratedSources = append(depPaths.GeneratedSources,
genRule.GeneratedSourceFiles()...)
} else {
- ctx.ModuleErrorf("module %q is not a gensrcs or genrule", name)
+ ctx.ModuleErrorf("module %q is not a gensrcs or genrule", depName)
}
// Support exported headers from a generated_sources dependency
fallthrough
case genHeaderDepTag, genHeaderExportDepTag:
- if genRule, ok := m.(genrule.SourceFileGenerator); ok {
+ if genRule, ok := dep.(genrule.SourceFileGenerator); ok {
depPaths.GeneratedHeaders = append(depPaths.GeneratedHeaders,
genRule.GeneratedSourceFiles()...)
flags := includeDirsToFlags(genRule.GeneratedHeaderDirs())
depPaths.Flags = append(depPaths.Flags, flags)
- if tag == genHeaderExportDepTag {
+ if depTag == genHeaderExportDepTag {
depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, flags)
depPaths.ReexportedFlagsDeps = append(depPaths.ReexportedFlagsDeps,
genRule.GeneratedSourceFiles()...)
@@ -962,35 +1078,38 @@
}
} else {
- ctx.ModuleErrorf("module %q is not a genrule", name)
+ ctx.ModuleErrorf("module %q is not a genrule", depName)
+ }
+ case linkerScriptDepTag:
+ if genRule, ok := dep.(genrule.SourceFileGenerator); ok {
+ files := genRule.GeneratedSourceFiles()
+ if len(files) == 1 {
+ depPaths.LinkerScript = android.OptionalPathForPath(files[0])
+ } else if len(files) > 1 {
+ ctx.ModuleErrorf("module %q can only generate a single file if used for a linker script", depName)
+ }
+ } else {
+ ctx.ModuleErrorf("module %q is not a genrule", depName)
}
default:
- ctx.ModuleErrorf("depends on non-cc module %q", name)
+ ctx.ModuleErrorf("depends on non-cc module %q", depName)
}
return
}
- if !a.Enabled() {
- if ctx.AConfig().AllowMissingDependencies() {
- ctx.AddMissingDependencies([]string{name})
- } else {
- ctx.ModuleErrorf("depends on disabled module %q", name)
- }
+ if dep.Target().Os != ctx.Os() {
+ ctx.ModuleErrorf("OS mismatch between %q and %q", ctx.ModuleName(), depName)
+ return
+ }
+ if dep.Target().Arch.ArchType != ctx.Arch().ArchType {
+ ctx.ModuleErrorf("Arch mismatch between %q and %q", ctx.ModuleName(), depName)
return
}
- if a.Target().Os != ctx.Os() {
- ctx.ModuleErrorf("OS mismatch between %q and %q", ctx.ModuleName(), name)
- return
- }
-
- if a.Target().Arch.ArchType != ctx.Arch().ArchType {
- ctx.ModuleErrorf("Arch mismatch between %q and %q", ctx.ModuleName(), name)
- return
- }
-
- if tag == reuseObjTag {
- if l, ok := cc.compiler.(libraryInterface); ok {
+ // re-exporting flags
+ if depTag == reuseObjTag {
+ if l, ok := ccDep.compiler.(libraryInterface); ok {
+ c.staticVariant = ccDep
objs, flags, deps := l.reuseObjs()
depPaths.Objs = depPaths.Objs.Append(objs)
depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, flags...)
@@ -998,9 +1117,8 @@
return
}
}
-
- if t, ok := tag.(dependencyTag); ok && t.library {
- if i, ok := cc.linker.(exportedFlagsProducer); ok {
+ if t, ok := depTag.(dependencyTag); ok && t.library {
+ if i, ok := ccDep.linker.(exportedFlagsProducer); ok {
flags := i.exportedFlags()
deps := i.exportedFlagsDeps()
depPaths.Flags = append(depPaths.Flags, flags...)
@@ -1010,46 +1128,46 @@
depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, flags...)
depPaths.ReexportedFlagsDeps = append(depPaths.ReexportedFlagsDeps, deps...)
// Add these re-exported flags to help header-abi-dumper to infer the abi exported by a library.
- // Re-exported flags from shared library dependencies are not included as those shared libraries
- // will be included in the vndk set.
- if tag == staticExportDepTag || tag == headerExportDepTag {
- c.sabi.Properties.ReexportedIncludeFlags = append(c.sabi.Properties.ReexportedIncludeFlags, flags...)
- }
+ // Re-exported shared library headers must be included as well since they can help us with type information
+ // about template instantiations (instantiated from their headers).
+ c.sabi.Properties.ReexportedIncludeFlags = append(c.sabi.Properties.ReexportedIncludeFlags, flags...)
}
}
- checkLinkType(c, cc)
+ checkLinkType(ctx, c, ccDep)
}
var ptr *android.Paths
var depPtr *android.Paths
- linkFile := cc.outputFile
+ linkFile := ccDep.outputFile
depFile := android.OptionalPath{}
- switch tag {
+ switch depTag {
case ndkStubDepTag, sharedDepTag, sharedExportDepTag:
ptr = &depPaths.SharedLibs
depPtr = &depPaths.SharedLibsDeps
- depFile = cc.linker.(libraryInterface).toc()
+ depFile = ccDep.linker.(libraryInterface).toc()
+ directSharedDeps = append(directSharedDeps, ccDep)
case lateSharedDepTag, ndkLateStubDepTag:
ptr = &depPaths.LateSharedLibs
depPtr = &depPaths.LateSharedLibsDeps
- depFile = cc.linker.(libraryInterface).toc()
+ depFile = ccDep.linker.(libraryInterface).toc()
case staticDepTag, staticExportDepTag:
- ptr = &depPaths.StaticLibs
+ ptr = nil
+ directStaticDeps = append(directStaticDeps, ccDep)
case lateStaticDepTag:
ptr = &depPaths.LateStaticLibs
case wholeStaticDepTag:
ptr = &depPaths.WholeStaticLibs
- staticLib, ok := cc.linker.(libraryInterface)
+ staticLib, ok := ccDep.linker.(libraryInterface)
if !ok || !staticLib.static() {
- ctx.ModuleErrorf("module %q not a static library", name)
+ ctx.ModuleErrorf("module %q not a static library", depName)
return
}
if missingDeps := staticLib.getWholeStaticMissingDeps(); missingDeps != nil {
- postfix := " (required by " + ctx.OtherModuleName(m) + ")"
+ postfix := " (required by " + ctx.OtherModuleName(dep) + ")"
for i := range missingDeps {
missingDeps[i] += postfix
}
@@ -1066,11 +1184,11 @@
depPaths.CrtEnd = linkFile
}
- switch tag {
+ switch depTag {
case staticDepTag, staticExportDepTag, lateStaticDepTag:
- staticLib, ok := cc.linker.(libraryInterface)
+ staticLib, ok := ccDep.linker.(libraryInterface)
if !ok || !staticLib.static() {
- ctx.ModuleErrorf("module %q not a static library", name)
+ ctx.ModuleErrorf("module %q not a static library", depName)
return
}
@@ -1081,11 +1199,12 @@
staticLib.objs().coverageFiles...)
depPaths.StaticLibObjs.sAbiDumpFiles = append(depPaths.StaticLibObjs.sAbiDumpFiles,
staticLib.objs().sAbiDumpFiles...)
+
}
if ptr != nil {
if !linkFile.Valid() {
- ctx.ModuleErrorf("module %q missing output file", name)
+ ctx.ModuleErrorf("module %q missing output file", depName)
return
}
*ptr = append(*ptr, linkFile.Path())
@@ -1099,29 +1218,39 @@
*depPtr = append(*depPtr, dep.Path())
}
- // Export the shared libs to the make world. In doing so, .vendor suffix
- // is added if the lib has both core and vendor variants and this module
- // is building against vndk. This is because the vendor variant will be
- // have .vendor suffix in its name in the make world. However, if the
- // lib is a vendor-only lib or this lib is not building against vndk,
- // then the suffix is not added.
- switch tag {
+ // Export the shared libs to Make.
+ switch depTag {
case sharedDepTag, sharedExportDepTag, lateSharedDepTag:
- libName := strings.TrimSuffix(name, llndkLibrarySuffix)
+ libName := strings.TrimSuffix(depName, llndkLibrarySuffix)
libName = strings.TrimPrefix(libName, "prebuilt_")
- isLLndk := inList(libName, config.LLndkLibraries())
- if c.vndk() && (Bool(cc.Properties.Vendor_available) || isLLndk) {
- libName += vendorSuffix
+ isLLndk := inList(libName, llndkLibraries)
+ var makeLibName string
+ bothVendorAndCoreVariantsExist := ccDep.hasVendorVariant() || isLLndk
+ if c.useVndk() && bothVendorAndCoreVariantsExist {
+ // The vendor module in Make will have been renamed to not conflict with the core
+ // module, so update the dependency name here accordingly.
+ makeLibName = libName + vendorSuffix
+ } else {
+ makeLibName = libName
}
// Note: the order of libs in this list is not important because
- // they merely serve as dependencies in the make world and do not
- // affect this lib itself.
- c.Properties.AndroidMkSharedLibs = append(c.Properties.AndroidMkSharedLibs, libName)
+ // they merely serve as Make dependencies and do not affect this lib itself.
+ c.Properties.AndroidMkSharedLibs = append(c.Properties.AndroidMkSharedLibs, makeLibName)
}
})
+ // use the ordered dependencies as this module's dependencies
+ depPaths.StaticLibs = append(depPaths.StaticLibs, orderStaticModuleDeps(c, directStaticDeps, directSharedDeps)...)
+
// Dedup exported flags from dependencies
- depPaths.Flags = firstUniqueElements(depPaths.Flags)
+ depPaths.Flags = android.FirstUniqueStrings(depPaths.Flags)
+ depPaths.GeneratedHeaders = android.FirstUniquePaths(depPaths.GeneratedHeaders)
+ depPaths.ReexportedFlags = android.FirstUniqueStrings(depPaths.ReexportedFlags)
+ depPaths.ReexportedFlagsDeps = android.FirstUniquePaths(depPaths.ReexportedFlagsDeps)
+
+ if c.sabi != nil {
+ c.sabi.Properties.ReexportedIncludeFlags = android.FirstUniqueStrings(c.sabi.Properties.ReexportedIncludeFlags)
+ }
return depPaths
}
@@ -1150,6 +1279,26 @@
return c.installer.hostToolPath()
}
+func (c *Module) IntermPathForModuleOut() android.OptionalPath {
+ return c.outputFile
+}
+
+func (c *Module) Srcs() android.Paths {
+ if c.outputFile.Valid() {
+ return android.Paths{c.outputFile.Path()}
+ }
+ return android.Paths{}
+}
+
+func (c *Module) static() bool {
+ if static, ok := c.linker.(interface {
+ static() bool
+ }); ok {
+ return static.static()
+ }
+ return false
+}
+
//
// Defaults
//
@@ -1174,6 +1323,7 @@
module.AddProperties(props...)
module.AddProperties(
&BaseProperties{},
+ &VendorProperties{},
&BaseCompilerProperties{},
&BaseLinkerProperties{},
&LibraryProperties{},
@@ -1190,6 +1340,8 @@
&CoverageProperties{},
&SAbiProperties{},
&VndkProperties{},
+ <OProperties{},
+ &PgoProperties{},
)
android.InitDefaultsModule(module)
@@ -1207,26 +1359,50 @@
vendorMode = "vendor"
)
+func squashVendorSrcs(m *Module) {
+ if lib, ok := m.compiler.(*libraryDecorator); ok {
+ lib.baseCompiler.Properties.Srcs = append(lib.baseCompiler.Properties.Srcs,
+ lib.baseCompiler.Properties.Target.Vendor.Srcs...)
+
+ lib.baseCompiler.Properties.Exclude_srcs = append(lib.baseCompiler.Properties.Exclude_srcs,
+ lib.baseCompiler.Properties.Target.Vendor.Exclude_srcs...)
+ }
+}
+
func vendorMutator(mctx android.BottomUpMutatorContext) {
if mctx.Os() != android.Android {
return
}
+ if genrule, ok := mctx.Module().(*genrule.Module); ok {
+ if props, ok := genrule.Extra.(*VendorProperties); ok {
+ if mctx.DeviceConfig().VndkVersion() == "" {
+ mctx.CreateVariations(coreMode)
+ } else if Bool(props.Vendor_available) {
+ mctx.CreateVariations(coreMode, vendorMode)
+ } else if mctx.InstallOnVendorPartition() {
+ mctx.CreateVariations(vendorMode)
+ } else {
+ mctx.CreateVariations(coreMode)
+ }
+ }
+ }
+
m, ok := mctx.Module().(*Module)
if !ok {
return
}
// Sanity check
- if Bool(m.Properties.Vendor_available) && mctx.Vendor() {
+ if m.VendorProperties.Vendor_available != nil && mctx.InstallOnVendorPartition() {
mctx.PropertyErrorf("vendor_available",
"doesn't make sense at the same time as `vendor: true` or `proprietary: true`")
return
}
if vndk := m.vndkdep; vndk != nil {
- if vndk.isVndk() && !Bool(m.Properties.Vendor_available) {
+ if vndk.isVndk() && m.VendorProperties.Vendor_available == nil {
mctx.PropertyErrorf("vndk",
- "has to define `vendor_available: true` to enable vndk")
+ "vendor_available must be set to either true or false when `vndk: {enabled: true}`")
return
}
if !vndk.isVndk() && vndk.isVndkSp() {
@@ -1236,7 +1412,7 @@
}
}
- if !mctx.DeviceConfig().CompileVndk() {
+ if mctx.DeviceConfig().VndkVersion() == "" {
// If the device isn't compiling against the VNDK, we always
// use the core mode.
mctx.CreateVariations(coreMode)
@@ -1244,15 +1420,28 @@
// LL-NDK stubs only exist in the vendor variant, since the
// real libraries will be used in the core variant.
mctx.CreateVariations(vendorMode)
- } else if Bool(m.Properties.Vendor_available) {
+ } else if _, ok := m.linker.(*llndkHeadersDecorator); ok {
+ // ... and LL-NDK headers as well
+ mctx.CreateVariations(vendorMode)
+ } else if _, ok := m.linker.(*vndkPrebuiltLibraryDecorator); ok {
+ // Make vendor variants only for the versions in BOARD_VNDK_VERSION and
+ // PRODUCT_EXTRA_VNDK_VERSIONS.
+ mod := mctx.CreateVariations(vendorMode)
+ vendor := mod[0].(*Module)
+ vendor.Properties.UseVndk = true
+ } else if m.hasVendorVariant() {
// This will be available in both /system and /vendor
// or a /system directory that is available to vendor.
mod := mctx.CreateVariations(coreMode, vendorMode)
- mod[1].(*Module).Properties.UseVndk = true
- } else if mctx.Vendor() && m.Properties.Sdk_version == "" {
+ vendor := mod[1].(*Module)
+ vendor.Properties.UseVndk = true
+ squashVendorSrcs(vendor)
+ } else if mctx.InstallOnVendorPartition() && String(m.Properties.Sdk_version) == "" {
// This will be available in /vendor only
mod := mctx.CreateVariations(vendorMode)
- mod[0].(*Module).Properties.UseVndk = true
+ vendor := mod[0].(*Module)
+ vendor.Properties.UseVndk = true
+ squashVendorSrcs(vendor)
} else {
// This is either in /system (or similar: /data), or is a
// modules built with the NDK. Modules built with the NDK
@@ -1261,46 +1450,14 @@
}
}
-// firstUniqueElements returns all unique elements of a slice, keeping the first copy of each
-// modifies the slice contents in place, and returns a subslice of the original slice
-func firstUniqueElements(list []string) []string {
- k := 0
-outer:
- for i := 0; i < len(list); i++ {
- for j := 0; j < k; j++ {
- if list[i] == list[j] {
- continue outer
- }
- }
- list[k] = list[i]
- k++
- }
- return list[:k]
-}
-
-// lastUniqueElements returns all unique elements of a slice, keeping the last copy of each
-// modifies the slice contents in place, and returns a subslice of the original slice
-func lastUniqueElements(list []string) []string {
- totalSkip := 0
- for i := len(list) - 1; i >= totalSkip; i-- {
- skip := 0
- for j := i - 1; j >= totalSkip; j-- {
- if list[i] == list[j] {
- skip++
- } else {
- list[j+skip] = list[j]
- }
- }
- totalSkip += skip
- }
- return list[totalSkip:]
-}
-
func getCurrentNdkPrebuiltVersion(ctx DepsContext) string {
- if ctx.AConfig().PlatformSdkVersionInt() > config.NdkMaxPrebuiltVersionInt {
+ if ctx.Config().PlatformSdkVersionInt() > config.NdkMaxPrebuiltVersionInt {
return strconv.Itoa(config.NdkMaxPrebuiltVersionInt)
}
- return ctx.AConfig().PlatformSdkVersion()
+ return ctx.Config().PlatformSdkVersion()
}
var Bool = proptools.Bool
+var BoolPtr = proptools.BoolPtr
+var String = proptools.String
+var StringPtr = proptools.StringPtr
diff --git a/cc/cc_test.go b/cc/cc_test.go
index 92120a5..148b4dd 100644
--- a/cc/cc_test.go
+++ b/cc/cc_test.go
@@ -1,108 +1,180 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package cc
import (
"android/soong/android"
+ "android/soong/genrule"
+
+ "fmt"
+ "io/ioutil"
+ "os"
"reflect"
+ "sort"
+ "strings"
"testing"
)
-var firstUniqueElementsTestCases = []struct {
- in []string
- out []string
-}{
- {
- in: []string{"a"},
- out: []string{"a"},
- },
- {
- in: []string{"a", "b"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"a", "a"},
- out: []string{"a"},
- },
- {
- in: []string{"a", "b", "a"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"b", "a", "a"},
- out: []string{"b", "a"},
- },
- {
- in: []string{"a", "a", "b"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"a", "b", "a", "b"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"liblog", "libdl", "libc++", "libdl", "libc", "libm"},
- out: []string{"liblog", "libdl", "libc++", "libc", "libm"},
- },
-}
+var buildDir string
-func TestFirstUniqueElements(t *testing.T) {
- for _, testCase := range firstUniqueElementsTestCases {
- out := firstUniqueElements(testCase.in)
- if !reflect.DeepEqual(out, testCase.out) {
- t.Errorf("incorrect output:")
- t.Errorf(" input: %#v", testCase.in)
- t.Errorf(" expected: %#v", testCase.out)
- t.Errorf(" got: %#v", out)
- }
+func setUp() {
+ var err error
+ buildDir, err = ioutil.TempDir("", "soong_cc_test")
+ if err != nil {
+ panic(err)
}
}
-var lastUniqueElementsTestCases = []struct {
- in []string
- out []string
-}{
- {
- in: []string{"a"},
- out: []string{"a"},
- },
- {
- in: []string{"a", "b"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"a", "a"},
- out: []string{"a"},
- },
- {
- in: []string{"a", "b", "a"},
- out: []string{"b", "a"},
- },
- {
- in: []string{"b", "a", "a"},
- out: []string{"b", "a"},
- },
- {
- in: []string{"a", "a", "b"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"a", "b", "a", "b"},
- out: []string{"a", "b"},
- },
- {
- in: []string{"liblog", "libdl", "libc++", "libdl", "libc", "libm"},
- out: []string{"liblog", "libc++", "libdl", "libc", "libm"},
- },
+func tearDown() {
+ os.RemoveAll(buildDir)
}
-func TestLastUniqueElements(t *testing.T) {
- for _, testCase := range lastUniqueElementsTestCases {
- out := lastUniqueElements(testCase.in)
- if !reflect.DeepEqual(out, testCase.out) {
- t.Errorf("incorrect output:")
- t.Errorf(" input: %#v", testCase.in)
- t.Errorf(" expected: %#v", testCase.out)
- t.Errorf(" got: %#v", out)
+func TestMain(m *testing.M) {
+ run := func() int {
+ setUp()
+ defer tearDown()
+
+ return m.Run()
+ }
+
+ os.Exit(run())
+}
+
+func testCc(t *testing.T, bp string) *android.TestContext {
+ config := android.TestArchConfig(buildDir, nil)
+ config.ProductVariables.DeviceVndkVersion = StringPtr("current")
+
+ ctx := android.NewTestArchContext()
+ ctx.RegisterModuleType("cc_library", android.ModuleFactoryAdaptor(LibraryFactory))
+ ctx.RegisterModuleType("cc_library_shared", android.ModuleFactoryAdaptor(LibrarySharedFactory))
+ ctx.RegisterModuleType("toolchain_library", android.ModuleFactoryAdaptor(toolchainLibraryFactory))
+ ctx.RegisterModuleType("llndk_library", android.ModuleFactoryAdaptor(llndkLibraryFactory))
+ ctx.RegisterModuleType("cc_object", android.ModuleFactoryAdaptor(objectFactory))
+ ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(genrule.FileGroupFactory))
+ ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) {
+ ctx.BottomUp("image", vendorMutator).Parallel()
+ ctx.BottomUp("link", linkageMutator).Parallel()
+ ctx.BottomUp("vndk", vndkMutator).Parallel()
+ })
+ ctx.Register()
+
+ // add some modules that are required by the compiler and/or linker
+ bp = bp + `
+ toolchain_library {
+ name: "libatomic",
+ vendor_available: true,
}
+
+ toolchain_library {
+ name: "libcompiler_rt-extras",
+ vendor_available: true,
+ }
+
+ toolchain_library {
+ name: "libgcc",
+ vendor_available: true,
+ }
+
+ cc_library {
+ name: "libc",
+ no_libgcc : true,
+ nocrt : true,
+ system_shared_libs: [],
+ }
+ llndk_library {
+ name: "libc",
+ symbol_file: "",
+ }
+ cc_library {
+ name: "libm",
+ no_libgcc : true,
+ nocrt : true,
+ system_shared_libs: [],
+ }
+ llndk_library {
+ name: "libm",
+ symbol_file: "",
+ }
+ cc_library {
+ name: "libdl",
+ no_libgcc : true,
+ nocrt : true,
+ system_shared_libs: [],
+ }
+ llndk_library {
+ name: "libdl",
+ symbol_file: "",
+ }
+
+ cc_object {
+ name: "crtbegin_so",
+ }
+
+ cc_object {
+ name: "crtend_so",
+ }
+
+ cc_library {
+ name: "libprotobuf-cpp-lite",
+ }
+
+`
+
+ ctx.MockFileSystem(map[string][]byte{
+ "Android.bp": []byte(bp),
+ "foo.c": nil,
+ "bar.c": nil,
+ "a.proto": nil,
+ "b.aidl": nil,
+ })
+
+ _, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+ failIfErrored(t, errs)
+ _, errs = ctx.PrepareBuildActions(config)
+ failIfErrored(t, errs)
+
+ return ctx
+}
+
+func TestVendorSrc(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libTest",
+ srcs: ["foo.c"],
+ no_libgcc : true,
+ nocrt : true,
+ system_shared_libs : [],
+ vendor_available: true,
+ target: {
+ vendor: {
+ srcs: ["bar.c"],
+ },
+ },
+ }
+ `)
+
+ ld := ctx.ModuleForTests("libTest", "android_arm_armv7-a-neon_vendor_shared").Rule("ld")
+ var objs []string
+ for _, o := range ld.Inputs {
+ objs = append(objs, o.Base())
+ }
+ if len(objs) != 2 {
+ t.Errorf("inputs of libTest is expected to 2, but was %d.", len(objs))
+ }
+ if objs[0] != "foo.o" || objs[1] != "bar.o" {
+ t.Errorf("inputs of libTest must be []string{\"foo.o\", \"bar.o\"}, but was %#v.", objs)
}
}
@@ -213,3 +285,385 @@
}
}
}
+
+var staticLinkDepOrderTestCases = []struct {
+ // This is a string representation of a map[moduleName][]moduleDependency .
+ // It models the dependencies declared in an Android.bp file.
+ inStatic string
+
+ // This is a string representation of a map[moduleName][]moduleDependency .
+ // It models the dependencies declared in an Android.bp file.
+ inShared string
+
+ // allOrdered is a string representation of a map[moduleName][]moduleDependency .
+ // The keys of allOrdered specify which modules we would like to check.
+ // The values of allOrdered specify the expected result (of the transitive closure of all
+ // dependencies) for each module to test
+ allOrdered string
+
+ // outOrdered is a string representation of a map[moduleName][]moduleDependency .
+ // The keys of outOrdered specify which modules we would like to check.
+ // The values of outOrdered specify the expected result (of the ordered linker command line)
+ // for each module to test.
+ outOrdered string
+}{
+ // Simple tests
+ {
+ inStatic: "",
+ outOrdered: "",
+ },
+ {
+ inStatic: "a:",
+ outOrdered: "a:",
+ },
+ {
+ inStatic: "a:b; b:",
+ outOrdered: "a:b; b:",
+ },
+ // Tests of reordering
+ {
+ // diamond example
+ inStatic: "a:d,b,c; b:d; c:d; d:",
+ outOrdered: "a:b,c,d; b:d; c:d; d:",
+ },
+ {
+ // somewhat real example
+ inStatic: "bsdiff_unittest:b,c,d,e,f,g,h,i; e:b",
+ outOrdered: "bsdiff_unittest:c,d,e,b,f,g,h,i; e:b",
+ },
+ {
+ // multiple reorderings
+ inStatic: "a:b,c,d,e; d:b; e:c",
+ outOrdered: "a:d,b,e,c; d:b; e:c",
+ },
+ {
+ // should reorder without adding new transitive dependencies
+ inStatic: "bin:lib2,lib1; lib1:lib2,liboptional",
+ allOrdered: "bin:lib1,lib2,liboptional; lib1:lib2,liboptional",
+ outOrdered: "bin:lib1,lib2; lib1:lib2,liboptional",
+ },
+ {
+ // multiple levels of dependencies
+ inStatic: "a:b,c,d,e,f,g,h; f:b,c,d; b:c,d; c:d",
+ allOrdered: "a:e,f,b,c,d,g,h; f:b,c,d; b:c,d; c:d",
+ outOrdered: "a:e,f,b,c,d,g,h; f:b,c,d; b:c,d; c:d",
+ },
+ // shared dependencies
+ {
+ // Note that this test doesn't recurse, to minimize the amount of logic it tests.
+ // So, we don't actually have to check that a shared dependency of c will change the order
+ // of a library that depends statically on b and on c. We only need to check that if c has
+ // a shared dependency on b, that that shows up in allOrdered.
+ inShared: "c:b",
+ allOrdered: "c:b",
+ outOrdered: "c:",
+ },
+ {
+ // This test doesn't actually include any shared dependencies but it's a reminder of what
+ // the second phase of the above test would look like
+ inStatic: "a:b,c; c:b",
+ allOrdered: "a:c,b; c:b",
+ outOrdered: "a:c,b; c:b",
+ },
+ // tiebreakers for when two modules specifying different orderings and there is no dependency
+ // to dictate an order
+ {
+ // if the tie is between two modules at the end of a's deps, then a's order wins
+ inStatic: "a1:b,c,d,e; a2:b,c,e,d; b:d,e; c:e,d",
+ outOrdered: "a1:b,c,d,e; a2:b,c,e,d; b:d,e; c:e,d",
+ },
+ {
+ // if the tie is between two modules at the start of a's deps, then c's order is used
+ inStatic: "a1:d,e,b1,c1; b1:d,e; c1:e,d; a2:d,e,b2,c2; b2:d,e; c2:d,e",
+ outOrdered: "a1:b1,c1,e,d; b1:d,e; c1:e,d; a2:b2,c2,d,e; b2:d,e; c2:d,e",
+ },
+ // Tests involving duplicate dependencies
+ {
+ // simple duplicate
+ inStatic: "a:b,c,c,b",
+ outOrdered: "a:c,b",
+ },
+ {
+ // duplicates with reordering
+ inStatic: "a:b,c,d,c; c:b",
+ outOrdered: "a:d,c,b",
+ },
+ // Tests to confirm the nonexistence of infinite loops.
+ // These cases should never happen, so as long as the test terminates and the
+ // result is deterministic then that should be fine.
+ {
+ inStatic: "a:a",
+ outOrdered: "a:a",
+ },
+ {
+ inStatic: "a:b; b:c; c:a",
+ allOrdered: "a:b,c; b:c,a; c:a,b",
+ outOrdered: "a:b; b:c; c:a",
+ },
+ {
+ inStatic: "a:b,c; b:c,a; c:a,b",
+ allOrdered: "a:c,a,b; b:a,b,c; c:b,c,a",
+ outOrdered: "a:c,b; b:a,c; c:b,a",
+ },
+}
+
+// converts from a string like "a:b,c; d:e" to (["a","b"], {"a":["b","c"], "d":["e"]}, [{"a", "a.o"}, {"b", "b.o"}])
+func parseModuleDeps(text string) (modulesInOrder []android.Path, allDeps map[android.Path][]android.Path) {
+ // convert from "a:b,c; d:e" to "a:b,c;d:e"
+ strippedText := strings.Replace(text, " ", "", -1)
+ if len(strippedText) < 1 {
+ return []android.Path{}, make(map[android.Path][]android.Path, 0)
+ }
+ allDeps = make(map[android.Path][]android.Path, 0)
+
+ // convert from "a:b,c;d:e" to ["a:b,c", "d:e"]
+ moduleTexts := strings.Split(strippedText, ";")
+
+ outputForModuleName := func(moduleName string) android.Path {
+ return android.PathForTesting(moduleName)
+ }
+
+ for _, moduleText := range moduleTexts {
+ // convert from "a:b,c" to ["a", "b,c"]
+ components := strings.Split(moduleText, ":")
+ if len(components) != 2 {
+ panic(fmt.Sprintf("illegal module dep string %q from larger string %q; must contain one ':', not %v", moduleText, text, len(components)-1))
+ }
+ moduleName := components[0]
+ moduleOutput := outputForModuleName(moduleName)
+ modulesInOrder = append(modulesInOrder, moduleOutput)
+
+ depString := components[1]
+ // convert from "b,c" to ["b", "c"]
+ depNames := strings.Split(depString, ",")
+ if len(depString) < 1 {
+ depNames = []string{}
+ }
+ var deps []android.Path
+ for _, depName := range depNames {
+ deps = append(deps, outputForModuleName(depName))
+ }
+ allDeps[moduleOutput] = deps
+ }
+ return modulesInOrder, allDeps
+}
+
+func TestLinkReordering(t *testing.T) {
+ for _, testCase := range staticLinkDepOrderTestCases {
+ errs := []string{}
+
+ // parse testcase
+ _, givenTransitiveDeps := parseModuleDeps(testCase.inStatic)
+ expectedModuleNames, expectedTransitiveDeps := parseModuleDeps(testCase.outOrdered)
+ if testCase.allOrdered == "" {
+ // allow the test case to skip specifying allOrdered
+ testCase.allOrdered = testCase.outOrdered
+ }
+ _, expectedAllDeps := parseModuleDeps(testCase.allOrdered)
+ _, givenAllSharedDeps := parseModuleDeps(testCase.inShared)
+
+ // For each module whose post-reordered dependencies were specified, validate that
+ // reordering the inputs produces the expected outputs.
+ for _, moduleName := range expectedModuleNames {
+ moduleDeps := givenTransitiveDeps[moduleName]
+ givenSharedDeps := givenAllSharedDeps[moduleName]
+ orderedAllDeps, orderedDeclaredDeps := orderDeps(moduleDeps, givenSharedDeps, givenTransitiveDeps)
+
+ correctAllOrdered := expectedAllDeps[moduleName]
+ if !reflect.DeepEqual(orderedAllDeps, correctAllOrdered) {
+ errs = append(errs, fmt.Sprintf("orderDeps returned incorrect orderedAllDeps."+
+ "\nin static:%q"+
+ "\nin shared:%q"+
+ "\nmodule: %v"+
+ "\nexpected: %s"+
+ "\nactual: %s",
+ testCase.inStatic, testCase.inShared, moduleName, correctAllOrdered, orderedAllDeps))
+ }
+
+ correctOutputDeps := expectedTransitiveDeps[moduleName]
+ if !reflect.DeepEqual(correctOutputDeps, orderedDeclaredDeps) {
+ errs = append(errs, fmt.Sprintf("orderDeps returned incorrect orderedDeclaredDeps."+
+ "\nin static:%q"+
+ "\nin shared:%q"+
+ "\nmodule: %v"+
+ "\nexpected: %s"+
+ "\nactual: %s",
+ testCase.inStatic, testCase.inShared, moduleName, correctOutputDeps, orderedDeclaredDeps))
+ }
+ }
+
+ if len(errs) > 0 {
+ sort.Strings(errs)
+ for _, err := range errs {
+ t.Error(err)
+ }
+ }
+ }
+}
+func failIfErrored(t *testing.T, errs []error) {
+ if len(errs) > 0 {
+ for _, err := range errs {
+ t.Error(err)
+ }
+ t.FailNow()
+ }
+}
+
+func getOutputPaths(ctx *android.TestContext, variant string, moduleNames []string) (paths android.Paths) {
+ for _, moduleName := range moduleNames {
+ module := ctx.ModuleForTests(moduleName, variant).Module().(*Module)
+ output := module.outputFile.Path()
+ paths = append(paths, output)
+ }
+ return paths
+}
+
+func TestStaticLibDepReordering(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "a",
+ static_libs: ["b", "c", "d"],
+ }
+ cc_library {
+ name: "b",
+ }
+ cc_library {
+ name: "c",
+ static_libs: ["b"],
+ }
+ cc_library {
+ name: "d",
+ }
+
+ `)
+
+ variant := "android_arm64_armv8-a_core_static"
+ moduleA := ctx.ModuleForTests("a", variant).Module().(*Module)
+ actual := moduleA.depsInLinkOrder
+ expected := getOutputPaths(ctx, variant, []string{"c", "b", "d"})
+
+ if !reflect.DeepEqual(actual, expected) {
+ t.Errorf("staticDeps orderings were not propagated correctly"+
+ "\nactual: %v"+
+ "\nexpected: %v",
+ actual,
+ expected,
+ )
+ }
+}
+
+func TestStaticLibDepReorderingWithShared(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "a",
+ static_libs: ["b", "c"],
+ }
+ cc_library {
+ name: "b",
+ }
+ cc_library {
+ name: "c",
+ shared_libs: ["b"],
+ }
+
+ `)
+
+ variant := "android_arm64_armv8-a_core_static"
+ moduleA := ctx.ModuleForTests("a", variant).Module().(*Module)
+ actual := moduleA.depsInLinkOrder
+ expected := getOutputPaths(ctx, variant, []string{"c", "b"})
+
+ if !reflect.DeepEqual(actual, expected) {
+ t.Errorf("staticDeps orderings did not account for shared libs"+
+ "\nactual: %v"+
+ "\nexpected: %v",
+ actual,
+ expected,
+ )
+ }
+}
+
+var compilerFlagsTestCases = []struct {
+ in string
+ out bool
+}{
+ {
+ in: "a",
+ out: false,
+ },
+ {
+ in: "-a",
+ out: true,
+ },
+ {
+ in: "-Ipath/to/something",
+ out: false,
+ },
+ {
+ in: "-isystempath/to/something",
+ out: false,
+ },
+ {
+ in: "--coverage",
+ out: false,
+ },
+ {
+ in: "-include a/b",
+ out: true,
+ },
+ {
+ in: "-include a/b c/d",
+ out: false,
+ },
+ {
+ in: "-DMACRO",
+ out: true,
+ },
+ {
+ in: "-DMAC RO",
+ out: false,
+ },
+ {
+ in: "-a -b",
+ out: false,
+ },
+ {
+ in: "-DMACRO=definition",
+ out: true,
+ },
+ {
+ in: "-DMACRO=defi nition",
+ out: true, // TODO(jiyong): this should be false
+ },
+ {
+ in: "-DMACRO(x)=x + 1",
+ out: true,
+ },
+ {
+ in: "-DMACRO=\"defi nition\"",
+ out: true,
+ },
+}
+
+type mockContext struct {
+ BaseModuleContext
+ result bool
+}
+
+func (ctx *mockContext) PropertyErrorf(property, format string, args ...interface{}) {
+ // CheckBadCompilerFlags calls this function when the flag should be rejected
+ ctx.result = false
+}
+
+func TestCompilerFlags(t *testing.T) {
+ for _, testCase := range compilerFlagsTestCases {
+ ctx := &mockContext{result: true}
+ CheckBadCompilerFlags(ctx, "", []string{testCase.in})
+ if ctx.result != testCase.out {
+ t.Errorf("incorrect output:")
+ t.Errorf(" input: %#v", testCase.in)
+ t.Errorf(" expected: %#v", testCase.out)
+ t.Errorf(" got: %#v", ctx.result)
+ }
+ }
+}
diff --git a/cc/check.go b/cc/check.go
index d04b145..4e9e160 100644
--- a/cc/check.go
+++ b/cc/check.go
@@ -50,6 +50,11 @@
} else if strings.HasPrefix("../", path) {
ctx.PropertyErrorf(prop, "Path must not start with `../`: `%s`. Use include_dirs to -include from a different directory", flag)
}
+ } else if strings.HasPrefix(flag, "-D") && strings.Contains(flag, "=") {
+ // Do nothing in this case.
+ // For now, we allow space characters in -DNAME=def form to allow use cases
+ // like -DNAME="value with string". Later, this check should be done more
+ // correctly to prevent multi flag cases like -DNAME=value -O2.
} else {
ctx.PropertyErrorf(prop, "Bad flag: `%s` is not an allowed multi-word flag. Should it be split into multiple flags?", flag)
}
diff --git a/cc/cmakelists.go b/cc/cmakelists.go
index c353f44..c25578e 100644
--- a/cc/cmakelists.go
+++ b/cc/cmakelists.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package cc
import (
@@ -9,8 +23,6 @@
"path"
"path/filepath"
"strings"
-
- "github.com/google/blueprint"
)
// This singleton generates CMakeLists.txt files. It does so for each blueprint Android.bp resulting in a cc.Module
@@ -21,7 +33,7 @@
android.RegisterSingletonType("cmakelists_generator", cMakeListsGeneratorSingleton)
}
-func cMakeListsGeneratorSingleton() blueprint.Singleton {
+func cMakeListsGeneratorSingleton() android.Singleton {
return &cmakelistsGeneratorSingleton{}
}
@@ -43,14 +55,14 @@
// This is done to ease investigating bug reports.
var outputDebugInfo = false
-func (c *cmakelistsGeneratorSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
+func (c *cmakelistsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonContext) {
if getEnvVariable(envVariableGenerateCMakeLists, ctx) != envVariableTrue {
return
}
outputDebugInfo = (getEnvVariable(envVariableGenerateDebugInfo, ctx) == envVariableTrue)
- ctx.VisitAllModules(func(module blueprint.Module) {
+ ctx.VisitAllModules(func(module android.Module) {
if ccModule, ok := module.(*Module); ok {
if compiledModule, ok := ccModule.compiler.(CompiledInterface); ok {
generateCLionProject(compiledModule, ctx, ccModule)
@@ -67,10 +79,10 @@
return
}
-func getEnvVariable(name string, ctx blueprint.SingletonContext) string {
+func getEnvVariable(name string, ctx android.SingletonContext) string {
// Using android.Config.Getenv instead of os.getEnv to guarantee soong will
// re-run in case this environment variable changes.
- return ctx.Config().(android.Config).Getenv(name)
+ return ctx.Config().Getenv(name)
}
func exists(path string) bool {
@@ -102,7 +114,7 @@
return nil
}
-func generateCLionProject(compiledModule CompiledInterface, ctx blueprint.SingletonContext, ccModule *Module) {
+func generateCLionProject(compiledModule CompiledInterface, ctx android.SingletonContext, ccModule *Module) {
srcs := compiledModule.Srcs()
if len(srcs) == 0 {
return
@@ -273,7 +285,7 @@
return flag
}
-func parseCompilerParameters(params []string, ctx blueprint.SingletonContext, f *os.File) compilerParameters {
+func parseCompilerParameters(params []string, ctx android.SingletonContext, f *os.File) compilerParameters {
var compilerParameters = makeCompilerParameters()
for i, str := range params {
@@ -374,7 +386,7 @@
c1.flags = append(c1.flags, c2.flags...)
}
-func evalVariable(ctx blueprint.SingletonContext, str string) (string, error) {
+func evalVariable(ctx android.SingletonContext, str string) (string, error) {
evaluated, err := ctx.Eval(pctx, str)
if err == nil {
return evaluated, nil
@@ -382,7 +394,7 @@
return "", err
}
-func getCMakeListsForModule(module *Module, ctx blueprint.SingletonContext) string {
+func getCMakeListsForModule(module *Module, ctx android.SingletonContext) string {
return filepath.Join(getAndroidSrcRootDirectory(ctx),
cLionOutputProjectsDirectory,
path.Dir(ctx.BlueprintFile(module)),
@@ -392,7 +404,7 @@
cMakeListsFilename)
}
-func getAndroidSrcRootDirectory(ctx blueprint.SingletonContext) string {
+func getAndroidSrcRootDirectory(ctx android.SingletonContext) string {
srcPath, _ := filepath.Abs(android.PathForSource(ctx).String())
return srcPath
}
diff --git a/cc/compiler.go b/cc/compiler.go
index f162878..40dbea4 100644
--- a/cc/compiler.go
+++ b/cc/compiler.go
@@ -62,7 +62,7 @@
// the instruction set architecture to use to compile the C/C++
// module.
- Instruction_set string `android:"arch_variant"`
+ Instruction_set *string `android:"arch_variant"`
// list of directories relative to the root of the source tree that will
// be added to the include path using -I.
@@ -89,12 +89,12 @@
// C standard version to use. Can be a specific version (such as "gnu11"),
// "experimental" (which will use draft versions like C1x when available),
// or the empty string (which will use the default).
- C_std string
+ C_std *string
// C++ standard version to use. Can be a specific version (such as
// "gnu++11"), "experimental" (which will use draft versions like C++1z when
// available), or the empty string (which will use the default).
- Cpp_std string
+ Cpp_std *string
// if set to false, use -std=c++* instead of -std=gnu++*
Gnu_extensions *bool
@@ -134,9 +134,18 @@
// list of source files that should not be used to
// build the vendor variant of the C/C++ module.
Exclude_srcs []string
+
+ // List of additional cflags that should be used to build the vendor
+ // variant of the C/C++ module.
+ Cflags []string
}
}
+ Proto struct {
+ // Link statically against the protobuf runtime
+ Static *bool `android:"arch_variant"`
+ } `android:"arch_variant"`
+
// Stores the original list of source files before being cleared by library reuse
OriginalSrcs []string `blueprint:"mutated"`
}
@@ -147,10 +156,17 @@
type baseCompiler struct {
Properties BaseCompilerProperties
- Proto ProtoProperties
+ Proto android.ProtoProperties
deps android.Paths
- srcs android.Paths
flags builderFlags
+
+ // Sources that were passed to the C/C++ compiler
+ srcs android.Paths
+
+ // Sources that were passed in the Android.bp file, including generated sources generated by
+ // other modules and filegroups. May include source files that have not yet been translated to
+ // C/C++ (.aidl, .proto, etc.)
+ srcsBeforeGen android.Paths
}
var _ compiler = (*baseCompiler)(nil)
@@ -184,17 +200,35 @@
android.ExtractSourcesDeps(ctx, compiler.Properties.Srcs)
if compiler.hasSrcExt(".proto") {
- deps = protoDeps(ctx, deps, &compiler.Proto)
+ deps = protoDeps(ctx, deps, &compiler.Proto, Bool(compiler.Properties.Proto.Static))
}
return deps
}
+// Return true if the module is in the WarningAllowedProjects.
+func warningsAreAllowed(subdir string) bool {
+ subdir += "/"
+ for _, prefix := range config.WarningAllowedProjects {
+ if strings.HasPrefix(subdir, prefix) {
+ return true
+ }
+ }
+ return false
+}
+
+func addToModuleList(ctx ModuleContext, list string, module string) {
+ getWallWerrorMap(ctx.Config(), list).Store(module, true)
+}
+
// Create a Flags struct that collects the compile flags from global values,
// per-target values, module type values, and per-module Blueprints properties
-func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flags {
+func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags {
tc := ctx.toolchain()
+ compiler.srcsBeforeGen = ctx.ExpandSources(compiler.Properties.Srcs, compiler.Properties.Exclude_srcs)
+ compiler.srcsBeforeGen = append(compiler.srcsBeforeGen, deps.GeneratedSources...)
+
CheckBadCompilerFlags(ctx, "cflags", compiler.Properties.Cflags)
CheckBadCompilerFlags(ctx, "cppflags", compiler.Properties.Cppflags)
CheckBadCompilerFlags(ctx, "conlyflags", compiler.Properties.Conlyflags)
@@ -223,19 +257,17 @@
flags.YasmFlags = append(flags.YasmFlags, f)
}
- if !ctx.noDefaultCompilerFlags() {
- flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.PathForModuleSrc(ctx).String())
- flags.YasmFlags = append(flags.YasmFlags, "-I"+android.PathForModuleSrc(ctx).String())
+ flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.PathForModuleSrc(ctx).String())
+ flags.YasmFlags = append(flags.YasmFlags, "-I"+android.PathForModuleSrc(ctx).String())
- if !(ctx.sdk() || ctx.vndk()) || ctx.Host() {
- flags.SystemIncludeFlags = append(flags.SystemIncludeFlags,
- "${config.CommonGlobalIncludes}",
- tc.IncludeFlags(),
- "${config.CommonNativehelperInclude}")
- }
+ if !(ctx.useSdk() || ctx.useVndk()) || ctx.Host() {
+ flags.SystemIncludeFlags = append(flags.SystemIncludeFlags,
+ "${config.CommonGlobalIncludes}",
+ tc.IncludeFlags(),
+ "${config.CommonNativehelperInclude}")
}
- if ctx.sdk() {
+ if ctx.useSdk() {
// The NDK headers are installed to a common sysroot. While a more
// typical Soong approach would be to only make the headers for the
// library you're using available, we're trying to emulate the NDK
@@ -263,12 +295,12 @@
flags.SystemIncludeFlags = append(flags.SystemIncludeFlags, "-isystem "+legacyIncludes)
}
- if ctx.vndk() {
+ if ctx.useVndk() {
flags.GlobalFlags = append(flags.GlobalFlags,
"-D__ANDROID_API__=__ANDROID_API_FUTURE__", "-D__ANDROID_VNDK__")
}
- instructionSet := compiler.Properties.Instruction_set
+ instructionSet := String(compiler.Properties.Instruction_set)
if flags.RequiredInstructionSet != "" {
instructionSet = flags.RequiredInstructionSet
}
@@ -309,71 +341,70 @@
hod = "Device"
}
- if !ctx.noDefaultCompilerFlags() {
- flags.GlobalFlags = append(flags.GlobalFlags, instructionSetFlags)
- flags.ConlyFlags = append([]string{"${config.CommonGlobalConlyflags}"}, flags.ConlyFlags...)
+ flags.GlobalFlags = append(flags.GlobalFlags, instructionSetFlags)
+ flags.ConlyFlags = append([]string{"${config.CommonGlobalConlyflags}"}, flags.ConlyFlags...)
+ flags.CppFlags = append([]string{fmt.Sprintf("${config.%sGlobalCppflags}", hod)}, flags.CppFlags...)
- if flags.Clang {
- flags.AsFlags = append(flags.AsFlags, tc.ClangAsflags())
- flags.CppFlags = append([]string{"${config.CommonClangGlobalCppflags}"}, flags.CppFlags...)
- flags.GlobalFlags = append(flags.GlobalFlags,
- tc.ClangCflags(),
- "${config.CommonClangGlobalCflags}",
- fmt.Sprintf("${config.%sClangGlobalCflags}", hod))
- } else {
- flags.CppFlags = append([]string{"${config.CommonGlobalCppflags}"}, flags.CppFlags...)
- flags.GlobalFlags = append(flags.GlobalFlags,
- tc.Cflags(),
- "${config.CommonGlobalCflags}",
- fmt.Sprintf("${config.%sGlobalCflags}", hod))
- }
-
- if Bool(ctx.AConfig().ProductVariables.Brillo) {
- flags.GlobalFlags = append(flags.GlobalFlags, "-D__BRILLO__")
- }
-
- if ctx.Device() {
- if Bool(compiler.Properties.Rtti) {
- flags.CppFlags = append(flags.CppFlags, "-frtti")
- } else {
- flags.CppFlags = append(flags.CppFlags, "-fno-rtti")
- }
- }
-
- flags.AsFlags = append(flags.AsFlags, "-D__ASSEMBLY__")
-
- if flags.Clang {
- flags.CppFlags = append(flags.CppFlags, tc.ClangCppflags())
- } else {
- flags.CppFlags = append(flags.CppFlags, tc.Cppflags())
- }
-
- flags.YasmFlags = append(flags.YasmFlags, tc.YasmFlags())
+ if flags.Clang {
+ flags.AsFlags = append(flags.AsFlags, tc.ClangAsflags())
+ flags.CppFlags = append([]string{"${config.CommonClangGlobalCppflags}"}, flags.CppFlags...)
+ flags.GlobalFlags = append(flags.GlobalFlags,
+ tc.ClangCflags(),
+ "${config.CommonClangGlobalCflags}",
+ fmt.Sprintf("${config.%sClangGlobalCflags}", hod))
+ } else {
+ flags.CppFlags = append([]string{"${config.CommonGlobalCppflags}"}, flags.CppFlags...)
+ flags.GlobalFlags = append(flags.GlobalFlags,
+ tc.Cflags(),
+ "${config.CommonGlobalCflags}",
+ fmt.Sprintf("${config.%sGlobalCflags}", hod))
}
+ if Bool(ctx.Config().ProductVariables.Brillo) {
+ flags.GlobalFlags = append(flags.GlobalFlags, "-D__BRILLO__")
+ }
+
+ if ctx.Device() {
+ if Bool(compiler.Properties.Rtti) {
+ flags.CppFlags = append(flags.CppFlags, "-frtti")
+ } else {
+ flags.CppFlags = append(flags.CppFlags, "-fno-rtti")
+ }
+ }
+
+ flags.AsFlags = append(flags.AsFlags, "-D__ASSEMBLY__")
+
+ if flags.Clang {
+ flags.CppFlags = append(flags.CppFlags, tc.ClangCppflags())
+ } else {
+ flags.CppFlags = append(flags.CppFlags, tc.Cppflags())
+ }
+
+ flags.YasmFlags = append(flags.YasmFlags, tc.YasmFlags())
+
if flags.Clang {
flags.GlobalFlags = append(flags.GlobalFlags, tc.ToolchainClangCflags())
} else {
flags.GlobalFlags = append(flags.GlobalFlags, tc.ToolchainCflags())
}
- if !ctx.sdk() {
+ if !ctx.useSdk() {
cStd := config.CStdVersion
- if compiler.Properties.C_std == "experimental" {
+ if String(compiler.Properties.C_std) == "experimental" {
cStd = config.ExperimentalCStdVersion
- } else if compiler.Properties.C_std != "" {
- cStd = compiler.Properties.C_std
+ } else if String(compiler.Properties.C_std) != "" {
+ cStd = String(compiler.Properties.C_std)
}
- cppStd := compiler.Properties.Cpp_std
- switch compiler.Properties.Cpp_std {
+ cppStd := String(compiler.Properties.Cpp_std)
+ switch String(compiler.Properties.Cpp_std) {
case "":
cppStd = config.CppStdVersion
case "experimental":
cppStd = config.ExperimentalCppStdVersion
case "c++17", "gnu++17":
// Map c++17 and gnu++17 to their 1z equivalents, until 17 is finalized.
- cppStd = strings.Replace(compiler.Properties.Cpp_std, "17", "1z", 1)
+ cppStd = strings.Replace(String(compiler.Properties.Cpp_std), "17", "1z", 1)
}
if !flags.Clang {
@@ -396,6 +427,10 @@
flags.CppFlags = append([]string{"-std=" + cppStd}, flags.CppFlags...)
}
+ if ctx.useVndk() {
+ flags.CFlags = append(flags.CFlags, esc(compiler.Properties.Target.Vendor.Cflags)...)
+ }
+
// We can enforce some rules more strictly in the code we own. strict
// indicates if this is code that we can be stricter with. If we have
// rules that we want to apply to *our* code (but maybe can't for
@@ -421,6 +456,11 @@
"-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String())
}
+ if compiler.hasSrcExt(".mc") {
+ flags.GlobalFlags = append(flags.GlobalFlags,
+ "-I"+android.PathForModuleGen(ctx, "windmc", ctx.ModuleDir()).String())
+ }
+
if compiler.hasSrcExt(".aidl") {
if len(compiler.Properties.Aidl.Local_include_dirs) > 0 {
localAidlIncludeDirs := android.PathsForModuleSrc(ctx, compiler.Properties.Aidl.Local_include_dirs)
@@ -439,10 +479,29 @@
flags = rsFlags(ctx, flags, &compiler.Properties)
}
+ if len(compiler.Properties.Srcs) > 0 {
+ module := ctx.ModuleDir() + "/Android.bp:" + ctx.ModuleName()
+ if inList("-Wno-error", flags.CFlags) || inList("-Wno-error", flags.CppFlags) {
+ addToModuleList(ctx, modulesUsingWnoError, module)
+ } else if !inList("-Werror", flags.CFlags) && !inList("-Werror", flags.CppFlags) {
+ if warningsAreAllowed(ctx.ModuleDir()) {
+ addToModuleList(ctx, modulesAddedWall, module)
+ flags.CFlags = append([]string{"-Wall"}, flags.CFlags...)
+ } else {
+ flags.CFlags = append([]string{"-Wall", "-Werror"}, flags.CFlags...)
+ }
+ }
+ }
+
return flags
}
func (compiler *baseCompiler) hasSrcExt(ext string) bool {
+ for _, src := range compiler.srcsBeforeGen {
+ if src.Ext() == ext {
+ return true
+ }
+ }
for _, src := range compiler.Properties.Srcs {
if filepath.Ext(src) == ext {
return true
@@ -460,7 +519,7 @@
var gnuToCReplacer = strings.NewReplacer("gnu", "c")
func ndkPathDeps(ctx ModuleContext) android.Paths {
- if ctx.sdk() {
+ if ctx.useSdk() {
// The NDK sysroot timestamp file depends on all the NDK sysroot files
// (headers and libraries).
return android.Paths{getNdkSysrootTimestampFile(ctx)}
@@ -472,19 +531,10 @@
pathDeps := deps.GeneratedHeaders
pathDeps = append(pathDeps, ndkPathDeps(ctx)...)
- if ctx.vndk() {
- compiler.Properties.Srcs = append(compiler.Properties.Srcs,
- compiler.Properties.Target.Vendor.Srcs...)
-
- compiler.Properties.Exclude_srcs = append(compiler.Properties.Exclude_srcs,
- compiler.Properties.Target.Vendor.Exclude_srcs...)
- }
-
- srcs := ctx.ExpandSources(compiler.Properties.Srcs, compiler.Properties.Exclude_srcs)
- srcs = append(srcs, deps.GeneratedSources...)
-
buildFlags := flagsToBuilderFlags(flags)
+ srcs := append(android.Paths(nil), compiler.srcsBeforeGen...)
+
srcs, genDeps := genSources(ctx, srcs, buildFlags)
pathDeps = append(pathDeps, genDeps...)
diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go
index 139c901..5bb7749 100644
--- a/cc/config/arm64_device.go
+++ b/cc/config/arm64_device.go
@@ -23,54 +23,19 @@
var (
arm64Cflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
- "-fno-strict-aliasing",
- "-fstack-protector-strong",
- "-ffunction-sections",
- "-fdata-sections",
- "-funwind-tables",
- "-Wa,--noexecstack",
- "-Werror=format-security",
- "-D_FORTIFY_SOURCE=2",
- "-fno-short-enums",
- "-no-canonical-prefixes",
- "-fno-canonical-system-headers",
-
// Help catch common 32/64-bit errors.
- "-Werror=pointer-to-int-cast",
- "-Werror=int-to-pointer-cast",
"-Werror=implicit-function-declaration",
-
- "-fno-strict-volatile-bitfields",
-
- // TARGET_RELEASE_CFLAGS
- "-DNDEBUG",
- "-O2 -g",
- "-Wstrict-aliasing=2",
- "-fgcse-after-reload",
- "-frerun-cse-after-loop",
- "-frename-registers",
}
arm64Ldflags = []string{
- "-Wl,-z,noexecstack",
- "-Wl,-z,relro",
- "-Wl,-z,now",
- "-Wl,--build-id=md5",
- "-Wl,--warn-shared-textrel",
- "-Wl,--fatal-warnings",
- "-Wl,-maarch64linux",
+ "-Wl,-m,aarch64_elf64_le_vec",
"-Wl,--hash-style=gnu",
"-Wl,--fix-cortex-a53-843419",
"-fuse-ld=gold",
"-Wl,--icf=safe",
- "-Wl,--no-undefined-version",
}
- arm64Cppflags = []string{
- "-fvisibility-inlines-hidden",
- }
+ arm64Cppflags = []string{}
arm64CpuVariantCflags = map[string][]string{
"cortex-a53": []string{
@@ -81,6 +46,9 @@
// don't support a Kryo specific target yet.
"-mcpu=cortex-a57",
},
+ "exynos-m1": []string{
+ "-mcpu=exynos-m1",
+ },
"exynos-m2": []string{
"-mcpu=exynos-m2",
},
@@ -99,6 +67,7 @@
"cortex-a53",
"cortex-a73",
"kryo",
+ "exynos-m1",
"exynos-m2",
"denver64")
@@ -113,7 +82,7 @@
pctx.StaticVariable("Arm64Cflags", strings.Join(arm64Cflags, " "))
pctx.StaticVariable("Arm64Ldflags", strings.Join(arm64Ldflags, " "))
pctx.StaticVariable("Arm64Cppflags", strings.Join(arm64Cppflags, " "))
- pctx.StaticVariable("Arm64IncludeFlags", bionicHeaders("arm64", "arm64"))
+ pctx.StaticVariable("Arm64IncludeFlags", bionicHeaders("arm64"))
pctx.StaticVariable("Arm64ClangCflags", strings.Join(ClangFilterUnknownCflags(arm64Cflags), " "))
pctx.StaticVariable("Arm64ClangLdflags", strings.Join(ClangFilterUnknownCflags(arm64Ldflags), " "))
@@ -129,6 +98,11 @@
pctx.StaticVariable("Arm64ClangKryoCflags",
strings.Join(arm64ClangCpuVariantCflags["kryo"], " "))
+ pctx.StaticVariable("Arm64ExynosM1Cflags",
+ strings.Join(arm64CpuVariantCflags["cortex-a53"], " "))
+ pctx.StaticVariable("Arm64ClangExynosM1Cflags",
+ strings.Join(arm64ClangCpuVariantCflags["exynos-m1"], " "))
+
pctx.StaticVariable("Arm64ExynosM2Cflags",
strings.Join(arm64CpuVariantCflags["cortex-a53"], " "))
pctx.StaticVariable("Arm64ClangExynosM2Cflags",
@@ -141,6 +115,7 @@
"cortex-a53": "${config.Arm64CortexA53Cflags}",
"cortex-a73": "${config.Arm64CortexA53Cflags}",
"kryo": "${config.Arm64KryoCflags}",
+ "exynos-m1": "${config.Arm64ExynosM1Cflags}",
"exynos-m2": "${config.Arm64ExynosM2Cflags}",
}
@@ -149,6 +124,7 @@
"cortex-a53": "${config.Arm64ClangCortexA53Cflags}",
"cortex-a73": "${config.Arm64ClangCortexA53Cflags}",
"kryo": "${config.Arm64ClangKryoCflags}",
+ "exynos-m1": "${config.Arm64ClangExynosM1Cflags}",
"exynos-m2": "${config.Arm64ClangExynosM2Cflags}",
}
)
diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go
index e97e723..fda4f9d 100644
--- a/cc/config/arm_device.go
+++ b/cc/config/arm_device.go
@@ -28,59 +28,24 @@
}
armCflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
- "-ffunction-sections",
- "-fdata-sections",
- "-funwind-tables",
- "-fstack-protector-strong",
- "-Wa,--noexecstack",
- "-Werror=format-security",
- "-D_FORTIFY_SOURCE=2",
- "-fno-short-enums",
- "-no-canonical-prefixes",
- "-fno-canonical-system-headers",
-
- "-fno-builtin-sin",
- "-fno-strict-volatile-bitfields",
-
- // TARGET_RELEASE_CFLAGS
- "-DNDEBUG",
- "-g",
- "-Wstrict-aliasing=2",
- "-fgcse-after-reload",
- "-frerun-cse-after-loop",
- "-frename-registers",
+ "-fomit-frame-pointer",
}
- armCppflags = []string{
- "-fvisibility-inlines-hidden",
- }
+ armCppflags = []string{}
armLdflags = []string{
- "-Wl,-z,noexecstack",
- "-Wl,-z,relro",
- "-Wl,-z,now",
- "-Wl,--build-id=md5",
- "-Wl,--warn-shared-textrel",
- "-Wl,--fatal-warnings",
"-Wl,--icf=safe",
"-Wl,--hash-style=gnu",
- "-Wl,--no-undefined-version",
+ "-Wl,-m,armelf",
}
armArmCflags = []string{
- "-O2",
- "-fomit-frame-pointer",
"-fstrict-aliasing",
- "-funswitch-loops",
}
armThumbCflags = []string{
"-mthumb",
"-Os",
- "-fomit-frame-pointer",
- "-fno-strict-aliasing",
}
armArchVariantCflags = map[string][]string{
@@ -98,15 +63,18 @@
"-mfpu=vfpv3-d16",
},
"armv7-a-neon": []string{
+ "-march=armv7-a",
"-mfloat-abi=softfp",
"-mfpu=neon",
},
+ "armv8-a": []string{
+ "-march=armv8-a",
+ "-mfloat-abi=softfp",
+ "-mfpu=neon-fp-armv8",
+ },
}
armCpuVariantCflags = map[string][]string{
- "": []string{
- "-march=armv7-a",
- },
"cortex-a7": []string{
"-mcpu=cortex-a7",
"-mfpu=neon-vfpv4",
@@ -147,7 +115,9 @@
"-D__ARM_FEATURE_LPAE=1",
},
"kryo": []string{
- "-mcpu=cortex-a15",
+ // Use cortex-a53 because the GNU assembler doesn't recognize -mcpu=kryo
+ // even though clang does.
+ "-mcpu=cortex-a53",
"-mfpu=neon-fp-armv8",
// Fake an ARM compiler flag as these processors support LPAE which GCC/clang
// don't advertise.
@@ -173,6 +143,7 @@
"armv5te",
"armv7-a",
"armv7-a-neon",
+ "armv8-a",
"cortex-a7",
"cortex-a8",
"cortex-a9",
@@ -182,15 +153,22 @@
"cortex-a73",
"krait",
"kryo",
+ "exynos-m1",
"exynos-m2",
"denver")
android.RegisterArchVariantFeatures(android.Arm, "armv7-a-neon", "neon")
+ android.RegisterArchVariantFeatures(android.Arm, "armv8-a", "neon")
- // Krait and Kryo targets are not supported by GCC, but are supported by Clang,
- // so override the definitions when building modules with Clang.
+ // Krait is not supported by GCC, but is supported by Clang, so
+ // override the definitions when building modules with Clang.
replaceFirst(armClangCpuVariantCflags["krait"], "-mcpu=cortex-a15", "-mcpu=krait")
- replaceFirst(armClangCpuVariantCflags["kryo"], "-mcpu=cortex-a15", "-mcpu=krait")
+
+ // The reason we use "-march=armv8-a+crc", instead of "-march=armv8-a", for
+ // gcc is the latter would conflict with any specified/supported -mcpu!
+ // All armv8-a cores supported by gcc 4.9 support crc, so it's safe
+ // to add +crc. Besides, the use of gcc is only for legacy code.
+ replaceFirst(armArchVariantCflags["armv8-a"], "-march=armv8-a", "-march=armv8-a+crc")
pctx.StaticVariable("armGccVersion", armGccVersion)
@@ -201,7 +179,7 @@
pctx.StaticVariable("ArmCflags", strings.Join(armCflags, " "))
pctx.StaticVariable("ArmLdflags", strings.Join(armLdflags, " "))
pctx.StaticVariable("ArmCppflags", strings.Join(armCppflags, " "))
- pctx.StaticVariable("ArmIncludeFlags", bionicHeaders("arm", "arm"))
+ pctx.StaticVariable("ArmIncludeFlags", bionicHeaders("arm"))
// Extended cflags
@@ -213,6 +191,7 @@
pctx.StaticVariable("ArmArmv5TECflags", strings.Join(armArchVariantCflags["armv5te"], " "))
pctx.StaticVariable("ArmArmv7ACflags", strings.Join(armArchVariantCflags["armv7-a"], " "))
pctx.StaticVariable("ArmArmv7ANeonCflags", strings.Join(armArchVariantCflags["armv7-a-neon"], " "))
+ pctx.StaticVariable("ArmArmv8ACflags", strings.Join(armArchVariantCflags["armv8-a"], " "))
// Cpu variant cflags
pctx.StaticVariable("ArmGenericCflags", strings.Join(armCpuVariantCflags[""], " "))
@@ -240,6 +219,8 @@
strings.Join(armClangArchVariantCflags["armv7-a"], " "))
pctx.StaticVariable("ArmClangArmv7ANeonCflags",
strings.Join(armClangArchVariantCflags["armv7-a-neon"], " "))
+ pctx.StaticVariable("ArmClangArmv8ACflags",
+ strings.Join(armClangArchVariantCflags["armv8-a"], " "))
// Clang cpu variant cflags
pctx.StaticVariable("ArmClangGenericCflags",
@@ -263,6 +244,7 @@
"armv5te": "${config.ArmArmv5TECflags}",
"armv7-a": "${config.ArmArmv7ACflags}",
"armv7-a-neon": "${config.ArmArmv7ANeonCflags}",
+ "armv8-a": "${config.ArmArmv8ACflags}",
}
armCpuVariantCflagsVar = map[string]string{
@@ -275,6 +257,7 @@
"cortex-a73": "${config.ArmCortexA53Cflags}",
"krait": "${config.ArmKraitCflags}",
"kryo": "${config.ArmKryoCflags}",
+ "exynos-m1": "${config.ArmCortexA53Cflags}",
"exynos-m2": "${config.ArmCortexA53Cflags}",
"denver": "${config.ArmCortexA15Cflags}",
}
@@ -283,6 +266,7 @@
"armv5te": "${config.ArmClangArmv5TECflags}",
"armv7-a": "${config.ArmClangArmv7ACflags}",
"armv7-a-neon": "${config.ArmClangArmv7ANeonCflags}",
+ "armv8-a": "${config.ArmClangArmv8ACflags}",
}
armClangCpuVariantCflagsVar = map[string]string{
@@ -295,6 +279,7 @@
"cortex-a73": "${config.ArmClangCortexA53Cflags}",
"krait": "${config.ArmClangKraitCflags}",
"kryo": "${config.ArmClangKryoCflags}",
+ "exynos-m1": "${config.ArmClangCortexA53Cflags}",
"exynos-m2": "${config.ArmClangCortexA53Cflags}",
"denver": "${config.ArmClangCortexA15Cflags}",
}
@@ -398,6 +383,11 @@
toolchainClangCflags[0] = "${config.ArmToolchainClangCflags}"
toolchainClangCflags[1] = armClangArchVariantCflagsVar[arch.ArchVariant]
+ toolchainCflags = append(toolchainCflags,
+ variantOrDefault(armCpuVariantCflagsVar, arch.CpuVariant))
+ toolchainClangCflags = append(toolchainClangCflags,
+ variantOrDefault(armClangCpuVariantCflagsVar, arch.CpuVariant))
+
switch arch.ArchVariant {
case "armv7-a-neon":
switch arch.CpuVariant {
@@ -407,15 +397,12 @@
default:
fixCortexA8 = "-Wl,--no-fix-cortex-a8"
}
-
- toolchainCflags = append(toolchainCflags,
- variantOrDefault(armCpuVariantCflagsVar, arch.CpuVariant))
- toolchainClangCflags = append(toolchainClangCflags,
- variantOrDefault(armClangCpuVariantCflagsVar, arch.CpuVariant))
case "armv7-a":
fixCortexA8 = "-Wl,--fix-cortex-a8"
case "armv5te":
// Nothing extra for armv5te
+ case "armv8-a":
+ // Nothing extra for armv8-a
default:
panic(fmt.Sprintf("Unknown ARM architecture version: %q", arch.ArchVariant))
}
diff --git a/cc/config/cfi_exports.map b/cc/config/cfi_exports.map
new file mode 100644
index 0000000..3d8f3e0
--- /dev/null
+++ b/cc/config/cfi_exports.map
@@ -0,0 +1,4 @@
+{
+ global:
+ __cfi_check;
+};
diff --git a/cc/config/clang.go b/cc/config/clang.go
index 977afe1..302c68e 100644
--- a/cc/config/clang.go
+++ b/cc/config/clang.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package config
import (
@@ -63,10 +77,13 @@
"-fno-inline-functions-called-once",
"-mfpmath=sse",
"-mbionic",
+
+ // windows
+ "--enable-stdcall-fixup",
})
var ClangLibToolingUnknownCflags = []string{
- "-flto",
+ "-flto*",
"-fsanitize*",
}
@@ -97,6 +114,9 @@
// http://b/29823425 Disable -Wexpansion-to-defined for Clang update to r271374
"-Wno-expansion-to-defined",
+ // http://b/68236239 Allow 0/NULL instead of using nullptr everywhere.
+ "-Wno-zero-as-null-pointer-constant",
+
// http://b/36463318 Clang executes with an absolute path, so clang-provided
// headers are now absolute.
"-fdebug-prefix-map=$$PWD/=",
diff --git a/cc/config/global.go b/cc/config/global.go
index f4a1230..6f27822 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -16,6 +16,7 @@
import (
"fmt"
+ "runtime"
"strings"
"android/soong/android"
@@ -33,9 +34,20 @@
"-Winit-self",
"-Wpointer-arith",
- // COMMON_RELEASE_CFLAGS
+ // Make paths in deps files relative
+ "-no-canonical-prefixes",
+ "-fno-canonical-system-headers",
+
"-DNDEBUG",
"-UDEBUG",
+
+ "-fno-exceptions",
+ "-Wno-multichar",
+
+ "-O2",
+ "-g",
+
+ "-fno-strict-aliasing",
}
commonGlobalConlyflags = []string{}
@@ -43,16 +55,44 @@
deviceGlobalCflags = []string{
"-fdiagnostics-color",
- // TARGET_ERROR_FLAGS
+ "-ffunction-sections",
+ "-fdata-sections",
+ "-fno-short-enums",
+ "-funwind-tables",
+ "-fstack-protector-strong",
+ "-Wa,--noexecstack",
+ "-D_FORTIFY_SOURCE=2",
+
+ "-Wstrict-aliasing=2",
+
"-Werror=return-type",
"-Werror=non-virtual-dtor",
"-Werror=address",
"-Werror=sequence-point",
"-Werror=date-time",
+ "-Werror=format-security",
+ }
+
+ deviceGlobalCppflags = []string{
+ "-fvisibility-inlines-hidden",
+ }
+
+ deviceGlobalLdflags = []string{
+ "-Wl,-z,noexecstack",
+ "-Wl,-z,relro",
+ "-Wl,-z,now",
+ "-Wl,--build-id=md5",
+ "-Wl,--warn-shared-textrel",
+ "-Wl,--fatal-warnings",
+ "-Wl,--no-undefined-version",
}
hostGlobalCflags = []string{}
+ hostGlobalCppflags = []string{}
+
+ hostGlobalLdflags = []string{}
+
commonGlobalCppflags = []string{
"-Wsign-promo",
}
@@ -76,8 +116,28 @@
// prebuilts/clang default settings.
ClangDefaultBase = "prebuilts/clang/host"
- ClangDefaultVersion = "clang-4053586"
- ClangDefaultShortVersion = "5.0"
+ ClangDefaultVersion = "clang-4393122"
+ ClangDefaultShortVersion = "5.0.1"
+
+ WarningAllowedProjects = []string{
+ "external/libese/third_party/NXPNFC_P61_JCOP_Kit/",
+ "external/skia/",
+ "device/",
+ "frameworks/av/media/libeffects/factory/",
+ "frameworks/av/media/libstagefright/codecs/",
+ "frameworks/native/libs/vr/libbufferhub/",
+ "vendor/",
+ }
+
+ // Some Android.mk files still have warnings.
+ WarningAllowedOldProjects = []string{
+ "frameworks/av/drm/mediacas/plugins/",
+ "frameworks/av/services/mediaextractor/",
+ "frameworks/webview/chromium/",
+ "hardware/libhardware/modules/",
+ "hardware/qcom/",
+ "tools/adt/idea/android/ultimate/get_modification_time/jni/",
+ }
)
var pctx = android.NewPackageContext("android/soong/cc/config")
@@ -90,7 +150,11 @@
pctx.StaticVariable("CommonGlobalCflags", strings.Join(commonGlobalCflags, " "))
pctx.StaticVariable("CommonGlobalConlyflags", strings.Join(commonGlobalConlyflags, " "))
pctx.StaticVariable("DeviceGlobalCflags", strings.Join(deviceGlobalCflags, " "))
+ pctx.StaticVariable("DeviceGlobalCppflags", strings.Join(deviceGlobalCppflags, " "))
+ pctx.StaticVariable("DeviceGlobalLdflags", strings.Join(deviceGlobalLdflags, " "))
pctx.StaticVariable("HostGlobalCflags", strings.Join(hostGlobalCflags, " "))
+ pctx.StaticVariable("HostGlobalCppflags", strings.Join(hostGlobalCppflags, " "))
+ pctx.StaticVariable("HostGlobalLdflags", strings.Join(hostGlobalLdflags, " "))
pctx.StaticVariable("NoOverrideGlobalCflags", strings.Join(noOverrideGlobalCflags, " "))
pctx.StaticVariable("CommonGlobalCppflags", strings.Join(commonGlobalCppflags, " "))
@@ -127,14 +191,14 @@
[]string{"libnativehelper/include_deprecated"})
pctx.SourcePathVariable("ClangDefaultBase", ClangDefaultBase)
- pctx.VariableFunc("ClangBase", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("LLVM_PREBUILTS_BASE"); override != "" {
+ pctx.VariableFunc("ClangBase", func(config android.Config) (string, error) {
+ if override := config.Getenv("LLVM_PREBUILTS_BASE"); override != "" {
return override, nil
}
return "${ClangDefaultBase}", nil
})
- pctx.VariableFunc("ClangVersion", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("LLVM_PREBUILTS_VERSION"); override != "" {
+ pctx.VariableFunc("ClangVersion", func(config android.Config) (string, error) {
+ if override := config.Getenv("LLVM_PREBUILTS_VERSION"); override != "" {
return override, nil
}
return ClangDefaultVersion, nil
@@ -142,13 +206,18 @@
pctx.StaticVariable("ClangPath", "${ClangBase}/${HostPrebuiltTag}/${ClangVersion}")
pctx.StaticVariable("ClangBin", "${ClangPath}/bin")
- pctx.VariableFunc("ClangShortVersion", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("LLVM_RELEASE_VERSION"); override != "" {
+ pctx.VariableFunc("ClangShortVersion", func(config android.Config) (string, error) {
+ if override := config.Getenv("LLVM_RELEASE_VERSION"); override != "" {
return override, nil
}
return ClangDefaultShortVersion, nil
})
pctx.StaticVariable("ClangAsanLibDir", "${ClangPath}/lib64/clang/${ClangShortVersion}/lib/linux")
+ if runtime.GOOS == "darwin" {
+ pctx.StaticVariable("LLVMGoldPlugin", "${ClangPath}/lib64/LLVMgold.dylib")
+ } else {
+ pctx.StaticVariable("LLVMGoldPlugin", "${ClangPath}/lib64/LLVMgold.so")
+ }
// These are tied to the version of LLVM directly in external/llvm, so they might trail the host prebuilts
// being used for the rest of the build process.
@@ -164,8 +233,8 @@
"frameworks/rs/script_api/include",
})
- pctx.VariableFunc("CcWrapper", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("CC_WRAPPER"); override != "" {
+ pctx.VariableFunc("CcWrapper", func(config android.Config) (string, error) {
+ if override := config.Getenv("CC_WRAPPER"); override != "" {
return override + " ", nil
}
return "", nil
@@ -174,9 +243,8 @@
var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS)
-func bionicHeaders(bionicArch, kernelArch string) string {
+func bionicHeaders(kernelArch string) string {
return strings.Join([]string{
- "-isystem bionic/libc/arch-" + bionicArch + "/include",
"-isystem bionic/libc/include",
"-isystem bionic/libc/kernel/uapi",
"-isystem bionic/libc/kernel/uapi/asm-" + kernelArch,
@@ -185,17 +253,6 @@
}, " ")
}
-func VndkLibraries() []string {
- return []string{}
-}
-
-// This needs to be kept up to date with the list in system/core/rootdir/etc/ld.config.txt:
-// [vendor]
-// namespace.default.link.system.shared_libs
-func LLndkLibraries() []string {
- return []string{"libc", "libm", "libdl", "liblog", "ld-android"}
-}
-
func replaceFirst(slice []string, from, to string) {
if slice[0] != from {
panic(fmt.Errorf("Expected %q, found %q", from, to))
diff --git a/cc/config/mips64_device.go b/cc/config/mips64_device.go
index 3a49e7b..a6191b6 100644
--- a/cc/config/mips64_device.go
+++ b/cc/config/mips64_device.go
@@ -22,56 +22,20 @@
var (
mips64Cflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
- "-O2",
- "-fomit-frame-pointer",
- "-fno-strict-aliasing",
- "-funswitch-loops",
- "-U__unix",
- "-U__unix__",
"-Umips",
- "-ffunction-sections",
- "-fdata-sections",
- "-funwind-tables",
- "-fstack-protector-strong",
- "-Wa,--noexecstack",
- "-Werror=format-security",
- "-D_FORTIFY_SOURCE=2",
- "-no-canonical-prefixes",
- "-fno-canonical-system-headers",
// Help catch common 32/64-bit errors.
- "-Werror=pointer-to-int-cast",
- "-Werror=int-to-pointer-cast",
"-Werror=implicit-function-declaration",
-
- // TARGET_RELEASE_CFLAGS
- "-DNDEBUG",
- "-g",
- "-Wstrict-aliasing=2",
- "-fgcse-after-reload",
- "-frerun-cse-after-loop",
- "-frename-registers",
}
mips64ClangCflags = append(mips64Cflags, []string{
"-fintegrated-as",
}...)
- mips64Cppflags = []string{
- "-fvisibility-inlines-hidden",
- }
+ mips64Cppflags = []string{}
mips64Ldflags = []string{
- "-Wl,-z,noexecstack",
- "-Wl,-z,relro",
- "-Wl,-z,now",
- "-Wl,--build-id=md5",
- "-Wl,--warn-shared-textrel",
- "-Wl,--fatal-warnings",
"-Wl,--allow-shlib-undefined",
- "-Wl,--no-undefined-version",
}
mips64ArchVariantCflags = map[string][]string{
@@ -108,7 +72,7 @@
pctx.StaticVariable("Mips64Cflags", strings.Join(mips64Cflags, " "))
pctx.StaticVariable("Mips64Ldflags", strings.Join(mips64Ldflags, " "))
pctx.StaticVariable("Mips64Cppflags", strings.Join(mips64Cppflags, " "))
- pctx.StaticVariable("Mips64IncludeFlags", bionicHeaders("mips64", "mips"))
+ pctx.StaticVariable("Mips64IncludeFlags", bionicHeaders("mips"))
// Clang cflags
pctx.StaticVariable("Mips64ClangCflags", strings.Join(ClangFilterUnknownCflags(mips64ClangCflags), " "))
diff --git a/cc/config/mips_device.go b/cc/config/mips_device.go
index c135029..9709ada 100644
--- a/cc/config/mips_device.go
+++ b/cc/config/mips_device.go
@@ -22,32 +22,8 @@
var (
mipsCflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
- "-O2",
"-fomit-frame-pointer",
- "-fno-strict-aliasing",
- "-funswitch-loops",
- "-U__unix",
- "-U__unix__",
"-Umips",
- "-ffunction-sections",
- "-fdata-sections",
- "-funwind-tables",
- "-fstack-protector-strong",
- "-Wa,--noexecstack",
- "-Werror=format-security",
- "-D_FORTIFY_SOURCE=2",
- "-no-canonical-prefixes",
- "-fno-canonical-system-headers",
-
- // TARGET_RELEASE_CFLAGS
- "-DNDEBUG",
- "-g",
- "-Wstrict-aliasing=2",
- "-fgcse-after-reload",
- "-frerun-cse-after-loop",
- "-frename-registers",
}
mipsClangCflags = append(mipsCflags, []string{
@@ -55,19 +31,10 @@
"-fintegrated-as",
}...)
- mipsCppflags = []string{
- "-fvisibility-inlines-hidden",
- }
+ mipsCppflags = []string{}
mipsLdflags = []string{
- "-Wl,-z,noexecstack",
- "-Wl,-z,relro",
- "-Wl,-z,now",
- "-Wl,--build-id=md5",
- "-Wl,--warn-shared-textrel",
- "-Wl,--fatal-warnings",
"-Wl,--allow-shlib-undefined",
- "-Wl,--no-undefined-version",
}
mipsToolchainLdflags = []string{
@@ -147,7 +114,7 @@
pctx.StaticVariable("MipsCflags", strings.Join(mipsCflags, " "))
pctx.StaticVariable("MipsLdflags", strings.Join(mipsLdflags, " "))
pctx.StaticVariable("MipsCppflags", strings.Join(mipsCppflags, " "))
- pctx.StaticVariable("MipsIncludeFlags", bionicHeaders("mips", "mips"))
+ pctx.StaticVariable("MipsIncludeFlags", bionicHeaders("mips"))
// Clang cflags
pctx.StaticVariable("MipsClangCflags", strings.Join(ClangFilterUnknownCflags(mipsClangCflags), " "))
diff --git a/cc/config/tidy.go b/cc/config/tidy.go
index a2fa5a2..76a5f9e 100644
--- a/cc/config/tidy.go
+++ b/cc/config/tidy.go
@@ -25,8 +25,8 @@
// Global tidy checks include only google*, performance*,
// and misc-macro-parentheses, but not google-readability*
// or google-runtime-references.
- pctx.VariableFunc("TidyDefaultGlobalChecks", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("DEFAULT_GLOBAL_TIDY_CHECKS"); override != "" {
+ pctx.VariableFunc("TidyDefaultGlobalChecks", func(config android.Config) (string, error) {
+ if override := config.Getenv("DEFAULT_GLOBAL_TIDY_CHECKS"); override != "" {
return override, nil
}
return strings.Join([]string{
@@ -41,8 +41,8 @@
// There are too many clang-tidy warnings in external and vendor projects.
// Enable only some google checks for these projects.
- pctx.VariableFunc("TidyExternalVendorChecks", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("DEFAULT_EXTERNAL_VENDOR_TIDY_CHECKS"); override != "" {
+ pctx.VariableFunc("TidyExternalVendorChecks", func(config android.Config) (string, error) {
+ if override := config.Getenv("DEFAULT_EXTERNAL_VENDOR_TIDY_CHECKS"); override != "" {
return override, nil
}
return strings.Join([]string{
diff --git a/cc/config/toolchain.go b/cc/config/toolchain.go
index fc0282b..796ccfb 100644
--- a/cc/config/toolchain.go
+++ b/cc/config/toolchain.go
@@ -70,6 +70,8 @@
YasmFlags() string
+ WindresFlags() string
+
Is64Bit() bool
ShlibSuffix() string
@@ -135,6 +137,10 @@
return ""
}
+func (toolchainBase) WindresFlags() string {
+ return ""
+}
+
func (toolchainBase) SanitizerRuntimeLibraryArch() string {
return ""
}
@@ -227,6 +233,10 @@
return SanitizerRuntimeLibrary(t, "tsan")
}
+func ProfileRuntimeLibrary(t Toolchain) string {
+ return SanitizerRuntimeLibrary(t, "profile")
+}
+
func ToolPath(t Toolchain) string {
if p := t.ToolPath(); p != "" {
return p
diff --git a/cc/config/x86_64_device.go b/cc/config/x86_64_device.go
index 2a6fe2a..12f3e6f 100644
--- a/cc/config/x86_64_device.go
+++ b/cc/config/x86_64_device.go
@@ -22,47 +22,14 @@
var (
x86_64Cflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
- "-O2",
- "-Wa,--noexecstack",
- "-Werror=format-security",
- "-D_FORTIFY_SOURCE=2",
- "-Wstrict-aliasing=2",
- "-ffunction-sections",
- "-finline-functions",
- "-finline-limit=300",
- "-fno-short-enums",
- "-fstrict-aliasing",
- "-funswitch-loops",
- "-funwind-tables",
- "-fstack-protector-strong",
- "-no-canonical-prefixes",
- "-fno-canonical-system-headers",
-
// Help catch common 32/64-bit errors.
- "-Werror=pointer-to-int-cast",
- "-Werror=int-to-pointer-cast",
"-Werror=implicit-function-declaration",
-
- // TARGET_RELEASE_CFLAGS from build/core/combo/select.mk
- "-O2",
- "-g",
- "-fno-strict-aliasing",
}
x86_64Cppflags = []string{}
x86_64Ldflags = []string{
- "-Wl,-z,noexecstack",
- "-Wl,-z,relro",
- "-Wl,-z,now",
- "-Wl,--build-id=md5",
- "-Wl,--warn-shared-textrel",
- "-Wl,--fatal-warnings",
- "-Wl,--gc-sections",
"-Wl,--hash-style=gnu",
- "-Wl,--no-undefined-version",
}
x86_64ArchVariantCflags = map[string][]string{
@@ -159,7 +126,7 @@
pctx.StaticVariable("X86_64Cflags", strings.Join(x86_64Cflags, " "))
pctx.StaticVariable("X86_64Ldflags", strings.Join(x86_64Ldflags, " "))
pctx.StaticVariable("X86_64Cppflags", strings.Join(x86_64Cppflags, " "))
- pctx.StaticVariable("X86_64IncludeFlags", bionicHeaders("x86_64", "x86"))
+ pctx.StaticVariable("X86_64IncludeFlags", bionicHeaders("x86"))
// Clang cflags
pctx.StaticVariable("X86_64ClangCflags", strings.Join(ClangFilterUnknownCflags(x86_64Cflags), " "))
diff --git a/cc/config/x86_darwin_host.go b/cc/config/x86_darwin_host.go
index 65fa1ed..dbaa6fa 100644
--- a/cc/config/x86_darwin_host.go
+++ b/cc/config/x86_darwin_host.go
@@ -25,9 +25,6 @@
var (
darwinCflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
-
"-fdiagnostics-color",
"-fPIC",
@@ -38,10 +35,6 @@
"-D__STDC_FORMAT_MACROS",
"-D__STDC_CONSTANT_MACROS",
- // HOST_RELEASE_CFLAGS
- "-O2", // from build/core/combo/select.mk
- "-g", // from build/core/combo/select.mk
- "-fno-strict-aliasing", // from build/core/combo/select.mk
"-isysroot ${macSdkRoot}",
"-mmacosx-version-min=${macMinVersion}",
"-DMACOSX_DEPLOYMENT_TARGET=${macMinVersion}",
@@ -89,6 +82,7 @@
"10.10",
"10.11",
"10.12",
+ "10.13",
}
darwinAvailableLibraries = append(
@@ -99,7 +93,6 @@
"ncurses",
"objc",
"pthread",
- "z",
}, "-l"),
"-framework AppKit",
"-framework CoreFoundation",
@@ -114,25 +107,25 @@
)
func init() {
- pctx.VariableFunc("macSdkPath", func(config interface{}) (string, error) {
- xcodeselect := config.(android.Config).HostSystemTool("xcode-select")
+ pctx.VariableFunc("macSdkPath", func(config android.Config) (string, error) {
+ xcodeselect := config.HostSystemTool("xcode-select")
bytes, err := exec.Command(xcodeselect, "--print-path").Output()
return strings.TrimSpace(string(bytes)), err
})
- pctx.VariableFunc("macSdkRoot", func(config interface{}) (string, error) {
- return xcrunSdk(config.(android.Config), "--show-sdk-path")
+ pctx.VariableFunc("macSdkRoot", func(config android.Config) (string, error) {
+ return xcrunSdk(config, "--show-sdk-path")
})
pctx.StaticVariable("macMinVersion", "10.8")
- pctx.VariableFunc("MacArPath", func(config interface{}) (string, error) {
- return xcrun(config.(android.Config), "--find", "ar")
+ pctx.VariableFunc("MacArPath", func(config android.Config) (string, error) {
+ return xcrun(config, "--find", "ar")
})
- pctx.VariableFunc("MacStripPath", func(config interface{}) (string, error) {
- return xcrun(config.(android.Config), "--find", "strip")
+ pctx.VariableFunc("MacStripPath", func(config android.Config) (string, error) {
+ return xcrun(config, "--find", "strip")
})
- pctx.VariableFunc("MacToolPath", func(config interface{}) (string, error) {
- path, err := xcrun(config.(android.Config), "--find", "ld")
+ pctx.VariableFunc("MacToolPath", func(config android.Config) (string, error) {
+ path, err := xcrun(config, "--find", "ld")
return filepath.Dir(path), err
})
@@ -159,6 +152,8 @@
strings.Join(ClangFilterUnknownCflags(darwinX8664Cflags), " "))
pctx.StaticVariable("DarwinX86ClangLdflags", strings.Join(darwinX86ClangLdflags, " "))
pctx.StaticVariable("DarwinX8664ClangLdflags", strings.Join(darwinX8664ClangLdflags, " "))
+ pctx.StaticVariable("DarwinX86YasmFlags", "-f macho -m x86")
+ pctx.StaticVariable("DarwinX8664YasmFlags", "-f macho -m amd64")
}
func xcrun(config android.Config, args ...string) (string, error) {
@@ -276,6 +271,14 @@
return "${config.DarwinClangLdflags} ${config.DarwinX8664ClangLdflags}"
}
+func (t *toolchainDarwinX86) YasmFlags() string {
+ return "${config.DarwinX86YasmFlags}"
+}
+
+func (t *toolchainDarwinX8664) YasmFlags() string {
+ return "${config.DarwinX8664YasmFlags}"
+}
+
func (t *toolchainDarwin) ShlibSuffix() string {
return ".dylib"
}
diff --git a/cc/config/x86_device.go b/cc/config/x86_device.go
index 23518b6..b9ce4af 100644
--- a/cc/config/x86_device.go
+++ b/cc/config/x86_device.go
@@ -21,30 +21,7 @@
)
var (
- x86Cflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
- "-O2",
- "-Wa,--noexecstack",
- "-Werror=format-security",
- "-D_FORTIFY_SOURCE=2",
- "-Wstrict-aliasing=2",
- "-ffunction-sections",
- "-finline-functions",
- "-finline-limit=300",
- "-fno-short-enums",
- "-fstrict-aliasing",
- "-funswitch-loops",
- "-funwind-tables",
- "-fstack-protector-strong",
- "-no-canonical-prefixes",
- "-fno-canonical-system-headers",
-
- // TARGET_RELEASE_CFLAGS from build/core/combo/select.mk
- "-O2",
- "-g",
- "-fno-strict-aliasing",
- }
+ x86Cflags = []string{}
x86ClangCflags = append(x86Cflags, []string{
"-msse3",
@@ -58,15 +35,7 @@
x86Cppflags = []string{}
x86Ldflags = []string{
- "-Wl,-z,noexecstack",
- "-Wl,-z,relro",
- "-Wl,-z,now",
- "-Wl,--build-id=md5",
- "-Wl,--warn-shared-textrel",
- "-Wl,--fatal-warnings",
- "-Wl,--gc-sections",
"-Wl,--hash-style=gnu",
- "-Wl,--no-undefined-version",
}
x86ArchVariantCflags = map[string][]string{
@@ -181,7 +150,7 @@
pctx.StaticVariable("X86Cflags", strings.Join(x86Cflags, " "))
pctx.StaticVariable("X86Ldflags", strings.Join(x86Ldflags, " "))
pctx.StaticVariable("X86Cppflags", strings.Join(x86Cppflags, " "))
- pctx.StaticVariable("X86IncludeFlags", bionicHeaders("x86", "x86"))
+ pctx.StaticVariable("X86IncludeFlags", bionicHeaders("x86"))
// Clang cflags
pctx.StaticVariable("X86ClangCflags", strings.Join(ClangFilterUnknownCflags(x86ClangCflags), " "))
diff --git a/cc/config/x86_linux_bionic_host.go b/cc/config/x86_linux_bionic_host.go
index bd6cd0e..9f0bbbd 100644
--- a/cc/config/x86_linux_bionic_host.go
+++ b/cc/config/x86_linux_bionic_host.go
@@ -22,15 +22,11 @@
var (
linuxBionicCflags = ClangFilterUnknownCflags([]string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
-
"-fdiagnostics-color",
"-Wa,--noexecstack",
"-fPIC",
- "-no-canonical-prefixes",
"-U_FORTIFY_SOURCE",
"-D_FORTIFY_SOURCE=2",
@@ -43,21 +39,11 @@
"-fno-short-enums",
"-funswitch-loops",
"-funwind-tables",
- "-no-canonical-prefixes",
"-fno-canonical-system-headers",
- // HOST_RELEASE_CFLAGS
- "-O2", // from build/core/combo/select.mk
- "-g", // from build/core/combo/select.mk
- "-fno-strict-aliasing", // from build/core/combo/select.mk
-
// Tell clang where the gcc toolchain is
"--gcc-toolchain=${LinuxBionicGccRoot}",
- // TODO: We're not really android, but we don't have a triple yet b/31393676
- "-U__ANDROID__",
- "-fno-emulated-tls",
-
// This is normally in ClangExtraTargetCflags, but this is considered host
"-nostdlibinc",
})
@@ -69,7 +55,6 @@
"-Wl,--build-id=md5",
"-Wl,--warn-shared-textrel",
"-Wl,--fatal-warnings",
- "-Wl,--gc-sections",
"-Wl,--hash-style=gnu",
"-Wl,--no-undefined-version",
@@ -82,7 +67,7 @@
pctx.StaticVariable("LinuxBionicCflags", strings.Join(linuxBionicCflags, " "))
pctx.StaticVariable("LinuxBionicLdflags", strings.Join(linuxBionicLdflags, " "))
- pctx.StaticVariable("LinuxBionicIncludeFlags", bionicHeaders("x86_64", "x86"))
+ pctx.StaticVariable("LinuxBionicIncludeFlags", bionicHeaders("x86"))
// Use the device gcc toolchain for now
pctx.StaticVariable("LinuxBionicGccRoot", "${X86_64GccRoot}")
@@ -142,7 +127,9 @@
}
func (t *toolchainLinuxBionic) ToolchainClangCflags() string {
- return "-m64 -march=x86-64"
+ return "-m64 -march=x86-64" +
+ // TODO: We're not really android, but we don't have a triple yet b/31393676
+ " -U__ANDROID__ -fno-emulated-tls"
}
func (t *toolchainLinuxBionic) ToolchainClangLdflags() string {
diff --git a/cc/config/x86_linux_host.go b/cc/config/x86_linux_host.go
index 80e9289..4f05068 100644
--- a/cc/config/x86_linux_host.go
+++ b/cc/config/x86_linux_host.go
@@ -22,15 +22,11 @@
var (
linuxCflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
-
"-fdiagnostics-color",
"-Wa,--noexecstack",
"-fPIC",
- "-no-canonical-prefixes",
"-U_FORTIFY_SOURCE",
"-D_FORTIFY_SOURCE=2",
@@ -40,11 +36,6 @@
//See bug 12708004.
"-D__STDC_FORMAT_MACROS",
"-D__STDC_CONSTANT_MACROS",
-
- // HOST_RELEASE_CFLAGS
- "-O2", // from build/core/combo/select.mk
- "-g", // from build/core/combo/select.mk
- "-fno-strict-aliasing", // from build/core/combo/select.mk
}
linuxLdflags = []string{
@@ -123,7 +114,6 @@
"resolv",
"rt",
"util",
- "z",
}, "-l")
)
@@ -160,6 +150,9 @@
pctx.StaticVariable("LinuxX8664ClangLdflags", strings.Join(linuxX8664ClangLdflags, " "))
pctx.StaticVariable("LinuxX86ClangCppflags", strings.Join(linuxX86ClangCppflags, " "))
pctx.StaticVariable("LinuxX8664ClangCppflags", strings.Join(linuxX8664ClangCppflags, " "))
+ // Yasm flags
+ pctx.StaticVariable("LinuxX86YasmFlags", "-f elf32 -m x86")
+ pctx.StaticVariable("LinuxX8664YasmFlags", "-f elf64 -m amd64")
}
type toolchainLinux struct {
@@ -252,6 +245,14 @@
return "${config.LinuxClangLdflags} ${config.LinuxX8664ClangLdflags}"
}
+func (t *toolchainLinuxX86) YasmFlags() string {
+ return "${config.LinuxX86YasmFlags}"
+}
+
+func (t *toolchainLinuxX8664) YasmFlags() string {
+ return "${config.LinuxX8664YasmFlags}"
+}
+
func (t *toolchainLinux) AvailableLibraries() []string {
return linuxAvailableLibraries
}
diff --git a/cc/config/x86_windows_host.go b/cc/config/x86_windows_host.go
index 4709823..016e711 100644
--- a/cc/config/x86_windows_host.go
+++ b/cc/config/x86_windows_host.go
@@ -22,9 +22,6 @@
var (
windowsCflags = []string{
- "-fno-exceptions", // from build/core/combo/select.mk
- "-Wno-multichar", // from build/core/combo/select.mk
-
"-DUSE_MINGW",
"-DWIN32_LEAN_AND_MEAN",
"-Wno-unused-parameter",
@@ -43,21 +40,31 @@
"-D_FILE_OFFSET_BITS=64",
"--sysroot ${WindowsGccRoot}/${WindowsGccTriple}",
-
- // HOST_RELEASE_CFLAGS
- "-O2", // from build/core/combo/select.mk
- "-g", // from build/core/combo/select.mk
- "-fno-strict-aliasing", // from build/core/combo/select.mk
}
+ windowsClangCflags = append(ClangFilterUnknownCflags(windowsCflags), []string{}...)
windowsIncludeFlags = []string{
"-isystem ${WindowsGccRoot}/${WindowsGccTriple}/include",
"-isystem ${WindowsGccRoot}/lib/gcc/${WindowsGccTriple}/4.8.3/include",
}
+ windowsClangCppflags = []string{
+ "-isystem ${WindowsGccRoot}/${WindowsGccTriple}/include/c++/4.8.3",
+ "-isystem ${WindowsGccRoot}/${WindowsGccTriple}/include/c++/4.8.3/backward",
+ }
+
+ windowsX86ClangCppflags = []string{
+ "-isystem ${WindowsGccRoot}/${WindowsGccTriple}/include/c++/4.8.3/${WindowsGccTriple}/32",
+ }
+
+ windowsX8664ClangCppflags = []string{
+ "-isystem ${WindowsGccRoot}/${WindowsGccTriple}/include/c++/4.8.3/${WindowsGccTriple}",
+ }
+
windowsLdflags = []string{
"--enable-stdcall-fixup",
}
+ windowsClangLdflags = append(ClangFilterUnknownCflags(windowsLdflags), []string{}...)
windowsX86Cflags = []string{
"-m32",
@@ -72,16 +79,29 @@
"-Wl,--large-address-aware",
"-L${WindowsGccRoot}/${WindowsGccTriple}/lib32",
}
+ windowsX86ClangLdflags = append(ClangFilterUnknownCflags(windowsX86Ldflags), []string{
+ "-B${WindowsGccRoot}/lib/gcc/${WindowsGccTriple}/4.8.3/32",
+ "-L${WindowsGccRoot}/lib/gcc/${WindowsGccTriple}/4.8.3/32",
+ "-B${WindowsGccRoot}/${WindowsGccTriple}/lib32",
+ }...)
windowsX8664Ldflags = []string{
"-m64",
"-L${WindowsGccRoot}/${WindowsGccTriple}/lib64",
}
+ windowsX8664ClangLdflags = append(ClangFilterUnknownCflags(windowsX8664Ldflags), []string{
+ "-B${WindowsGccRoot}/lib/gcc/${WindowsGccTriple}/4.8.3",
+ "-L${WindowsGccRoot}/lib/gcc/${WindowsGccTriple}/4.8.3",
+ "-B${WindowsGccRoot}/${WindowsGccTriple}/lib64",
+ }...)
windowsAvailableLibraries = addPrefix([]string{
"gdi32",
"imagehlp",
+ "iphlpapi",
+ "netapi32",
"ole32",
+ "powrprof",
"psapi",
"pthread",
"userenv",
@@ -106,11 +126,24 @@
pctx.StaticVariable("WindowsCflags", strings.Join(windowsCflags, " "))
pctx.StaticVariable("WindowsLdflags", strings.Join(windowsLdflags, " "))
+ pctx.StaticVariable("WindowsClangCflags", strings.Join(windowsClangCflags, " "))
+ pctx.StaticVariable("WindowsClangLdflags", strings.Join(windowsClangLdflags, " "))
+ pctx.StaticVariable("WindowsClangCppflags", strings.Join(windowsClangCppflags, " "))
+
pctx.StaticVariable("WindowsX86Cflags", strings.Join(windowsX86Cflags, " "))
pctx.StaticVariable("WindowsX8664Cflags", strings.Join(windowsX8664Cflags, " "))
pctx.StaticVariable("WindowsX86Ldflags", strings.Join(windowsX86Ldflags, " "))
pctx.StaticVariable("WindowsX8664Ldflags", strings.Join(windowsX8664Ldflags, " "))
+ pctx.StaticVariable("WindowsX86ClangCflags",
+ strings.Join(ClangFilterUnknownCflags(windowsX86Cflags), " "))
+ pctx.StaticVariable("WindowsX8664ClangCflags",
+ strings.Join(ClangFilterUnknownCflags(windowsX8664Cflags), " "))
+ pctx.StaticVariable("WindowsX86ClangLdflags", strings.Join(windowsX86ClangLdflags, " "))
+ pctx.StaticVariable("WindowsX8664ClangLdflags", strings.Join(windowsX8664ClangLdflags, " "))
+ pctx.StaticVariable("WindowsX86ClangCppflags", strings.Join(windowsX86ClangCppflags, " "))
+ pctx.StaticVariable("WindowsX8664ClangCppflags", strings.Join(windowsX8664ClangCppflags, " "))
+
pctx.StaticVariable("WindowsIncludeFlags", strings.Join(windowsIncludeFlags, " "))
}
@@ -172,24 +205,48 @@
return "${config.WindowsIncludeFlags}"
}
+func (t *toolchainWindowsX86) WindresFlags() string {
+ return "-F pe-i386"
+}
+
+func (t *toolchainWindowsX8664) WindresFlags() string {
+ return "-F pe-x86-64"
+}
+
func (t *toolchainWindows) ClangSupported() bool {
return false
}
-func (t *toolchainWindows) ClangTriple() string {
- panic("Clang is not supported under mingw")
+func (t *toolchainWindowsX86) ClangTriple() string {
+ return "i686-windows-gnu"
}
-func (t *toolchainWindows) ClangCflags() string {
- panic("Clang is not supported under mingw")
+func (t *toolchainWindowsX8664) ClangTriple() string {
+ return "x86_64-pc-windows-gnu"
}
-func (t *toolchainWindows) ClangCppflags() string {
- panic("Clang is not supported under mingw")
+func (t *toolchainWindowsX86) ClangCflags() string {
+ return "${config.WindowsClangCflags} ${config.WindowsX86ClangCflags}"
}
-func (t *toolchainWindows) ClangLdflags() string {
- panic("Clang is not supported under mingw")
+func (t *toolchainWindowsX8664) ClangCflags() string {
+ return "${config.WindowsClangCflags} ${config.WindowsX8664ClangCflags}"
+}
+
+func (t *toolchainWindowsX86) ClangCppflags() string {
+ return "${config.WindowsClangCppflags} ${config.WindowsX86ClangCppflags}"
+}
+
+func (t *toolchainWindowsX8664) ClangCppflags() string {
+ return "${config.WindowsClangCppflags} ${config.WindowsX8664ClangCppflags}"
+}
+
+func (t *toolchainWindowsX86) ClangLdflags() string {
+ return "${config.WindowsClangLdflags} ${config.WindowsX86ClangLdflags}"
+}
+
+func (t *toolchainWindowsX8664) ClangLdflags() string {
+ return "${config.WindowsClangLdflags} ${config.WindowsX8664ClangLdflags}"
}
func (t *toolchainWindows) ShlibSuffix() string {
diff --git a/cc/coverage.go b/cc/coverage.go
index 0b4188f..d2eede2 100644
--- a/cc/coverage.go
+++ b/cc/coverage.go
@@ -16,8 +16,6 @@
import (
"android/soong/android"
-
- "github.com/google/blueprint"
)
type CoverageProperties struct {
@@ -61,7 +59,7 @@
// For static libraries, the only thing that changes our object files
// are included whole static libraries, so check to see if any of
// those have coverage enabled.
- ctx.VisitDirectDeps(func(m blueprint.Module) {
+ ctx.VisitDirectDeps(func(m android.Module) {
if ctx.OtherModuleDependencyTag(m) != wholeStaticDepTag {
return
}
@@ -75,7 +73,7 @@
} else {
// For executables and shared libraries, we need to check all of
// our static dependencies.
- ctx.VisitDirectDeps(func(m blueprint.Module) {
+ ctx.VisitDirectDeps(func(m android.Module) {
cc, ok := m.(*Module)
if !ok || cc.coverage == nil {
return
diff --git a/cc/gen.go b/cc/gen.go
index 7a22abd..15b37b5 100644
--- a/cc/gen.go
+++ b/cc/gen.go
@@ -54,12 +54,19 @@
Deps: blueprint.DepsGCC,
},
"aidlFlags", "outDir")
+
+ windmc = pctx.AndroidStaticRule("windmc",
+ blueprint.RuleParams{
+ Command: "$windmcCmd -r$$(dirname $out) -h$$(dirname $out) $in",
+ CommandDeps: []string{"$windmcCmd"},
+ },
+ "windmcCmd")
)
func genYacc(ctx android.ModuleContext, yaccFile android.Path, outFile android.ModuleGenPath, yaccFlags string) (headerFile android.ModuleGenPath) {
headerFile = android.GenPathWithExt(ctx, "yacc", yaccFile, "h")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: yacc,
Description: "yacc " + yaccFile.Rel(),
Output: outFile,
@@ -76,7 +83,7 @@
func genAidl(ctx android.ModuleContext, aidlFile android.Path, outFile android.ModuleGenPath, aidlFlags string) android.Paths {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: aidl,
Description: "aidl " + aidlFile.Rel(),
Output: outFile,
@@ -92,7 +99,7 @@
}
func genLex(ctx android.ModuleContext, lexFile android.Path, outFile android.ModuleGenPath) {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: lex,
Description: "lex " + lexFile.Rel(),
Output: outFile,
@@ -100,6 +107,26 @@
})
}
+func genWinMsg(ctx android.ModuleContext, srcFile android.Path, flags builderFlags) (android.Path, android.Path) {
+ headerFile := android.GenPathWithExt(ctx, "windmc", srcFile, "h")
+ rcFile := android.GenPathWithExt(ctx, "windmc", srcFile, "rc")
+
+ windmcCmd := gccCmd(flags.toolchain, "windmc")
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: windmc,
+ Description: "windmc " + srcFile.Rel(),
+ Output: rcFile,
+ ImplicitOutput: headerFile,
+ Input: srcFile,
+ Args: map[string]string{
+ "windmcCmd": windmcCmd,
+ },
+ })
+
+ return rcFile, headerFile
+}
+
func genSources(ctx android.ModuleContext, srcFiles android.Paths,
buildFlags builderFlags) (android.Paths, android.Paths) {
@@ -126,8 +153,8 @@
srcFiles[i] = cppFile
genLex(ctx, srcFile, cppFile)
case ".proto":
- cppFile, headerFile := genProto(ctx, srcFile, buildFlags.protoFlags)
- srcFiles[i] = cppFile
+ ccFile, headerFile := genProto(ctx, srcFile, buildFlags.protoFlags)
+ srcFiles[i] = ccFile
deps = append(deps, headerFile)
case ".aidl":
cppFile := android.GenPathWithExt(ctx, "aidl", srcFile, "cpp")
@@ -137,6 +164,10 @@
cppFile := rsGeneratedCppFile(ctx, srcFile)
rsFiles = append(rsFiles, srcFiles[i])
srcFiles[i] = cppFile
+ case ".mc":
+ rcFile, headerFile := genWinMsg(ctx, srcFile, buildFlags)
+ srcFiles[i] = rcFile
+ deps = append(deps, headerFile)
}
}
diff --git a/cc/gen_stub_libs.py b/cc/gen_stub_libs.py
index bed718c..abb39c2 100755
--- a/cc/gen_stub_libs.py
+++ b/cc/gen_stub_libs.py
@@ -347,10 +347,16 @@
if section_versioned and emit_version:
self.version_script.write(' ' + symbol.name + ';\n')
+ weak = ''
+ if 'weak' in symbol.tags:
+ weak = '__attribute__((weak)) '
+
if 'var' in symbol.tags:
- self.src_file.write('int {} = 0;\n'.format(symbol.name))
+ self.src_file.write('{}int {} = 0;\n'.format(
+ weak, symbol.name))
else:
- self.src_file.write('void {}() {{}}\n'.format(symbol.name))
+ self.src_file.write('{}void {}() {{}}\n'.format(
+ weak, symbol.name))
if not version_empty and section_versioned:
base = '' if version.base is None else ' ' + version.base
diff --git a/cc/gen_test.go b/cc/gen_test.go
new file mode 100644
index 0000000..a0f7308
--- /dev/null
+++ b/cc/gen_test.go
@@ -0,0 +1,63 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "testing"
+)
+
+func TestGen(t *testing.T) {
+ t.Run("simple", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library_shared {
+ name: "libfoo",
+ srcs: [
+ "foo.c",
+ "b.aidl",
+ ],
+ }`)
+
+ aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("aidl")
+ libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Module().(*Module)
+
+ if !inList("-I"+aidl.Args["outDir"], libfoo.flags.GlobalFlags) {
+ t.Errorf("missing aidl includes in global flags")
+ }
+ })
+
+ t.Run("filegroup", func(t *testing.T) {
+ ctx := testCc(t, `
+ filegroup {
+ name: "fg",
+ srcs: ["b.aidl"],
+ }
+
+ cc_library_shared {
+ name: "libfoo",
+ srcs: [
+ "foo.c",
+ ":fg",
+ ],
+ }`)
+
+ aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("aidl")
+ libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Module().(*Module)
+
+ if !inList("-I"+aidl.Args["outDir"], libfoo.flags.GlobalFlags) {
+ t.Errorf("missing aidl includes in global flags")
+ }
+ })
+
+}
diff --git a/cc/genrule.go b/cc/genrule.go
new file mode 100644
index 0000000..51c0d16
--- /dev/null
+++ b/cc/genrule.go
@@ -0,0 +1,38 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "android/soong/android"
+ "android/soong/genrule"
+)
+
+func init() {
+ android.RegisterModuleType("cc_genrule", genRuleFactory)
+}
+
+// cc_genrule is a genrule that can depend on other cc_* objects.
+// The cmd may be run multiple times, once for each of the different arch/etc
+// variations.
+func genRuleFactory() android.Module {
+ module := genrule.NewGenRule()
+
+ module.Extra = &VendorProperties{}
+ module.AddProperties(module.Extra)
+
+ android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibBoth)
+
+ return module
+}
diff --git a/cc/installer.go b/cc/installer.go
index 7bedc56..33f29f2 100644
--- a/cc/installer.go
+++ b/cc/installer.go
@@ -24,7 +24,7 @@
type InstallerProperties struct {
// install to a subdirectory of the default install path for the module
- Relative_install_path string `android:"arch_variant"`
+ Relative_install_path *string `android:"arch_variant"`
}
type installLocation int
@@ -69,14 +69,15 @@
if !ctx.Host() && !ctx.Arch().Native {
dir = filepath.Join(dir, ctx.Arch().ArchType.String())
}
- if installer.location == InstallInData && ctx.vndk() {
+ if installer.location == InstallInData && ctx.useVndk() {
dir = filepath.Join(dir, "vendor")
}
- return android.PathForModuleInstall(ctx, dir, installer.subDir, installer.Properties.Relative_install_path, installer.relative)
+ return android.PathForModuleInstall(ctx, dir, installer.subDir,
+ String(installer.Properties.Relative_install_path), installer.relative)
}
func (installer *baseInstaller) install(ctx ModuleContext, file android.Path) {
- installer.path = ctx.InstallFile(installer.installDir(ctx), file)
+ installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file)
}
func (installer *baseInstaller) inData() bool {
diff --git a/cc/library.go b/cc/library.go
index 3d463bd..cf10617 100644
--- a/cc/library.go
+++ b/cc/library.go
@@ -21,7 +21,6 @@
"github.com/google/blueprint/pathtools"
"android/soong/android"
- "android/soong/cc/config"
)
type LibraryProperties struct {
@@ -58,13 +57,21 @@
Aidl struct {
// export headers generated from .aidl sources
- Export_aidl_headers bool
+ Export_aidl_headers *bool
}
Proto struct {
// export headers generated from .proto sources
- Export_proto_headers bool
+ Export_proto_headers *bool
}
+ Target struct {
+ Vendor struct {
+ // version script for this vendor variant
+ Version_script *string `android:"arch_variant"`
+ }
+ }
+
+ Static_ndk_lib *bool
}
type LibraryMutatedProperties struct {
@@ -83,7 +90,8 @@
type FlagExporterProperties struct {
// list of directories relative to the Blueprints file that will
// be added to the include path (using -I) for this module and any module that links
- // against this module
+ // against this module. Directories listed in export_include_dirs do not need to be
+ // listed in local_include_dirs.
Export_include_dirs []string `android:"arch_variant"`
Target struct {
@@ -98,51 +106,51 @@
}
func init() {
- android.RegisterModuleType("cc_library_static", libraryStaticFactory)
- android.RegisterModuleType("cc_library_shared", librarySharedFactory)
- android.RegisterModuleType("cc_library", libraryFactory)
- android.RegisterModuleType("cc_library_host_static", libraryHostStaticFactory)
- android.RegisterModuleType("cc_library_host_shared", libraryHostSharedFactory)
- android.RegisterModuleType("cc_library_headers", libraryHeaderFactory)
+ android.RegisterModuleType("cc_library_static", LibraryStaticFactory)
+ android.RegisterModuleType("cc_library_shared", LibrarySharedFactory)
+ android.RegisterModuleType("cc_library", LibraryFactory)
+ android.RegisterModuleType("cc_library_host_static", LibraryHostStaticFactory)
+ android.RegisterModuleType("cc_library_host_shared", LibraryHostSharedFactory)
+ android.RegisterModuleType("cc_library_headers", LibraryHeaderFactory)
}
// Module factory for combined static + shared libraries, device by default but with possible host
// support
-func libraryFactory() android.Module {
+func LibraryFactory() android.Module {
module, _ := NewLibrary(android.HostAndDeviceSupported)
return module.Init()
}
// Module factory for static libraries
-func libraryStaticFactory() android.Module {
+func LibraryStaticFactory() android.Module {
module, library := NewLibrary(android.HostAndDeviceSupported)
library.BuildOnlyStatic()
return module.Init()
}
// Module factory for shared libraries
-func librarySharedFactory() android.Module {
+func LibrarySharedFactory() android.Module {
module, library := NewLibrary(android.HostAndDeviceSupported)
library.BuildOnlyShared()
return module.Init()
}
// Module factory for host static libraries
-func libraryHostStaticFactory() android.Module {
+func LibraryHostStaticFactory() android.Module {
module, library := NewLibrary(android.HostSupported)
library.BuildOnlyStatic()
return module.Init()
}
// Module factory for host shared libraries
-func libraryHostSharedFactory() android.Module {
+func LibraryHostSharedFactory() android.Module {
module, library := NewLibrary(android.HostSupported)
library.BuildOnlyShared()
return module.Init()
}
// Module factory for header-only libraries
-func libraryHeaderFactory() android.Module {
+func LibraryHeaderFactory() android.Module {
module, library := NewLibrary(android.HostAndDeviceSupported)
library.HeaderOnly()
return module.Init()
@@ -156,7 +164,7 @@
}
func (f *flagExporter) exportedIncludes(ctx ModuleContext) android.Paths {
- if ctx.vndk() && f.Properties.Target.Vendor.Export_include_dirs != nil {
+ if ctx.useVndk() && f.Properties.Target.Vendor.Export_include_dirs != nil {
return android.PathsForModuleSrc(ctx, f.Properties.Target.Vendor.Export_include_dirs)
} else {
return android.PathsForModuleSrc(ctx, f.Properties.Export_include_dirs)
@@ -235,6 +243,10 @@
// Source Abi Diff
sAbiDiff android.OptionalPath
+ // Location of the static library in the sysroot. Empty if the library is
+ // not included in the NDK.
+ ndkSysrootPath android.Path
+
// Decorated interafaces
*baseCompiler
*baseLinker
@@ -306,7 +318,7 @@
return flags
}
-func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags {
+func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags {
exportIncludeDirs := library.flagExporter.exportedIncludes(ctx)
if len(exportIncludeDirs) > 0 {
f := includeDirsToFlags(exportIncludeDirs)
@@ -314,7 +326,7 @@
flags.YasmFlags = append(flags.YasmFlags, f)
}
- return library.baseCompiler.compilerFlags(ctx, flags)
+ return library.baseCompiler.compilerFlags(ctx, flags, deps)
}
func extractExportIncludesFromFlags(flags []string) []string {
@@ -324,9 +336,7 @@
// from a source. We extract the include flags exported by a library.
// This includes the flags exported which are re-exported from static
// library dependencies, exported header library dependencies and
- // generated header dependencies. Re-exported shared library include
- // flags are not in this set since shared library dependencies will
- // themselves be included in the vndk. -isystem headers are not included
+ // generated header dependencies. -isystem headers are not included
// since for bionic libraries, abi-filtering is taken care of by version
// scripts.
var exportedIncludes []string
@@ -351,8 +361,8 @@
}
return Objects{}
}
- if ctx.createVndkSourceAbiDump() {
- exportIncludeDirs := android.PathsForModuleSrc(ctx, library.flagExporter.Properties.Export_include_dirs)
+ if ctx.createVndkSourceAbiDump() || library.sabi.Properties.CreateSAbiDumps {
+ exportIncludeDirs := library.flagExporter.exportedIncludes(ctx)
var SourceAbiFlags []string
for _, dir := range exportIncludeDirs.Strings() {
SourceAbiFlags = append(SourceAbiFlags, "-I"+dir)
@@ -437,7 +447,7 @@
deps.SharedLibs = append(deps.SharedLibs, library.Properties.Static.Shared_libs...)
} else if library.shared() {
if ctx.toolchain().Bionic() && !Bool(library.baseLinker.Properties.Nocrt) {
- if !ctx.sdk() {
+ if !ctx.useSdk() {
deps.CrtBegin = "crtbegin_so"
deps.CrtEnd = "crtend_so"
} else {
@@ -457,7 +467,11 @@
deps.StaticLibs = append(deps.StaticLibs, library.Properties.Shared.Static_libs...)
deps.SharedLibs = append(deps.SharedLibs, library.Properties.Shared.Shared_libs...)
}
-
+ if ctx.useVndk() {
+ deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, library.baseLinker.Properties.Target.Vendor.Exclude_static_libs)
+ deps.SharedLibs = removeListFromList(deps.SharedLibs, library.baseLinker.Properties.Target.Vendor.Exclude_shared_libs)
+ deps.StaticLibs = removeListFromList(deps.StaticLibs, library.baseLinker.Properties.Target.Vendor.Exclude_static_libs)
+ }
return deps
}
@@ -487,15 +501,23 @@
flags Flags, deps PathDeps, objs Objects) android.Path {
var linkerDeps android.Paths
+ linkerDeps = append(linkerDeps, flags.LdFlagsDeps...)
versionScript := android.OptionalPathForModuleSrc(ctx, library.Properties.Version_script)
unexportedSymbols := android.OptionalPathForModuleSrc(ctx, library.Properties.Unexported_symbols_list)
forceNotWeakSymbols := android.OptionalPathForModuleSrc(ctx, library.Properties.Force_symbols_not_weak_list)
forceWeakSymbols := android.OptionalPathForModuleSrc(ctx, library.Properties.Force_symbols_weak_list)
+ if ctx.useVndk() && library.Properties.Target.Vendor.Version_script != nil {
+ versionScript = android.OptionalPathForModuleSrc(ctx, library.Properties.Target.Vendor.Version_script)
+ }
if !ctx.Darwin() {
if versionScript.Valid() {
flags.LdFlags = append(flags.LdFlags, "-Wl,--version-script,"+versionScript.String())
linkerDeps = append(linkerDeps, versionScript.Path())
+ if library.sanitize.isSanitizerEnabled(cfi) {
+ flags.LdFlags = append(flags.LdFlags, "-Wl,--version-script,"+cfiExportsMap.String())
+ linkerDeps = append(linkerDeps, cfiExportsMap)
+ }
}
if unexportedSymbols.Valid() {
ctx.PropertyErrorf("unexported_symbols_list", "Only supported on Darwin")
@@ -603,7 +625,7 @@
if versionScript.Valid() {
symbolFile = versionScript
}
- exportIncludeDirs := android.PathsForModuleSrc(ctx, library.flagExporter.Properties.Export_include_dirs)
+ exportIncludeDirs := library.flagExporter.exportedIncludes(ctx)
var SourceAbiFlags []string
for _, dir := range exportIncludeDirs.Strings() {
SourceAbiFlags = append(SourceAbiFlags, "-I"+dir)
@@ -621,7 +643,7 @@
}
func vndkVsNdk(ctx ModuleContext) bool {
- if inList(ctx.baseModuleName(), config.LLndkLibraries()) {
+ if inList(ctx.baseModuleName(), llndkLibraries) {
return false
}
return true
@@ -630,8 +652,7 @@
func (library *libraryDecorator) link(ctx ModuleContext,
flags Flags, deps PathDeps, objs Objects) android.Path {
- objs = objs.Append(deps.Objs)
-
+ objs = deps.Objs.Copy().Append(objs)
var out android.Path
if library.static() || library.header() {
out = library.linkStatic(ctx, flags, deps, objs)
@@ -643,7 +664,7 @@
library.reexportFlags(deps.ReexportedFlags)
library.reexportDeps(deps.ReexportedFlagsDeps)
- if library.Properties.Aidl.Export_aidl_headers {
+ if Bool(library.Properties.Aidl.Export_aidl_headers) {
if library.baseCompiler.hasSrcExt(".aidl") {
flags := []string{
"-I" + android.PathForModuleGen(ctx, "aidl").String(),
@@ -655,11 +676,11 @@
}
}
- if library.Properties.Proto.Export_proto_headers {
+ if Bool(library.Properties.Proto.Export_proto_headers) {
if library.baseCompiler.hasSrcExt(".proto") {
flags := []string{
- "-I" + protoSubDir(ctx).String(),
- "-I" + protoDir(ctx).String(),
+ "-I" + android.ProtoSubDir(ctx).String(),
+ "-I" + android.ProtoDir(ctx).String(),
}
library.reexportFlags(flags)
library.reuseExportedFlags = append(library.reuseExportedFlags, flags...)
@@ -700,7 +721,7 @@
func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) {
if library.shared() {
if ctx.Device() {
- if ctx.vndk() {
+ if ctx.useVndk() {
if ctx.isVndkSp() {
library.baseInstaller.subDir = "vndk-sp"
} else if ctx.isVndk() {
@@ -710,6 +731,20 @@
}
library.baseInstaller.install(ctx, file)
}
+
+ if Bool(library.Properties.Static_ndk_lib) && library.static() {
+ installPath := getNdkSysrootBase(ctx).Join(
+ ctx, "usr/lib", ctx.toolchain().ClangTriple(), file.Base())
+
+ ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ Rule: android.Cp,
+ Description: "install " + installPath.Base(),
+ Output: installPath,
+ Input: file,
+ })
+
+ library.ndkSysrootPath = installPath
+ }
}
func (library *libraryDecorator) static() bool {
diff --git a/cc/library_test.go b/cc/library_test.go
new file mode 100644
index 0000000..859b05a
--- /dev/null
+++ b/cc/library_test.go
@@ -0,0 +1,185 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestLibraryReuse(t *testing.T) {
+ t.Run("simple", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libfoo",
+ srcs: ["foo.c"],
+ }`)
+
+ libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld")
+ libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a")
+
+ if len(libfooShared.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings())
+ }
+
+ if len(libfooStatic.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings())
+ }
+
+ if libfooShared.Inputs[0] != libfooStatic.Inputs[0] {
+ t.Errorf("static object not reused for shared library")
+ }
+ })
+
+ t.Run("extra static source", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libfoo",
+ srcs: ["foo.c"],
+ static: {
+ srcs: ["bar.c"]
+ },
+ }`)
+
+ libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld")
+ libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a")
+
+ if len(libfooShared.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings())
+ }
+
+ if len(libfooStatic.Inputs) != 2 {
+ t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings())
+ }
+
+ if libfooShared.Inputs[0] != libfooStatic.Inputs[0] {
+ t.Errorf("static object not reused for shared library")
+ }
+ })
+
+ t.Run("extra shared source", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libfoo",
+ srcs: ["foo.c"],
+ shared: {
+ srcs: ["bar.c"]
+ },
+ }`)
+
+ libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld")
+ libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a")
+
+ if len(libfooShared.Inputs) != 2 {
+ t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings())
+ }
+
+ if len(libfooStatic.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings())
+ }
+
+ if libfooShared.Inputs[0] != libfooStatic.Inputs[0] {
+ t.Errorf("static object not reused for shared library")
+ }
+ })
+
+ t.Run("extra static cflags", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libfoo",
+ srcs: ["foo.c"],
+ static: {
+ cflags: ["-DFOO"],
+ },
+ }`)
+
+ libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld")
+ libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a")
+
+ if len(libfooShared.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings())
+ }
+
+ if len(libfooStatic.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings())
+ }
+
+ if libfooShared.Inputs[0] == libfooStatic.Inputs[0] {
+ t.Errorf("static object reused for shared library when it shouldn't be")
+ }
+ })
+
+ t.Run("extra shared cflags", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libfoo",
+ srcs: ["foo.c"],
+ shared: {
+ cflags: ["-DFOO"],
+ },
+ }`)
+
+ libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld")
+ libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a")
+
+ if len(libfooShared.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings())
+ }
+
+ if len(libfooStatic.Inputs) != 1 {
+ t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings())
+ }
+
+ if libfooShared.Inputs[0] == libfooStatic.Inputs[0] {
+ t.Errorf("static object reused for shared library when it shouldn't be")
+ }
+ })
+
+ t.Run("global cflags for reused generated sources", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library {
+ name: "libfoo",
+ srcs: [
+ "foo.c",
+ "a.proto",
+ ],
+ shared: {
+ srcs: [
+ "bar.c",
+ ],
+ },
+ }`)
+
+ libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld")
+ libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a")
+
+ if len(libfooShared.Inputs) != 3 {
+ t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings())
+ }
+
+ if len(libfooStatic.Inputs) != 2 {
+ t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings())
+ }
+
+ if !reflect.DeepEqual(libfooShared.Inputs[0:2].Strings(), libfooStatic.Inputs.Strings()) {
+ t.Errorf("static objects not reused for shared library")
+ }
+
+ libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Module().(*Module)
+ if !inList("-DGOOGLE_PROTOBUF_NO_RTTI", libfoo.flags.CFlags) {
+ t.Errorf("missing protobuf cflags")
+ }
+ })
+}
diff --git a/cc/linker.go b/cc/linker.go
index 1a842ae..fae5542 100644
--- a/cc/linker.go
+++ b/cc/linker.go
@@ -43,13 +43,9 @@
// list of module-specific flags that will be used for all link steps
Ldflags []string `android:"arch_variant"`
- // don't insert default compiler flags into asflags, cflags,
- // cppflags, conlyflags, ldflags, or include_dirs
- No_default_compiler_flags *bool
-
// list of system libraries that will be dynamically linked to
- // shared library and executable modules. If unset, generally defaults to libc
- // and libm. Set to [] to prevent linking against libc and libm.
+ // shared library and executable modules. If unset, generally defaults to libc,
+ // libm, and libdl. Set to [] to prevent linking against the defaults.
System_shared_libs []string
// allow the module to contain undefined symbols. By default,
@@ -93,6 +89,10 @@
// list of shared libs that should not be used to build
// the vendor variant of the C/C++ module.
Exclude_shared_libs []string
+
+ // list of static libs that should not be used to build
+ // the vendor variant of the C/C++ module.
+ Exclude_static_libs []string
}
}
}
@@ -131,15 +131,19 @@
deps.StaticLibs = append(deps.StaticLibs, linker.Properties.Static_libs...)
deps.SharedLibs = append(deps.SharedLibs, linker.Properties.Shared_libs...)
- if ctx.vndk() {
- deps.SharedLibs = removeListFromList(deps.SharedLibs, linker.Properties.Target.Vendor.Exclude_shared_libs)
- }
-
deps.ReexportHeaderLibHeaders = append(deps.ReexportHeaderLibHeaders, linker.Properties.Export_header_lib_headers...)
deps.ReexportStaticLibHeaders = append(deps.ReexportStaticLibHeaders, linker.Properties.Export_static_lib_headers...)
deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, linker.Properties.Export_shared_lib_headers...)
deps.ReexportGeneratedHeaders = append(deps.ReexportGeneratedHeaders, linker.Properties.Export_generated_headers...)
+ if ctx.useVndk() {
+ deps.SharedLibs = removeListFromList(deps.SharedLibs, linker.Properties.Target.Vendor.Exclude_shared_libs)
+ deps.ReexportSharedLibHeaders = removeListFromList(deps.ReexportSharedLibHeaders, linker.Properties.Target.Vendor.Exclude_shared_libs)
+ deps.StaticLibs = removeListFromList(deps.StaticLibs, linker.Properties.Target.Vendor.Exclude_static_libs)
+ deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Vendor.Exclude_static_libs)
+ deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Vendor.Exclude_static_libs)
+ }
+
if ctx.ModuleName() != "libcompiler_rt-extras" {
deps.LateStaticLibs = append(deps.LateStaticLibs, "libcompiler_rt-extras")
}
@@ -152,34 +156,32 @@
}
if !ctx.static() {
- // libdl should always appear after libc in dt_needed list - see below
- // the only exception is when libc is not in linker.Properties.System_shared_libs
- // such as for libc module itself
- if inList("libc", linker.Properties.System_shared_libs) {
- _, deps.SharedLibs = removeFromList("libdl", deps.SharedLibs)
+ systemSharedLibs := linker.Properties.System_shared_libs
+ if systemSharedLibs == nil {
+ systemSharedLibs = []string{"libc", "libm", "libdl"}
}
- if linker.Properties.System_shared_libs != nil {
- if !inList("libdl", linker.Properties.System_shared_libs) &&
- inList("libc", linker.Properties.System_shared_libs) {
- linker.Properties.System_shared_libs = append(linker.Properties.System_shared_libs,
- "libdl")
+ if inList("libdl", deps.SharedLibs) {
+ // If system_shared_libs has libc but not libdl, make sure shared_libs does not
+ // have libdl to avoid loading libdl before libc.
+ if inList("libc", systemSharedLibs) {
+ if !inList("libdl", systemSharedLibs) {
+ ctx.PropertyErrorf("shared_libs",
+ "libdl must be in system_shared_libs, not shared_libs")
+ }
+ _, deps.SharedLibs = removeFromList("libdl", deps.SharedLibs)
}
- deps.LateSharedLibs = append(deps.LateSharedLibs,
- linker.Properties.System_shared_libs...)
- } else if !ctx.sdk() && !ctx.vndk() {
- deps.LateSharedLibs = append(deps.LateSharedLibs, "libc", "libm", "libdl")
}
- }
- if ctx.sdk() {
- deps.SharedLibs = append(deps.SharedLibs,
- "libc",
- "libm",
- "libdl",
- )
- }
- if ctx.vndk() {
+ // If libc and libdl are both in system_shared_libs make sure libd comes after libc
+ // to avoid loading libdl before libc.
+ if inList("libdl", systemSharedLibs) && inList("libc", systemSharedLibs) &&
+ indexList("libdl", systemSharedLibs) < indexList("libc", systemSharedLibs) {
+ ctx.PropertyErrorf("system_shared_libs", "libdl must be after libc")
+ }
+
+ deps.LateSharedLibs = append(deps.LateSharedLibs, systemSharedLibs...)
+ } else if ctx.useSdk() || ctx.useVndk() {
deps.LateSharedLibs = append(deps.LateSharedLibs, "libc", "libm", "libdl")
}
}
@@ -194,26 +196,43 @@
func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags {
toolchain := ctx.toolchain()
- if !ctx.noDefaultCompilerFlags() {
- if Bool(linker.Properties.Allow_undefined_symbols) {
- if ctx.Darwin() {
- // darwin defaults to treating undefined symbols as errors
- flags.LdFlags = append(flags.LdFlags, "-Wl,-undefined,dynamic_lookup")
+ hod := "Host"
+ if ctx.Os().Class == android.Device {
+ hod = "Device"
+ }
+
+ flags.LdFlags = append(flags.LdFlags, fmt.Sprintf("${config.%sGlobalLdflags}", hod))
+ if Bool(linker.Properties.Allow_undefined_symbols) {
+ if ctx.Darwin() {
+ // darwin defaults to treating undefined symbols as errors
+ flags.LdFlags = append(flags.LdFlags, "-Wl,-undefined,dynamic_lookup")
+ }
+ } else if !ctx.Darwin() {
+ flags.LdFlags = append(flags.LdFlags, "-Wl,--no-undefined")
+ }
+
+ if flags.Clang {
+ flags.LdFlags = append(flags.LdFlags, toolchain.ClangLdflags())
+ } else {
+ flags.LdFlags = append(flags.LdFlags, toolchain.Ldflags())
+ }
+
+ if !ctx.toolchain().Bionic() {
+ CheckBadHostLdlibs(ctx, "host_ldlibs", linker.Properties.Host_ldlibs)
+
+ flags.LdFlags = append(flags.LdFlags, linker.Properties.Host_ldlibs...)
+
+ if !ctx.Windows() {
+ // Add -ldl, -lpthread, -lm and -lrt to host builds to match the default behavior of device
+ // builds
+ flags.LdFlags = append(flags.LdFlags,
+ "-ldl",
+ "-lpthread",
+ "-lm",
+ )
+ if !ctx.Darwin() {
+ flags.LdFlags = append(flags.LdFlags, "-lrt")
}
- } else if !ctx.Darwin() {
- flags.LdFlags = append(flags.LdFlags, "-Wl,--no-undefined")
- }
-
- if flags.Clang {
- flags.LdFlags = append(flags.LdFlags, toolchain.ClangLdflags())
- } else {
- flags.LdFlags = append(flags.LdFlags, toolchain.Ldflags())
- }
-
- if !ctx.toolchain().Bionic() {
- CheckBadHostLdlibs(ctx, "host_ldlibs", linker.Properties.Host_ldlibs)
-
- flags.LdFlags = append(flags.LdFlags, linker.Properties.Host_ldlibs...)
}
}
@@ -227,11 +246,21 @@
rpath_prefix = "@loader_path/"
}
- for _, rpath := range linker.dynamicProperties.RunPaths {
- flags.LdFlags = append(flags.LdFlags, "-Wl,-rpath,"+rpath_prefix+rpath)
+ if !ctx.static() {
+ for _, rpath := range linker.dynamicProperties.RunPaths {
+ flags.LdFlags = append(flags.LdFlags, "-Wl,-rpath,"+rpath_prefix+rpath)
+ }
}
}
+ if ctx.useSdk() && (ctx.Arch().ArchType != android.Mips && ctx.Arch().ArchType != android.Mips64) {
+ // The bionic linker now has support gnu style hashes (which are much faster!), but shipping
+ // to older devices requires the old style hash. Fortunately, we can build with both and
+ // it'll work anywhere.
+ // This is not currently supported on MIPS architectures.
+ flags.LdFlags = append(flags.LdFlags, "-Wl,--hash-style=both")
+ }
+
if flags.Clang {
flags.LdFlags = append(flags.LdFlags, toolchain.ToolchainClangLdflags())
} else {
diff --git a/cc/llndk_library.go b/cc/llndk_library.go
index c3d3462..9a29964 100644
--- a/cc/llndk_library.go
+++ b/cc/llndk_library.go
@@ -23,6 +23,7 @@
var (
llndkLibrarySuffix = ".llndk"
+ llndkHeadersSuffix = ".llndk"
)
// Creates a stub shared library based on the provided version file.
@@ -38,18 +39,26 @@
type llndkLibraryProperties struct {
// Relative path to the symbol map.
// An example file can be seen here: TODO(danalbert): Make an example.
- Symbol_file string
+ Symbol_file *string
// Whether to export any headers as -isystem instead of -I. Mainly for use by
// bionic/libc.
- Export_headers_as_system bool
+ Export_headers_as_system *bool
// Which headers to process with versioner. This really only handles
// bionic/libc/include right now.
Export_preprocessed_headers []string
// Whether the system library uses symbol versions.
- Unversioned bool
+ Unversioned *bool
+
+ // whether this module can be directly depended upon by libs that are installed to /vendor.
+ // When set to false, this module can only be depended on by VNDK libraries, not vendor
+ // libraries. This effectively hides this module from vendors. Default value is true.
+ Vendor_available *bool
+
+ // list of llndk headers to re-export include directories from.
+ Export_llndk_headers []string `android:"arch_variant"`
}
type llndkStubDecorator struct {
@@ -61,19 +70,22 @@
versionScriptPath android.ModuleGenPath
}
-func (stub *llndkStubDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags {
- flags = stub.baseCompiler.compilerFlags(ctx, flags)
+func (stub *llndkStubDecorator) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags {
+ flags = stub.baseCompiler.compilerFlags(ctx, flags, deps)
return addStubLibraryCompilerFlags(flags)
}
func (stub *llndkStubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects {
- objs, versionScript := compileStubLibrary(ctx, flags, stub.Properties.Symbol_file, "current", "--vndk")
+ objs, versionScript := compileStubLibrary(ctx, flags, String(stub.Properties.Symbol_file), "current", "--vndk")
stub.versionScriptPath = versionScript
return objs
}
func (stub *llndkStubDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps {
- return Deps{}
+ headers := addSuffix(stub.Properties.Export_llndk_headers, llndkHeadersSuffix)
+ deps.HeaderLibs = append(deps.HeaderLibs, headers...)
+ deps.ReexportHeaderLibHeaders = append(deps.ReexportHeaderLibHeaders, headers...)
+ return deps
}
func (stub *llndkStubDecorator) Name(name string) string {
@@ -109,7 +121,7 @@
func (stub *llndkStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps,
objs Objects) android.Path {
- if !stub.Properties.Unversioned {
+ if !Bool(stub.Properties.Unversioned) {
linkerScriptFlag := "-Wl,--version-script," + stub.versionScriptPath.String()
flags.LdFlags = append(flags.LdFlags, linkerScriptFlag)
}
@@ -123,7 +135,7 @@
}
includePrefix := "-I "
- if stub.Properties.Export_headers_as_system {
+ if Bool(stub.Properties.Export_headers_as_system) {
includePrefix = "-isystem "
}
@@ -131,7 +143,7 @@
stub.reexportDeps(timestampFiles)
}
- if stub.Properties.Export_headers_as_system {
+ if Bool(stub.Properties.Export_headers_as_system) {
stub.exportIncludes(ctx, "-isystem")
stub.libraryDecorator.flagExporter.Properties.Export_include_dirs = []string{}
}
@@ -139,16 +151,17 @@
return stub.libraryDecorator.link(ctx, flags, deps, objs)
}
-func newLLndkStubLibrary() *Module {
+func NewLLndkStubLibrary() *Module {
module, library := NewLibrary(android.DeviceSupported)
library.BuildOnlyShared()
module.stl = nil
module.sanitize = nil
- library.StripProperties.Strip.None = true
+ library.StripProperties.Strip.None = BoolPtr(true)
stub := &llndkStubDecorator{
libraryDecorator: library,
}
+ stub.Properties.Vendor_available = BoolPtr(true)
module.compiler = stub
module.linker = stub
module.installer = nil
@@ -162,11 +175,40 @@
}
func llndkLibraryFactory() android.Module {
- module := newLLndkStubLibrary()
+ module := NewLLndkStubLibrary()
android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth)
return module
}
+type llndkHeadersDecorator struct {
+ *libraryDecorator
+}
+
+func (headers *llndkHeadersDecorator) Name(name string) string {
+ return name + llndkHeadersSuffix
+}
+
+func llndkHeadersFactory() android.Module {
+ module, library := NewLibrary(android.DeviceSupported)
+ library.HeaderOnly()
+ library.setStatic()
+
+ decorator := &llndkHeadersDecorator{
+ libraryDecorator: library,
+ }
+
+ module.compiler = nil
+ module.linker = decorator
+ module.installer = nil
+
+ module.AddProperties(&library.MutatedProperties, &library.flagExporter.Properties)
+
+ android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth)
+
+ return module
+}
+
func init() {
android.RegisterModuleType("llndk_library", llndkLibraryFactory)
+ android.RegisterModuleType("llndk_headers", llndkHeadersFactory)
}
diff --git a/cc/lto.go b/cc/lto.go
new file mode 100644
index 0000000..fdb7688
--- /dev/null
+++ b/cc/lto.go
@@ -0,0 +1,133 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "android/soong/android"
+)
+
+// LTO (link-time optimization) allows the compiler to optimize and generate
+// code for the entire module at link time, rather than per-compilation
+// unit. LTO is required for Clang CFI and other whole-program optimization
+// techniques. LTO also allows cross-compilation unit optimizations that should
+// result in faster and smaller code, at the expense of additional compilation
+// time.
+//
+// To properly build a module with LTO, the module and all recursive static
+// dependencies should be compiled with -flto which directs the compiler to emit
+// bitcode rather than native object files. These bitcode files are then passed
+// by the linker to the LLVM plugin for compilation at link time. Static
+// dependencies not built as bitcode will still function correctly but cannot be
+// optimized at link time and may not be compatible with features that require
+// LTO, such as CFI.
+//
+// This file adds support to soong to automatically propogate LTO options to a
+// new variant of all static dependencies for each module with LTO enabled.
+
+type LTOProperties struct {
+ // Lto must violate capitialization style for acronyms so that it can be
+ // referred to in blueprint files as "lto"
+ Lto struct {
+ Full *bool `android:"arch_variant"`
+ Thin *bool `android:"arch_variant"`
+ } `android:"arch_variant"`
+ LTODep bool `blueprint:"mutated"`
+}
+
+type lto struct {
+ Properties LTOProperties
+}
+
+func (lto *lto) props() []interface{} {
+ return []interface{}{<o.Properties}
+}
+
+func (lto *lto) begin(ctx BaseModuleContext) {
+}
+
+func (lto *lto) deps(ctx BaseModuleContext, deps Deps) Deps {
+ return deps
+}
+
+func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags {
+ if lto.LTO() {
+ var ltoFlag string
+ if Bool(lto.Properties.Lto.Thin) {
+ ltoFlag = "-flto=thin"
+ } else {
+ ltoFlag = "-flto"
+ }
+
+ flags.CFlags = append(flags.CFlags, ltoFlag)
+ flags.LdFlags = append(flags.LdFlags, ltoFlag)
+ if ctx.Device() {
+ // Work around bug in Clang that doesn't pass correct emulated
+ // TLS option to target
+ flags.LdFlags = append(flags.LdFlags, "-Wl,-plugin-opt,-emulated-tls")
+ }
+ flags.ArFlags = append(flags.ArFlags, " --plugin ${config.LLVMGoldPlugin}")
+ }
+ return flags
+}
+
+// Can be called with a null receiver
+func (lto *lto) LTO() bool {
+ if lto == nil {
+ return false
+ }
+
+ full := Bool(lto.Properties.Lto.Full)
+ thin := Bool(lto.Properties.Lto.Thin)
+ return full || thin
+}
+
+// Propagate lto requirements down from binaries
+func ltoDepsMutator(mctx android.TopDownMutatorContext) {
+ if c, ok := mctx.Module().(*Module); ok && c.lto.LTO() {
+ full := Bool(c.lto.Properties.Lto.Full)
+ thin := Bool(c.lto.Properties.Lto.Thin)
+ if full && thin {
+ mctx.PropertyErrorf("LTO", "FullLTO and ThinLTO are mutually exclusive")
+ }
+
+ mctx.VisitDepsDepthFirst(func(m android.Module) {
+ tag := mctx.OtherModuleDependencyTag(m)
+ switch tag {
+ case staticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag, objDepTag, reuseObjTag:
+ if cc, ok := m.(*Module); ok && cc.lto != nil {
+ cc.lto.Properties.LTODep = true
+ }
+ }
+ })
+ }
+}
+
+// Create lto variants for modules that need them
+func ltoMutator(mctx android.BottomUpMutatorContext) {
+ if c, ok := mctx.Module().(*Module); ok && c.lto != nil {
+ if c.lto.LTO() {
+ mctx.SetDependencyVariation("lto")
+ } else if c.lto.Properties.LTODep {
+ modules := mctx.CreateVariations("", "lto")
+ modules[0].(*Module).lto.Properties.Lto.Full = boolPtr(false)
+ modules[0].(*Module).lto.Properties.Lto.Thin = boolPtr(false)
+ modules[0].(*Module).lto.Properties.LTODep = false
+ modules[1].(*Module).lto.Properties.LTODep = false
+ modules[1].(*Module).Properties.PreventInstall = true
+ modules[1].(*Module).Properties.HideFromMake = true
+ }
+ c.lto.Properties.LTODep = false
+ }
+}
diff --git a/cc/makevars.go b/cc/makevars.go
index 294f3e6..0d2569a 100644
--- a/cc/makevars.go
+++ b/cc/makevars.go
@@ -18,15 +18,50 @@
"fmt"
"sort"
"strings"
+ "sync"
"android/soong/android"
"android/soong/cc/config"
)
+const (
+ modulesAddedWall = "ModulesAddedWall"
+ modulesUsingWnoError = "ModulesUsingWnoError"
+)
+
func init() {
android.RegisterMakeVarsProvider(pctx, makeVarsProvider)
}
+func getWallWerrorMap(config android.Config, name string) *sync.Map {
+ return config.Once(name, func() interface{} {
+ return &sync.Map{}
+ }).(*sync.Map)
+}
+
+func makeStringOfKeys(ctx android.MakeVarsContext, setName string) string {
+ set := getWallWerrorMap(ctx.Config(), setName)
+ keys := []string{}
+ set.Range(func(key interface{}, value interface{}) bool {
+ keys = append(keys, key.(string))
+ return true
+ })
+ sort.Strings(keys)
+ return strings.Join(keys, " ")
+}
+
+func makeStringOfWarningAllowedProjects() string {
+ allProjects := append([]string{}, config.WarningAllowedProjects...)
+ allProjects = append(allProjects, config.WarningAllowedOldProjects...)
+ sort.Strings(allProjects)
+ // Makefile rules use pattern "path/%" to match module paths.
+ if len(allProjects) > 0 {
+ return strings.Join(allProjects, "% ") + "%"
+ } else {
+ return ""
+ }
+}
+
func makeVarsProvider(ctx android.MakeVarsContext) {
ctx.Strict("LLVM_RELEASE_VERSION", "${config.ClangShortVersion}")
ctx.Strict("LLVM_PREBUILTS_VERSION", "${config.ClangVersion}")
@@ -59,6 +94,15 @@
ctx.Strict("BOARD_VNDK_VERSION", "")
}
+ ctx.Strict("VNDK_CORE_LIBRARIES", strings.Join(vndkCoreLibraries, " "))
+ ctx.Strict("VNDK_SAMEPROCESS_LIBRARIES", strings.Join(vndkSpLibraries, " "))
+ ctx.Strict("LLNDK_LIBRARIES", strings.Join(llndkLibraries, " "))
+ ctx.Strict("VNDK_PRIVATE_LIBRARIES", strings.Join(vndkPrivateLibraries, " "))
+
+ ctx.Strict("ANDROID_WARNING_ALLOWED_PROJECTS", makeStringOfWarningAllowedProjects())
+ ctx.Strict("SOONG_MODULES_ADDED_WALL", makeStringOfKeys(ctx, modulesAddedWall))
+ ctx.Strict("SOONG_MODULES_USING_WNO_ERROR", makeStringOfKeys(ctx, modulesUsingWnoError))
+
ctx.Strict("ADDRESS_SANITIZER_CONFIG_EXTRA_CFLAGS", strings.Join(asanCflags, " "))
ctx.Strict("ADDRESS_SANITIZER_CONFIG_EXTRA_LDFLAGS", strings.Join(asanLdflags, " "))
ctx.Strict("ADDRESS_SANITIZER_CONFIG_EXTRA_STATIC_LIBRARIES", strings.Join(asanLibs, " "))
@@ -166,9 +210,11 @@
}, " "))
ctx.Strict(makePrefix+"GLOBAL_CPPFLAGS", strings.Join([]string{
"${config.CommonGlobalCppflags}",
+ fmt.Sprintf("${config.%sGlobalCppflags}", hod),
toolchain.Cppflags(),
}, " "))
ctx.Strict(makePrefix+"GLOBAL_LDFLAGS", strings.Join([]string{
+ fmt.Sprintf("${config.%sGlobalLdflags}", hod),
toolchain.Ldflags(),
toolchain.ToolchainLdflags(),
productExtraLdflags,
@@ -211,9 +257,11 @@
}, " "))
ctx.Strict(clangPrefix+"GLOBAL_CPPFLAGS", strings.Join([]string{
"${config.CommonClangGlobalCppflags}",
+ fmt.Sprintf("${config.%sGlobalCppflags}", hod),
toolchain.ClangCppflags(),
}, " "))
ctx.Strict(clangPrefix+"GLOBAL_LDFLAGS", strings.Join([]string{
+ fmt.Sprintf("${config.%sGlobalLdflags}", hod),
toolchain.ClangLdflags(),
toolchain.ToolchainClangLdflags(),
productExtraLdflags,
@@ -229,6 +277,10 @@
// This is used by external/gentoo/...
ctx.Strict("CLANG_CONFIG_"+target.Arch.ArchType.Name+"_"+typePrefix+"TRIPLE",
toolchain.ClangTriple())
+
+ ctx.Strict(makePrefix+"CLANG_SUPPORTED", "true")
+ } else {
+ ctx.Strict(makePrefix+"CLANG_SUPPORTED", "")
}
ctx.Strict(makePrefix+"CC", gccCmd(toolchain, "gcc"))
@@ -255,6 +307,10 @@
ctx.Strict(makePrefix+"NDK_TRIPLE", toolchain.ClangTriple())
}
+ if target.Os.Class == android.Host || target.Os.Class == android.HostCross {
+ ctx.Strict(makePrefix+"AVAILABLE_LIBRARIES", strings.Join(toolchain.AvailableLibraries(), " "))
+ }
+
ctx.Strict(makePrefix+"TOOLCHAIN_ROOT", toolchain.GccRoot())
ctx.Strict(makePrefix+"TOOLS_PREFIX", gccCmd(toolchain, ""))
ctx.Strict(makePrefix+"SHLIB_SUFFIX", toolchain.ShlibSuffix())
diff --git a/cc/ndk_headers.go b/cc/ndk_headers.go
index 5fa3232..d7c2a06 100644
--- a/cc/ndk_headers.go
+++ b/cc/ndk_headers.go
@@ -18,6 +18,7 @@
"fmt"
"os"
"path/filepath"
+ "strings"
"github.com/google/blueprint"
@@ -56,16 +57,16 @@
//
// Will install $SYSROOT/usr/include/foo/bar/baz.h. If `from` were instead
// "include/foo", it would have installed $SYSROOT/usr/include/bar/baz.h.
- From string
+ From *string
// Install path within the sysroot. This is relative to usr/include.
- To string
+ To *string
// List of headers to install. Glob compatible. Common case is "include/**/*.h".
Srcs []string
// Path to the NOTICE file associated with the headers.
- License string
+ License *string
}
type headerModule struct {
@@ -73,7 +74,7 @@
properties headerProperies
- installPaths []string
+ installPaths android.Paths
licensePath android.ModuleSrcPath
}
@@ -113,23 +114,32 @@
}
func (m *headerModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- if m.properties.License == "" {
+ if String(m.properties.License) == "" {
ctx.PropertyErrorf("license", "field is required")
}
- m.licensePath = android.PathForModuleSrc(ctx, m.properties.License)
+ m.licensePath = android.PathForModuleSrc(ctx, String(m.properties.License))
+
+ // When generating NDK prebuilts, skip installing MIPS headers,
+ // but keep them when doing regular platform build.
+ // Ndk_abis property is only set to true with build/soong/scripts/build-ndk-prebuilts.sh
+ // TODO: Revert this once MIPS is supported in NDK again.
+ if Bool(ctx.AConfig().Ndk_abis) && strings.Contains(ctx.ModuleName(), "mips") {
+ return
+ }
srcFiles := ctx.ExpandSources(m.properties.Srcs, nil)
for _, header := range srcFiles {
- installDir := getHeaderInstallDir(ctx, header, m.properties.From, m.properties.To)
- installedPath := ctx.InstallFile(installDir, header)
+ installDir := getHeaderInstallDir(ctx, header, String(m.properties.From),
+ String(m.properties.To))
+ installedPath := ctx.InstallFile(installDir, header.Base(), header)
installPath := installDir.Join(ctx, header.Base())
if installPath != installedPath {
panic(fmt.Sprintf(
"expected header install path (%q) not equal to actual install path %q",
installPath, installedPath))
}
- m.installPaths = append(m.installPaths, installPath.String())
+ m.installPaths = append(m.installPaths, installPath)
}
if len(m.installPaths) == 0 {
@@ -155,13 +165,13 @@
//
// Will install $SYSROOT/usr/include/foo/bar/baz.h. If `from` were instead
// "include/foo", it would have installed $SYSROOT/usr/include/bar/baz.h.
- From string
+ From *string
// Install path within the sysroot. This is relative to usr/include.
- To string
+ To *string
// Path to the NOTICE file associated with the headers.
- License string
+ License *string
}
// Like ndk_headers, but preprocesses the headers with the bionic versioner:
@@ -176,7 +186,7 @@
properties preprocessedHeaderProperies
- installPaths []string
+ installPaths android.Paths
licensePath android.ModuleSrcPath
}
@@ -184,25 +194,25 @@
}
func (m *preprocessedHeaderModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- if m.properties.License == "" {
+ if String(m.properties.License) == "" {
ctx.PropertyErrorf("license", "field is required")
}
- m.licensePath = android.PathForModuleSrc(ctx, m.properties.License)
+ m.licensePath = android.PathForModuleSrc(ctx, String(m.properties.License))
- fromSrcPath := android.PathForModuleSrc(ctx, m.properties.From)
- toOutputPath := getCurrentIncludePath(ctx).Join(ctx, m.properties.To)
+ fromSrcPath := android.PathForModuleSrc(ctx, String(m.properties.From))
+ toOutputPath := getCurrentIncludePath(ctx).Join(ctx, String(m.properties.To))
srcFiles := ctx.Glob(filepath.Join(fromSrcPath.String(), "**/*.h"), nil)
var installPaths []android.WritablePath
for _, header := range srcFiles {
- installDir := getHeaderInstallDir(ctx, header, m.properties.From, m.properties.To)
+ installDir := getHeaderInstallDir(ctx, header, String(m.properties.From), String(m.properties.To))
installPath := installDir.Join(ctx, header.Base())
installPaths = append(installPaths, installPath)
- m.installPaths = append(m.installPaths, installPath.String())
+ m.installPaths = append(m.installPaths, installPath)
}
if len(m.installPaths) == 0 {
- ctx.ModuleErrorf("glob %q matched zero files", m.properties.From)
+ ctx.ModuleErrorf("glob %q matched zero files", String(m.properties.From))
}
processHeadersWithVersioner(ctx, fromSrcPath, toOutputPath, srcFiles, installPaths)
@@ -237,7 +247,7 @@
}
timestampFile := android.PathForModuleOut(ctx, "versioner.timestamp")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: preprocessBionicHeaders,
Description: "versioner preprocess " + srcDir.Rel(),
Output: timestampFile,
diff --git a/cc/ndk_library.go b/cc/ndk_library.go
index dbfc5be..459d980 100644
--- a/cc/ndk_library.go
+++ b/cc/ndk_library.go
@@ -76,17 +76,17 @@
type libraryProperties struct {
// Relative path to the symbol map.
// An example file can be seen here: TODO(danalbert): Make an example.
- Symbol_file string
+ Symbol_file *string
// The first API level a library was available. A library will be generated
// for every API level beginning with this one.
- First_version string
+ First_version *string
// The first API level that library should have the version script applied.
// This defaults to the value of first_version, and should almost never be
// used. This is only needed to work around platform bugs like
// https://github.com/android-ndk/ndk/issues/265.
- Unversioned_until string
+ Unversioned_until *string
// Private property for use by the mutator that splits per-API level.
ApiLevel string `blueprint:"mutated"`
@@ -98,7 +98,7 @@
properties libraryProperties
versionScriptPath android.ModuleGenPath
- installPath string
+ installPath android.Path
}
// OMG GO
@@ -110,18 +110,20 @@
}
}
-func normalizeNdkApiLevel(apiLevel string, arch android.Arch) (string, error) {
+func normalizeNdkApiLevel(ctx android.BaseContext, apiLevel string,
+ arch android.Arch) (string, error) {
+
if apiLevel == "current" {
return apiLevel, nil
}
- minVersion := 9 // Minimum version supported by the NDK.
+ minVersion := ctx.Config().MinSupportedSdkVersion()
firstArchVersions := map[android.ArchType]int{
- android.Arm: 9,
+ android.Arm: minVersion,
android.Arm64: 21,
- android.Mips: 9,
+ android.Mips: minVersion,
android.Mips64: 21,
- android.X86: 9,
+ android.X86: minVersion,
android.X86_64: 21,
}
@@ -156,11 +158,11 @@
func shouldUseVersionScript(stub *stubDecorator) (bool, error) {
// unversioned_until is normally empty, in which case we should use the version script.
- if stub.properties.Unversioned_until == "" {
+ if String(stub.properties.Unversioned_until) == "" {
return true, nil
}
- if stub.properties.Unversioned_until == "current" {
+ if String(stub.properties.Unversioned_until) == "current" {
if stub.properties.ApiLevel == "current" {
return true, nil
} else {
@@ -172,7 +174,7 @@
return true, nil
}
- unversionedUntil, err := strconv.Atoi(stub.properties.Unversioned_until)
+ unversionedUntil, err := strconv.Atoi(String(stub.properties.Unversioned_until))
if err != nil {
return true, err
}
@@ -186,9 +188,9 @@
}
func generateStubApiVariants(mctx android.BottomUpMutatorContext, c *stubDecorator) {
- platformVersion := mctx.AConfig().PlatformSdkVersionInt()
+ platformVersion := mctx.Config().PlatformSdkVersionInt()
- firstSupportedVersion, err := normalizeNdkApiLevel(c.properties.First_version,
+ firstSupportedVersion, err := normalizeNdkApiLevel(mctx, String(c.properties.First_version),
mctx.Arch())
if err != nil {
mctx.PropertyErrorf("first_version", err.Error())
@@ -205,7 +207,7 @@
for version := firstGenVersion; version <= platformVersion; version++ {
versionStrs = append(versionStrs, strconv.Itoa(version))
}
- versionStrs = append(versionStrs, mctx.AConfig().PlatformVersionAllCodenames()...)
+ versionStrs = append(versionStrs, mctx.Config().PlatformVersionActiveCodenames()...)
versionStrs = append(versionStrs, "current")
modules := mctx.CreateVariations(versionStrs...)
@@ -249,6 +251,8 @@
"-Wno-incompatible-library-redeclaration",
"-Wno-builtin-requires-header",
"-Wno-invalid-noreturn",
+ "-Wall",
+ "-Werror",
// These libraries aren't actually used. Don't worry about unwinding
// (avoids the need to link an unwinder into a fake library).
"-fno-unwind-tables",
@@ -256,8 +260,8 @@
return flags
}
-func (stub *stubDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags {
- flags = stub.baseCompiler.compilerFlags(ctx, flags)
+func (stub *stubDecorator) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags {
+ flags = stub.baseCompiler.compilerFlags(ctx, flags, deps)
return addStubLibraryCompilerFlags(flags)
}
@@ -268,7 +272,7 @@
versionScriptPath := android.PathForModuleGen(ctx, "stub.map")
symbolFilePath := android.PathForModuleSrc(ctx, symbolFile)
apiLevelsJson := android.GetApiLevelsJson(ctx)
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: genStubSrc,
Description: "generate stubs " + symbolFilePath.Rel(),
Outputs: []android.WritablePath{stubSrcPath, versionScriptPath},
@@ -288,11 +292,12 @@
}
func (c *stubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects {
- if !strings.HasSuffix(c.properties.Symbol_file, ".map.txt") {
+ if !strings.HasSuffix(String(c.properties.Symbol_file), ".map.txt") {
ctx.PropertyErrorf("symbol_file", "must end with .map.txt")
}
- objs, versionScript := compileStubLibrary(ctx, flags, c.properties.Symbol_file, c.properties.ApiLevel, "")
+ objs, versionScript := compileStubLibrary(ctx, flags, String(c.properties.Symbol_file),
+ c.properties.ApiLevel, "")
c.versionScriptPath = versionScript
return objs
}
@@ -339,7 +344,7 @@
installDir := getNdkInstallBase(ctx).Join(ctx, fmt.Sprintf(
"platforms/android-%s/arch-%s/usr/%s", apiLevel, arch, libDir))
- stub.installPath = ctx.InstallFile(installDir, path).String()
+ stub.installPath = ctx.InstallFile(installDir, path.Base(), path)
}
func newStubLibrary() *Module {
@@ -347,7 +352,7 @@
library.BuildOnlyShared()
module.stl = nil
module.sanitize = nil
- library.StripProperties.Strip.None = true
+ library.StripProperties.Strip.None = BoolPtr(true)
stub := &stubDecorator{
libraryDecorator: library,
diff --git a/cc/ndk_sysroot.go b/cc/ndk_sysroot.go
index 5b4cfbe..4324458 100644
--- a/cc/ndk_sysroot.go
+++ b/cc/ndk_sysroot.go
@@ -53,8 +53,6 @@
// TODO(danalbert): Write `ndk_static_library` rule.
import (
- "github.com/google/blueprint"
-
"android/soong/android"
)
@@ -76,58 +74,62 @@
return getNdkInstallBase(ctx).Join(ctx, "sysroot")
}
-func getNdkSysrootTimestampFile(ctx android.PathContext) android.Path {
+func getNdkSysrootTimestampFile(ctx android.PathContext) android.WritablePath {
return android.PathForOutput(ctx, "ndk.timestamp")
}
-func NdkSingleton() blueprint.Singleton {
+func NdkSingleton() android.Singleton {
return &ndkSingleton{}
}
type ndkSingleton struct{}
-func (n *ndkSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
- installPaths := []string{}
- licensePaths := []string{}
- ctx.VisitAllModules(func(module blueprint.Module) {
+func (n *ndkSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+ var installPaths android.Paths
+ var licensePaths android.Paths
+ ctx.VisitAllModules(func(module android.Module) {
if m, ok := module.(android.Module); ok && !m.Enabled() {
return
}
if m, ok := module.(*headerModule); ok {
installPaths = append(installPaths, m.installPaths...)
- licensePaths = append(licensePaths, m.licensePath.String())
+ licensePaths = append(licensePaths, m.licensePath)
}
if m, ok := module.(*preprocessedHeaderModule); ok {
installPaths = append(installPaths, m.installPaths...)
- licensePaths = append(licensePaths, m.licensePath.String())
+ licensePaths = append(licensePaths, m.licensePath)
}
if m, ok := module.(*Module); ok {
if installer, ok := m.installer.(*stubDecorator); ok {
installPaths = append(installPaths, installer.installPath)
}
+
+ if library, ok := m.linker.(*libraryDecorator); ok {
+ if library.ndkSysrootPath != nil {
+ installPaths = append(installPaths, library.ndkSysrootPath)
+ }
+ }
}
})
combinedLicense := getNdkInstallBase(ctx).Join(ctx, "NOTICE")
- ctx.Build(pctx, blueprint.BuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: android.Cat,
Description: "combine licenses",
- Outputs: []string{combinedLicense.String()},
+ Output: combinedLicense,
Inputs: licensePaths,
- Optional: true,
})
- depPaths := append(installPaths, combinedLicense.String())
+ depPaths := append(installPaths, combinedLicense)
// There's a dummy "ndk" rule defined in ndk/Android.mk that depends on
// this. `m ndk` will build the sysroots.
- ctx.Build(pctx, blueprint.BuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: android.Touch,
- Outputs: []string{getNdkSysrootTimestampFile(ctx).String()},
+ Output: getNdkSysrootTimestampFile(ctx),
Implicits: depPaths,
- Optional: true,
})
}
diff --git a/cc/object.go b/cc/object.go
index 1478908..d0f4f20 100644
--- a/cc/object.go
+++ b/cc/object.go
@@ -43,7 +43,7 @@
}
func (object *objectLinker) appendLdflags(flags []string) {
- panic(fmt.Errorf("appendLdflags on object Linker not supported"))
+ panic(fmt.Errorf("appendLdflags on objectLinker not supported"))
}
func (object *objectLinker) linkerProps() []interface{} {
@@ -53,6 +53,11 @@
func (*objectLinker) linkerInit(ctx BaseModuleContext) {}
func (object *objectLinker) linkerDeps(ctx DepsContext, deps Deps) Deps {
+ if ctx.useVndk() && ctx.toolchain().Bionic() {
+ // Needed for VNDK builds where bionic headers aren't automatically added.
+ deps.LateSharedLibs = append(deps.LateSharedLibs, "libc")
+ }
+
deps.ObjFiles = append(deps.ObjFiles, object.Properties.Objs...)
return deps
}
@@ -73,12 +78,29 @@
objs = objs.Append(deps.Objs)
var outputFile android.Path
+ builderFlags := flagsToBuilderFlags(flags)
+
if len(objs.objFiles) == 1 {
outputFile = objs.objFiles[0]
+
+ if String(object.Properties.Prefix_symbols) != "" {
+ output := android.PathForModuleOut(ctx, ctx.ModuleName()+objectExtension)
+ TransformBinaryPrefixSymbols(ctx, String(object.Properties.Prefix_symbols), outputFile,
+ builderFlags, output)
+ outputFile = output
+ }
} else {
output := android.PathForModuleOut(ctx, ctx.ModuleName()+objectExtension)
- TransformObjsToObj(ctx, objs.objFiles, flagsToBuilderFlags(flags), output)
outputFile = output
+
+ if String(object.Properties.Prefix_symbols) != "" {
+ input := android.PathForModuleOut(ctx, "unprefixed", ctx.ModuleName()+objectExtension)
+ TransformBinaryPrefixSymbols(ctx, String(object.Properties.Prefix_symbols), input,
+ builderFlags, output)
+ output = input
+ }
+
+ TransformObjsToObj(ctx, objs.objFiles, builderFlags, output)
}
ctx.CheckbuildFile(outputFile)
diff --git a/cc/pgo.go b/cc/pgo.go
new file mode 100644
index 0000000..9fea154
--- /dev/null
+++ b/cc/pgo.go
@@ -0,0 +1,223 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "fmt"
+ "strings"
+
+ "android/soong/android"
+ "android/soong/cc/config"
+)
+
+var (
+ // Add flags to ignore warnings that profiles are old or missing for
+ // some functions
+ profileUseOtherFlags = []string{"-Wno-backend-plugin"}
+)
+
+const pgoProfileProject = "toolchain/pgo-profiles"
+
+const profileInstrumentFlag = "-fprofile-generate=/data/local/tmp"
+const profileSamplingFlag = "-gline-tables-only"
+const profileUseInstrumentFormat = "-fprofile-use=%s"
+const profileUseSamplingFormat = "-fprofile-sample-use=%s"
+
+type PgoProperties struct {
+ Pgo struct {
+ Instrumentation *bool
+ Sampling *bool
+ Profile_file *string `android:"arch_variant"`
+ Benchmarks []string
+ Enable_profile_use *bool `android:"arch_variant"`
+ } `android:"arch_variant"`
+
+ PgoPresent bool `blueprint:"mutated"`
+ ShouldProfileModule bool `blueprint:"mutated"`
+}
+
+type pgo struct {
+ Properties PgoProperties
+}
+
+func (props *PgoProperties) isInstrumentation() bool {
+ return props.Pgo.Instrumentation != nil && *props.Pgo.Instrumentation == true
+}
+
+func (props *PgoProperties) isSampling() bool {
+ return props.Pgo.Sampling != nil && *props.Pgo.Sampling == true
+}
+
+func (pgo *pgo) props() []interface{} {
+ return []interface{}{&pgo.Properties}
+}
+
+func (props *PgoProperties) addProfileGatherFlags(ctx ModuleContext, flags Flags) Flags {
+ if props.isInstrumentation() {
+ flags.CFlags = append(flags.CFlags, profileInstrumentFlag)
+ // The profile runtime is added below in deps(). Add the below
+ // flag, which is the only other link-time action performed by
+ // the Clang driver during link.
+ flags.LdFlags = append(flags.LdFlags, "-u__llvm_profile_runtime")
+ }
+ if props.isSampling() {
+ flags.CFlags = append(flags.CFlags, profileSamplingFlag)
+ flags.LdFlags = append(flags.LdFlags, profileSamplingFlag)
+ }
+ return flags
+}
+
+func (props *PgoProperties) profileUseFlag(ctx ModuleContext, file string) string {
+ if props.isInstrumentation() {
+ return fmt.Sprintf(profileUseInstrumentFormat, file)
+ }
+ if props.isSampling() {
+ return fmt.Sprintf(profileUseSamplingFormat, file)
+ }
+ return ""
+}
+
+func (props *PgoProperties) profileUseFlags(ctx ModuleContext, file string) []string {
+ flags := []string{props.profileUseFlag(ctx, file)}
+ flags = append(flags, profileUseOtherFlags...)
+ return flags
+}
+
+func (props *PgoProperties) addProfileUseFlags(ctx ModuleContext, flags Flags) Flags {
+ // Skip -fprofile-use if 'enable_profile_use' property is set
+ if props.Pgo.Enable_profile_use != nil && *props.Pgo.Enable_profile_use == false {
+ return flags
+ }
+
+ // If the PGO profiles project is found, and this module has PGO
+ // enabled, add flags to use the profile
+ if profilesDir := getPgoProfilesDir(ctx); props.PgoPresent && profilesDir.Valid() {
+ profileFile := android.PathForSource(ctx, profilesDir.String(), *props.Pgo.Profile_file)
+ profileUseFlags := props.profileUseFlags(ctx, profileFile.String())
+
+ flags.CFlags = append(flags.CFlags, profileUseFlags...)
+ flags.LdFlags = append(flags.LdFlags, profileUseFlags...)
+
+ // Update CFlagsDeps and LdFlagsDeps so the module is rebuilt
+ // if profileFile gets updated
+ flags.CFlagsDeps = append(flags.CFlagsDeps, profileFile)
+ flags.LdFlagsDeps = append(flags.LdFlagsDeps, profileFile)
+ }
+ return flags
+}
+
+func (props *PgoProperties) isPGO(ctx BaseModuleContext) bool {
+ isInstrumentation := props.isInstrumentation()
+ isSampling := props.isSampling()
+
+ profileKindPresent := isInstrumentation || isSampling
+ filePresent := props.Pgo.Profile_file != nil
+ benchmarksPresent := len(props.Pgo.Benchmarks) > 0
+
+ // If all three properties are absent, PGO is OFF for this module
+ if !profileKindPresent && !filePresent && !benchmarksPresent {
+ return false
+ }
+
+ // If at least one property exists, validate that all properties exist
+ if !profileKindPresent || !filePresent || !benchmarksPresent {
+ var missing []string
+ if !profileKindPresent {
+ missing = append(missing, "profile kind (either \"instrumentation\" or \"sampling\" property)")
+ }
+ if !filePresent {
+ missing = append(missing, "profile_file property")
+ }
+ if !benchmarksPresent {
+ missing = append(missing, "non-empty benchmarks property")
+ }
+ missingProps := strings.Join(missing, ", ")
+ ctx.ModuleErrorf("PGO specification is missing properties: " + missingProps)
+ }
+
+ // Sampling not supported yet
+ if isSampling {
+ ctx.PropertyErrorf("pgo.sampling", "\"sampling\" is not supported yet)")
+ }
+
+ if isSampling && isInstrumentation {
+ ctx.PropertyErrorf("pgo", "Exactly one of \"instrumentation\" and \"sampling\" properties must be set")
+ }
+
+ return true
+}
+
+func getPgoProfilesDir(ctx ModuleContext) android.OptionalPath {
+ return android.ExistentPathForSource(ctx, "", pgoProfileProject)
+}
+
+func (pgo *pgo) begin(ctx BaseModuleContext) {
+ // TODO Evaluate if we need to support PGO for host modules
+ if ctx.Host() {
+ return
+ }
+
+ // Check if PGO is needed for this module
+ pgo.Properties.PgoPresent = pgo.Properties.isPGO(ctx)
+
+ if !pgo.Properties.PgoPresent {
+ return
+ }
+
+ // This module should be instrumented if ANDROID_PGO_INSTRUMENT is set
+ // and includes a benchmark listed for this module
+ //
+ // TODO Validate that each benchmark instruments at least one module
+ pgo.Properties.ShouldProfileModule = false
+ pgoBenchmarks := ctx.Config().Getenv("ANDROID_PGO_INSTRUMENT")
+ pgoBenchmarksMap := make(map[string]bool)
+ for _, b := range strings.Split(pgoBenchmarks, ",") {
+ pgoBenchmarksMap[b] = true
+ }
+
+ for _, b := range pgo.Properties.Pgo.Benchmarks {
+ if pgoBenchmarksMap[b] == true {
+ pgo.Properties.ShouldProfileModule = true
+ break
+ }
+ }
+}
+
+func (pgo *pgo) deps(ctx BaseModuleContext, deps Deps) Deps {
+ if pgo.Properties.ShouldProfileModule {
+ runtimeLibrary := config.ProfileRuntimeLibrary(ctx.toolchain())
+ deps.LateStaticLibs = append(deps.LateStaticLibs, runtimeLibrary)
+ }
+ return deps
+}
+
+func (pgo *pgo) flags(ctx ModuleContext, flags Flags) Flags {
+ if ctx.Host() {
+ return flags
+ }
+
+ props := pgo.Properties
+
+ // Add flags to profile this module based on its profile_kind
+ if props.ShouldProfileModule {
+ return props.addProfileGatherFlags(ctx, flags)
+ }
+
+ if !ctx.Config().IsEnvTrue("ANDROID_PGO_NO_PROFILE_USE") {
+ return props.addProfileUseFlags(ctx, flags)
+ }
+
+ return flags
+}
diff --git a/cc/prebuilt.go b/cc/prebuilt.go
index ac05df0..3f277aa 100644
--- a/cc/prebuilt.go
+++ b/cc/prebuilt.go
@@ -31,12 +31,19 @@
type prebuiltLinker struct {
android.Prebuilt
+ properties struct {
+ Srcs []string `android:"arch_variant"`
+ }
}
func (p *prebuiltLinker) prebuilt() *android.Prebuilt {
return &p.Prebuilt
}
+func (p *prebuiltLinker) PrebuiltSrcs() []string {
+ return p.properties.Srcs
+}
+
type prebuiltLibraryLinker struct {
*libraryDecorator
prebuiltLinker
@@ -44,20 +51,15 @@
var _ prebuiltLinkerInterface = (*prebuiltLibraryLinker)(nil)
-func (p *prebuiltLibraryLinker) linkerProps() []interface{} {
- props := p.libraryDecorator.linkerProps()
- return append(props, &p.Prebuilt.Properties)
-}
-
func (p *prebuiltLibraryLinker) link(ctx ModuleContext,
flags Flags, deps PathDeps, objs Objects) android.Path {
// TODO(ccross): verify shared library dependencies
- if len(p.Prebuilt.Properties.Srcs) > 0 {
+ if len(p.properties.Srcs) > 0 {
p.libraryDecorator.exportIncludes(ctx, "-I")
p.libraryDecorator.reexportFlags(deps.ReexportedFlags)
p.libraryDecorator.reexportDeps(deps.ReexportedFlagsDeps)
// TODO(ccross): .toc optimization, stripping, packing
- return p.Prebuilt.Path(ctx)
+ return p.Prebuilt.SingleSourcePath(ctx)
}
return nil
@@ -78,6 +80,9 @@
}
module.linker = prebuilt
+ module.AddProperties(&prebuilt.properties)
+
+ android.InitPrebuiltModule(module, &prebuilt.properties.Srcs)
return module, library
}
@@ -96,6 +101,9 @@
}
module.linker = prebuilt
+ module.AddProperties(&prebuilt.properties)
+
+ android.InitPrebuiltModule(module, &prebuilt.properties.Srcs)
return module, library
}
@@ -106,17 +114,24 @@
var _ prebuiltLinkerInterface = (*prebuiltBinaryLinker)(nil)
-func (p *prebuiltBinaryLinker) linkerProps() []interface{} {
- props := p.binaryDecorator.linkerProps()
- return append(props, &p.Prebuilt.Properties)
-}
-
func (p *prebuiltBinaryLinker) link(ctx ModuleContext,
flags Flags, deps PathDeps, objs Objects) android.Path {
// TODO(ccross): verify shared library dependencies
- if len(p.Prebuilt.Properties.Srcs) > 0 {
+ if len(p.properties.Srcs) > 0 {
// TODO(ccross): .toc optimization, stripping, packing
- return p.Prebuilt.Path(ctx)
+
+ // Copy binaries to a name matching the final installed name
+ fileName := p.getStem(ctx) + flags.Toolchain.ExecutableSuffix()
+ outputFile := android.PathForModuleOut(ctx, fileName)
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: android.CpExecutable,
+ Description: "prebuilt",
+ Output: outputFile,
+ Input: p.Prebuilt.SingleSourcePath(ctx),
+ })
+
+ return outputFile
}
return nil
@@ -136,5 +151,8 @@
}
module.linker = prebuilt
+ module.AddProperties(&prebuilt.properties)
+
+ android.InitPrebuiltModule(module, &prebuilt.properties.Srcs)
return module, binary
}
diff --git a/cc/proto.go b/cc/proto.go
index 6c1789a..e7f1d41 100644
--- a/cc/proto.go
+++ b/cc/proto.go
@@ -16,7 +16,6 @@
import (
"github.com/google/blueprint"
- "github.com/google/blueprint/proptools"
"android/soong/android"
)
@@ -33,85 +32,49 @@
}, "protoFlags", "outDir")
)
-// TODO(ccross): protos are often used to communicate between multiple modules. If the only
-// way to convert a proto to source is to reference it as a source file, and external modules cannot
-// reference source files in other modules, then every module that owns a proto file will need to
-// export a library for every type of external user (lite vs. full, c vs. c++ vs. java). It would
-// be better to support a proto module type that exported a proto file along with some include dirs,
-// and then external modules could depend on the proto module but use their own settings to
-// generate the source.
-
+// genProto creates a rule to convert a .proto file to generated .pb.cc and .pb.h files and returns
+// the paths to the generated files.
func genProto(ctx android.ModuleContext, protoFile android.Path,
- protoFlags string) (android.ModuleGenPath, android.ModuleGenPath) {
+ protoFlags string) (ccFile, headerFile android.WritablePath) {
- outFile := android.GenPathWithExt(ctx, "proto", protoFile, "pb.cc")
- headerFile := android.GenPathWithExt(ctx, "proto", protoFile, "pb.h")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ccFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb.cc")
+ headerFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb.h")
+
+ ctx.Build(pctx, android.BuildParams{
Rule: proto,
Description: "protoc " + protoFile.Rel(),
- Outputs: android.WritablePaths{outFile, headerFile},
+ Outputs: android.WritablePaths{ccFile, headerFile},
Input: protoFile,
Args: map[string]string{
- "outDir": protoDir(ctx).String(),
+ "outDir": android.ProtoDir(ctx).String(),
"protoFlags": protoFlags,
},
})
- return outFile, headerFile
+ return ccFile, headerFile
}
-// protoDir returns the module's "gen/proto" directory
-func protoDir(ctx android.ModuleContext) android.ModuleGenPath {
- return android.PathForModuleGen(ctx, "proto")
-}
-
-// protoSubDir returns the module's "gen/proto/path/to/module" directory
-func protoSubDir(ctx android.ModuleContext) android.ModuleGenPath {
- return android.PathForModuleGen(ctx, "proto", ctx.ModuleDir())
-}
-
-type ProtoProperties struct {
- Proto struct {
- // Proto generator type (full, lite)
- Type *string `android:"arch_variant"`
-
- // Link statically against the protobuf runtime
- Static bool `android:"arch_variant"`
-
- // list of directories that will be added to the protoc include paths.
- Include_dirs []string
-
- // list of directories relative to the Android.bp file that will
- // be added to the protoc include paths.
- Local_include_dirs []string
- } `android:"arch_variant"`
-}
-
-func protoDeps(ctx BaseModuleContext, deps Deps, p *ProtoProperties) Deps {
+func protoDeps(ctx BaseModuleContext, deps Deps, p *android.ProtoProperties, static bool) Deps {
var lib string
- var static bool
- switch proptools.String(p.Proto.Type) {
+ switch String(p.Proto.Type) {
case "full":
- if ctx.sdk() {
+ if ctx.useSdk() {
lib = "libprotobuf-cpp-full-ndk"
static = true
} else {
lib = "libprotobuf-cpp-full"
}
case "lite", "":
- if ctx.sdk() {
+ if ctx.useSdk() {
lib = "libprotobuf-cpp-lite-ndk"
static = true
} else {
lib = "libprotobuf-cpp-lite"
- if p.Proto.Static {
- static = true
- }
}
default:
ctx.PropertyErrorf("proto.type", "unknown proto type %q",
- proptools.String(p.Proto.Type))
+ String(p.Proto.Type))
}
if static {
@@ -125,23 +88,14 @@
return deps
}
-func protoFlags(ctx ModuleContext, flags Flags, p *ProtoProperties) Flags {
+func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flags {
flags.CFlags = append(flags.CFlags, "-DGOOGLE_PROTOBUF_NO_RTTI")
flags.GlobalFlags = append(flags.GlobalFlags,
- "-I"+protoSubDir(ctx).String(),
- "-I"+protoDir(ctx).String(),
+ "-I"+android.ProtoSubDir(ctx).String(),
+ "-I"+android.ProtoDir(ctx).String(),
)
- if len(p.Proto.Local_include_dirs) > 0 {
- localProtoIncludeDirs := android.PathsForModuleSrc(ctx, p.Proto.Local_include_dirs)
- flags.protoFlags = append(flags.protoFlags, includeDirsToFlags(localProtoIncludeDirs))
- }
- if len(p.Proto.Include_dirs) > 0 {
- rootProtoIncludeDirs := android.PathsForSource(ctx, p.Proto.Include_dirs)
- flags.protoFlags = append(flags.protoFlags, includeDirsToFlags(rootProtoIncludeDirs))
- }
-
- flags.protoFlags = append(flags.protoFlags, "-I .")
+ flags.protoFlags = android.ProtoFlags(ctx, p)
return flags
}
diff --git a/cc/relocation_packer.go b/cc/relocation_packer.go
index c9f82ba..5006623 100644
--- a/cc/relocation_packer.go
+++ b/cc/relocation_packer.go
@@ -53,10 +53,10 @@
if ctx.Target().Os != android.Android {
enabled = false
}
- if ctx.AConfig().Getenv("DISABLE_RELOCATION_PACKER") == "true" {
+ if ctx.Config().Getenv("DISABLE_RELOCATION_PACKER") == "true" {
enabled = false
}
- if ctx.sdk() {
+ if ctx.useSdk() {
enabled = false
}
if p.Properties.Pack_relocations != nil &&
@@ -68,14 +68,14 @@
}
func (p *relocationPacker) needsPacking(ctx ModuleContext) bool {
- if ctx.AConfig().EmbeddedInMake() {
+ if ctx.Config().EmbeddedInMake() {
return false
}
return p.Properties.PackingRelocations
}
func (p *relocationPacker) pack(ctx ModuleContext, in, out android.ModuleOutPath, flags builderFlags) {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: relocationPackerRule,
Description: "pack relocations",
Output: out,
diff --git a/cc/rs.go b/cc/rs.go
index 976107e..68ba54b 100644
--- a/cc/rs.go
+++ b/cc/rs.go
@@ -19,7 +19,6 @@
"strings"
"github.com/google/blueprint"
- "github.com/google/blueprint/proptools"
)
func init() {
@@ -64,7 +63,7 @@
cppFiles[i] = rsGeneratedCppFile(ctx, rsFile)
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: rsCpp,
Description: "llvm-rs-cc",
Output: stampFile,
@@ -81,13 +80,13 @@
}
func rsFlags(ctx ModuleContext, flags Flags, properties *BaseCompilerProperties) Flags {
- targetApi := proptools.String(properties.Renderscript.Target_api)
- if targetApi == "" && ctx.sdk() {
+ targetApi := String(properties.Renderscript.Target_api)
+ if targetApi == "" && ctx.useSdk() {
switch ctx.sdkVersion() {
case "current", "system_current", "test_current":
// Nothing
default:
- targetApi = ctx.sdkVersion()
+ targetApi = android.GetNumericSdkVersion(ctx.sdkVersion())
}
}
diff --git a/cc/sabi.go b/cc/sabi.go
index 1a5361d..ec1d246 100644
--- a/cc/sabi.go
+++ b/cc/sabi.go
@@ -17,8 +17,6 @@
import (
"strings"
- "github.com/google/blueprint"
-
"android/soong/android"
"android/soong/cc/config"
)
@@ -79,9 +77,9 @@
func sabiDepsMutator(mctx android.TopDownMutatorContext) {
if c, ok := mctx.Module().(*Module); ok &&
- (c.isVndk() || inList(c.Name(), config.LLndkLibraries()) ||
+ ((c.isVndk() && c.useVndk()) || inList(c.Name(), llndkLibraries) ||
(c.sabi != nil && c.sabi.Properties.CreateSAbiDumps)) {
- mctx.VisitDirectDeps(func(m blueprint.Module) {
+ mctx.VisitDirectDeps(func(m android.Module) {
tag := mctx.OtherModuleDependencyTag(m)
switch tag {
case staticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag:
diff --git a/cc/sanitize.go b/cc/sanitize.go
index 1fcb32c..1afec26 100644
--- a/cc/sanitize.go
+++ b/cc/sanitize.go
@@ -17,9 +17,9 @@
import (
"fmt"
"io"
+ "sort"
"strings"
-
- "github.com/google/blueprint"
+ "sync"
"android/soong/android"
"android/soong/cc/config"
@@ -34,12 +34,14 @@
asanLdflags = []string{"-Wl,-u,__asan_preinit"}
asanLibs = []string{"libasan"}
- cfiCflags = []string{"-flto", "-fsanitize-cfi-cross-dso", "-fvisibility=default",
+ cfiCflags = []string{"-flto", "-fsanitize-cfi-cross-dso",
"-fsanitize-blacklist=external/compiler-rt/lib/cfi/cfi_blacklist.txt"}
- // FIXME: revert the __cfi_check flag when clang is updated to r280031.
cfiLdflags = []string{"-flto", "-fsanitize-cfi-cross-dso", "-fsanitize=cfi",
- "-Wl,-plugin-opt,O1 -Wl,-export-dynamic-symbol=__cfi_check"}
- cfiArflags = []string{"--plugin ${config.ClangBin}/../lib64/LLVMgold.so"}
+ "-Wl,-plugin-opt,O1"}
+ cfiArflags = []string{"--plugin ${config.ClangBin}/../lib64/LLVMgold.so"}
+ cfiExportsMapPath = "build/soong/cc/config/cfi_exports.map"
+ cfiExportsMap android.Path
+ cfiStaticLibsMutex sync.Mutex
intOverflowCflags = []string{"-fsanitize-blacklist=build/soong/cc/config/integer_overflow_blacklist.txt"}
)
@@ -58,6 +60,7 @@
asan sanitizerType = iota + 1
tsan
intOverflow
+ cfi
)
func (t sanitizerType) String() string {
@@ -68,6 +71,8 @@
return "tsan"
case intOverflow:
return "intOverflow"
+ case cfi:
+ return "cfi"
default:
panic(fmt.Errorf("unknown sanitizerType %d", t))
}
@@ -76,7 +81,7 @@
type SanitizeProperties struct {
// enable AddressSanitizer, ThreadSanitizer, or UndefinedBehaviorSanitizer
Sanitize struct {
- Never bool `android:"arch_variant"`
+ Never *bool `android:"arch_variant"`
// main sanitizers
Address *bool `android:"arch_variant"`
@@ -120,6 +125,10 @@
androidMkRuntimeLibrary string
}
+func init() {
+ android.RegisterMakeVarsProvider(pctx, cfiMakeVarsProvider)
+}
+
func (sanitize *sanitize) props() []interface{} {
return []interface{}{&sanitize.Properties}
}
@@ -128,12 +137,12 @@
s := &sanitize.Properties.Sanitize
// Don't apply sanitizers to NDK code.
- if ctx.sdk() {
- s.Never = true
+ if ctx.useSdk() {
+ s.Never = BoolPtr(true)
}
// Never always wins.
- if s.Never {
+ if Bool(s.Never) {
return
}
@@ -142,12 +151,12 @@
if ctx.clang() {
if ctx.Host() {
- globalSanitizers = ctx.AConfig().SanitizeHost()
+ globalSanitizers = ctx.Config().SanitizeHost()
} else {
- arches := ctx.AConfig().SanitizeDeviceArch()
+ arches := ctx.Config().SanitizeDeviceArch()
if len(arches) == 0 || inList(ctx.Arch().ArchType.Name, arches) {
- globalSanitizers = ctx.AConfig().SanitizeDevice()
- globalSanitizersDiag = ctx.AConfig().SanitizeDeviceDiag()
+ globalSanitizers = ctx.Config().SanitizeDevice()
+ globalSanitizersDiag = ctx.Config().SanitizeDeviceDiag()
}
}
}
@@ -185,11 +194,13 @@
}
if found, globalSanitizers = removeFromList("cfi", globalSanitizers); found && s.Cfi == nil {
- s.Cfi = boolPtr(true)
+ if !ctx.Config().CFIDisabledForPath(ctx.ModuleDir()) {
+ s.Cfi = boolPtr(true)
+ }
}
if found, globalSanitizers = removeFromList("integer_overflow", globalSanitizers); found && s.Integer_overflow == nil {
- if !ctx.AConfig().IntegerOverflowDisabledForPath(ctx.ModuleDir()) {
+ if !ctx.Config().IntegerOverflowDisabledForPath(ctx.ModuleDir()) {
s.Integer_overflow = boolPtr(true)
}
}
@@ -203,13 +214,26 @@
s.Diag.Integer_overflow = boolPtr(true)
}
+ if found, globalSanitizersDiag = removeFromList("cfi", globalSanitizersDiag); found &&
+ s.Diag.Cfi == nil && Bool(s.Cfi) {
+ s.Diag.Cfi = boolPtr(true)
+ }
+
if len(globalSanitizersDiag) > 0 {
ctx.ModuleErrorf("unknown global sanitizer diagnostics option %s", globalSanitizersDiag[0])
}
}
+ // Enable CFI for all components in the include paths
+ if s.Cfi == nil && ctx.Config().CFIEnabledForPath(ctx.ModuleDir()) {
+ s.Cfi = boolPtr(true)
+ if inList("cfi", ctx.Config().SanitizeDeviceDiag()) {
+ s.Diag.Cfi = boolPtr(true)
+ }
+ }
+
// CFI needs gold linker, and mips toolchain does not have one.
- if !ctx.AConfig().EnableCFI() || ctx.Arch().ArchType == android.Mips || ctx.Arch().ArchType == android.Mips64 {
+ if !ctx.Config().EnableCFI() || ctx.Arch().ArchType == android.Mips || ctx.Arch().ArchType == android.Mips64 {
s.Cfi = nil
s.Diag.Cfi = nil
}
@@ -226,6 +250,12 @@
s.Diag.Cfi = nil
}
+ // Also disable CFI for host builds.
+ if ctx.Host() {
+ s.Cfi = nil
+ s.Diag.Cfi = nil
+ }
+
if ctx.staticBinary() {
s.Address = nil
s.Coverage = nil
@@ -253,6 +283,8 @@
ctx.ModuleErrorf(`Use of "coverage" also requires "address"`)
}
}
+
+ cfiExportsMap = android.PathForSource(ctx, cfiExportsMapPath)
}
func (sanitize *sanitize) deps(ctx BaseModuleContext, deps Deps) Deps {
@@ -264,9 +296,6 @@
if Bool(sanitize.Properties.Sanitize.Address) {
deps.StaticLibs = append(deps.StaticLibs, asanLibs...)
}
- if Bool(sanitize.Properties.Sanitize.Address) || Bool(sanitize.Properties.Sanitize.Thread) {
- deps.SharedLibs = append(deps.SharedLibs, "libdl")
- }
}
return deps
@@ -333,7 +362,6 @@
if ctx.Host() {
// -nodefaultlibs (provided with libc++) prevents the driver from linking
// libraries needed with -fsanitize=address. http://b/18650275 (WAI)
- flags.LdFlags = append(flags.LdFlags, "-lm", "-lpthread")
flags.LdFlags = append(flags.LdFlags, "-Wl,--no-as-needed")
} else {
flags.CFlags = append(flags.CFlags, "-mllvm", "-asan-globals=0")
@@ -346,8 +374,12 @@
diagSanitizers = append(diagSanitizers, "address")
}
+ if Bool(sanitize.Properties.Sanitize.Thread) {
+ sanitizers = append(sanitizers, "thread")
+ }
+
if Bool(sanitize.Properties.Sanitize.Coverage) {
- flags.CFlags = append(flags.CFlags, "-fsanitize-coverage=trace-pc-guard")
+ flags.CFlags = append(flags.CFlags, "-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp")
}
if Bool(sanitize.Properties.Sanitize.Safestack) {
@@ -359,17 +391,25 @@
// __cfi_check needs to be built as Thumb (see the code in linker_cfi.cpp). LLVM is not set up
// to do this on a function basis, so force Thumb on the entire module.
flags.RequiredInstructionSet = "thumb"
- // Workaround for b/33678192. CFI jumptables need Thumb2 codegen. Revert when
- // Clang is updated past r290384.
- flags.LdFlags = append(flags.LdFlags, "-march=armv7-a")
}
sanitizers = append(sanitizers, "cfi")
+
flags.CFlags = append(flags.CFlags, cfiCflags...)
+ // Only append the default visibility flag if -fvisibility has not already been set
+ // to hidden.
+ if !inList("-fvisibility=hidden", flags.CFlags) {
+ flags.CFlags = append(flags.CFlags, "-fvisibility=default")
+ }
flags.LdFlags = append(flags.LdFlags, cfiLdflags...)
flags.ArFlags = append(flags.ArFlags, cfiArflags...)
if Bool(sanitize.Properties.Sanitize.Diag.Cfi) {
diagSanitizers = append(diagSanitizers, "cfi")
}
+
+ if ctx.staticBinary() {
+ _, flags.CFlags = removeFromList("-fsanitize-cfi-cross-dso", flags.CFlags)
+ _, flags.LdFlags = removeFromList("-fsanitize-cfi-cross-dso", flags.LdFlags)
+ }
}
if Bool(sanitize.Properties.Sanitize.Integer_overflow) {
@@ -390,10 +430,6 @@
if ctx.Host() {
flags.CFlags = append(flags.CFlags, "-fno-sanitize-recover=all")
flags.LdFlags = append(flags.LdFlags, sanitizeArg)
- if ctx.Os() == android.Linux {
- flags.LdFlags = append(flags.LdFlags, "-lrt")
- }
- flags.LdFlags = append(flags.LdFlags, "-ldl")
// Host sanitizers only link symbols in the final executable, so
// there will always be undefined symbols in intermediate libraries.
_, flags.LdFlags = removeFromList("-Wl,--no-undefined", flags.LdFlags)
@@ -416,6 +452,8 @@
runtimeLibrary := ""
if Bool(sanitize.Properties.Sanitize.Address) {
runtimeLibrary = config.AddressSanitizerRuntimeLibrary(ctx.toolchain())
+ } else if Bool(sanitize.Properties.Sanitize.Thread) {
+ runtimeLibrary = config.ThreadSanitizerRuntimeLibrary(ctx.toolchain())
} else if len(diagSanitizers) > 0 {
runtimeLibrary = config.UndefinedBehaviorSanitizerRuntimeLibrary(ctx.toolchain())
}
@@ -429,7 +467,7 @@
// When linking against VNDK, use the vendor variant of the runtime lib
sanitize.androidMkRuntimeLibrary = sanitize.runtimeLibrary
- if ctx.vndk() {
+ if ctx.useVndk() {
sanitize.androidMkRuntimeLibrary = sanitize.runtimeLibrary + vendorSuffix
}
}
@@ -444,31 +482,33 @@
}
func (sanitize *sanitize) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
if sanitize.androidMkRuntimeLibrary != "" {
fmt.Fprintln(w, "LOCAL_SHARED_LIBRARIES += "+sanitize.androidMkRuntimeLibrary)
}
-
- return nil
})
+
+ // Add a suffix for CFI-enabled static libraries to allow surfacing both to make without a
+ // name conflict.
+ if ret.Class == "STATIC_LIBRARIES" && Bool(sanitize.Properties.Sanitize.Cfi) {
+ ret.SubName += ".cfi"
+ }
}
func (sanitize *sanitize) inSanitizerDir() bool {
return sanitize.Properties.InSanitizerDir
}
-func (sanitize *sanitize) Sanitizer(t sanitizerType) bool {
- if sanitize == nil {
- return false
- }
-
+func (sanitize *sanitize) getSanitizerBoolPtr(t sanitizerType) *bool {
switch t {
case asan:
- return Bool(sanitize.Properties.Sanitize.Address)
+ return sanitize.Properties.Sanitize.Address
case tsan:
- return Bool(sanitize.Properties.Sanitize.Thread)
+ return sanitize.Properties.Sanitize.Thread
case intOverflow:
- return Bool(sanitize.Properties.Sanitize.Integer_overflow)
+ return sanitize.Properties.Sanitize.Integer_overflow
+ case cfi:
+ return sanitize.Properties.Sanitize.Cfi
default:
panic(fmt.Errorf("unknown sanitizerType %d", t))
}
@@ -485,6 +525,9 @@
sanitize.Properties.Sanitize.Thread = boolPtr(b)
case intOverflow:
sanitize.Properties.Sanitize.Integer_overflow = boolPtr(b)
+ case cfi:
+ sanitize.Properties.Sanitize.Cfi = boolPtr(b)
+ sanitize.Properties.Sanitize.Diag.Cfi = boolPtr(b)
default:
panic(fmt.Errorf("unknown sanitizerType %d", t))
}
@@ -493,43 +536,120 @@
}
}
+// Check if the sanitizer is explicitly disabled (as opposed to nil by
+// virtue of not being set).
+func (sanitize *sanitize) isSanitizerExplicitlyDisabled(t sanitizerType) bool {
+ if sanitize == nil {
+ return false
+ }
+
+ sanitizerVal := sanitize.getSanitizerBoolPtr(t)
+ return sanitizerVal != nil && *sanitizerVal == false
+}
+
+// There isn't an analog of the method above (ie:isSanitizerExplicitlyEnabled)
+// because enabling a sanitizer either directly (via the blueprint) or
+// indirectly (via a mutator) sets the bool ptr to true, and you can't
+// distinguish between the cases. It isn't needed though - both cases can be
+// treated identically.
+func (sanitize *sanitize) isSanitizerEnabled(t sanitizerType) bool {
+ if sanitize == nil {
+ return false
+ }
+
+ sanitizerVal := sanitize.getSanitizerBoolPtr(t)
+ return sanitizerVal != nil && *sanitizerVal == true
+}
+
// Propagate asan requirements down from binaries
func sanitizerDepsMutator(t sanitizerType) func(android.TopDownMutatorContext) {
return func(mctx android.TopDownMutatorContext) {
- if c, ok := mctx.Module().(*Module); ok && c.sanitize.Sanitizer(t) {
- mctx.VisitDepsDepthFirst(func(module blueprint.Module) {
- if d, ok := mctx.Module().(*Module); ok && c.sanitize != nil &&
- !c.sanitize.Properties.Sanitize.Never {
- d.sanitize.Properties.SanitizeDep = true
+ if c, ok := mctx.Module().(*Module); ok && c.sanitize.isSanitizerEnabled(t) {
+ mctx.VisitDepsDepthFirst(func(module android.Module) {
+ if d, ok := module.(*Module); ok && d.sanitize != nil &&
+ !Bool(d.sanitize.Properties.Sanitize.Never) &&
+ !d.sanitize.isSanitizerExplicitlyDisabled(t) {
+ if (t == cfi && d.static()) || t != cfi {
+ d.sanitize.Properties.SanitizeDep = true
+ }
}
})
}
}
}
-// Create asan variants for modules that need them
+// Create sanitized variants for modules that need them
func sanitizerMutator(t sanitizerType) func(android.BottomUpMutatorContext) {
return func(mctx android.BottomUpMutatorContext) {
if c, ok := mctx.Module().(*Module); ok && c.sanitize != nil {
- if c.isDependencyRoot() && c.sanitize.Sanitizer(t) {
+ if c.isDependencyRoot() && c.sanitize.isSanitizerEnabled(t) {
modules := mctx.CreateVariations(t.String())
modules[0].(*Module).sanitize.SetSanitizer(t, true)
- } else if c.sanitize.Properties.SanitizeDep {
+ } else if c.sanitize.isSanitizerEnabled(t) || c.sanitize.Properties.SanitizeDep {
+ // Save original sanitizer status before we assign values to variant
+ // 0 as that overwrites the original.
+ isSanitizerEnabled := c.sanitize.isSanitizerEnabled(t)
+
modules := mctx.CreateVariations("", t.String())
modules[0].(*Module).sanitize.SetSanitizer(t, false)
modules[1].(*Module).sanitize.SetSanitizer(t, true)
+
modules[0].(*Module).sanitize.Properties.SanitizeDep = false
modules[1].(*Module).sanitize.Properties.SanitizeDep = false
- if mctx.Device() {
- modules[1].(*Module).sanitize.Properties.InSanitizerDir = true
- } else {
- modules[0].(*Module).Properties.PreventInstall = true
- }
- if mctx.AConfig().EmbeddedInMake() {
- modules[0].(*Module).Properties.HideFromMake = true
+
+ // We don't need both variants active for anything but CFI-enabled
+ // target static libraries, so suppress the appropriate variant in
+ // all other cases.
+ if t == cfi {
+ if c.static() {
+ if !mctx.Device() {
+ if isSanitizerEnabled {
+ modules[0].(*Module).Properties.PreventInstall = true
+ modules[0].(*Module).Properties.HideFromMake = true
+ } else {
+ modules[1].(*Module).Properties.PreventInstall = true
+ modules[1].(*Module).Properties.HideFromMake = true
+ }
+ } else {
+ cfiStaticLibs := cfiStaticLibs(mctx.Config())
+
+ cfiStaticLibsMutex.Lock()
+ *cfiStaticLibs = append(*cfiStaticLibs, c.Name())
+ cfiStaticLibsMutex.Unlock()
+ }
+ } else {
+ modules[0].(*Module).Properties.PreventInstall = true
+ modules[0].(*Module).Properties.HideFromMake = true
+ }
+ } else if t == asan {
+ if mctx.Device() {
+ // CFI and ASAN are currently mutually exclusive so disable
+ // CFI if this is an ASAN variant.
+ modules[1].(*Module).sanitize.Properties.InSanitizerDir = true
+ modules[1].(*Module).sanitize.SetSanitizer(cfi, false)
+ }
+ if isSanitizerEnabled {
+ modules[0].(*Module).Properties.PreventInstall = true
+ modules[0].(*Module).Properties.HideFromMake = true
+ } else {
+ modules[1].(*Module).Properties.PreventInstall = true
+ modules[1].(*Module).Properties.HideFromMake = true
+ }
}
}
c.sanitize.Properties.SanitizeDep = false
}
}
}
+
+func cfiStaticLibs(config android.Config) *[]string {
+ return config.Once("cfiStaticLibs", func() interface{} {
+ return &[]string{}
+ }).(*[]string)
+}
+
+func cfiMakeVarsProvider(ctx android.MakeVarsContext) {
+ cfiStaticLibs := cfiStaticLibs(ctx.Config())
+ sort.Strings(*cfiStaticLibs)
+ ctx.Strict("SOONG_CFI_STATIC_LIBRARIES", strings.Join(*cfiStaticLibs, " "))
+}
diff --git a/cc/stl.go b/cc/stl.go
index 65d7e40..347db99 100644
--- a/cc/stl.go
+++ b/cc/stl.go
@@ -42,7 +42,7 @@
if stl.Properties.Stl != nil {
s = *stl.Properties.Stl
}
- if ctx.sdk() && ctx.Device() {
+ if ctx.useSdk() && ctx.Device() {
switch s {
case "":
return "ndk_system"
@@ -107,8 +107,6 @@
}
if ctx.staticBinary() {
deps.StaticLibs = append(deps.StaticLibs, "libm", "libc", "libdl")
- } else {
- deps.SharedLibs = append(deps.SharedLibs, "libdl")
}
}
case "":
@@ -118,15 +116,9 @@
// The system STL doesn't have a prebuilt (it uses the system's libstdc++), but it does have
// its own includes. The includes are handled in CCBase.Flags().
deps.SharedLibs = append([]string{"libstdc++"}, deps.SharedLibs...)
- case "ndk_libc++_shared":
- deps.SharedLibs = append(deps.SharedLibs, stl.Properties.SelectedStl,
- "libdl")
- case "ndk_libc++_static":
- deps.StaticLibs = append(deps.StaticLibs, stl.Properties.SelectedStl)
- deps.SharedLibs = append(deps.SharedLibs, "libdl")
- case "ndk_libstlport_shared":
+ case "ndk_libc++_shared", "ndk_libstlport_shared":
deps.SharedLibs = append(deps.SharedLibs, stl.Properties.SelectedStl)
- case "ndk_libstlport_static", "ndk_libgnustl_static":
+ case "ndk_libc++_static", "ndk_libstlport_static", "ndk_libgnustl_static":
deps.StaticLibs = append(deps.StaticLibs, stl.Properties.SelectedStl)
default:
panic(fmt.Errorf("Unknown stl: %q", stl.Properties.SelectedStl))
@@ -142,7 +134,6 @@
if !ctx.toolchain().Bionic() {
flags.CppFlags = append(flags.CppFlags, "-nostdinc++")
flags.LdFlags = append(flags.LdFlags, "-nodefaultlibs")
- flags.LdFlags = append(flags.LdFlags, "-lpthread", "-lm")
if ctx.staticBinary() {
flags.LdFlags = append(flags.LdFlags, hostStaticGccLibs[ctx.Os()]...)
} else {
diff --git a/cc/strip.go b/cc/strip.go
index bc16bbc..a7c2d4e 100644
--- a/cc/strip.go
+++ b/cc/strip.go
@@ -14,12 +14,14 @@
package cc
-import "android/soong/android"
+import (
+ "android/soong/android"
+)
type StripProperties struct {
Strip struct {
- None bool
- Keep_symbols bool
+ None *bool
+ Keep_symbols *bool
}
}
@@ -28,7 +30,7 @@
}
func (stripper *stripper) needsStrip(ctx ModuleContext) bool {
- return !ctx.AConfig().EmbeddedInMake() && !stripper.StripProperties.Strip.None
+ return !ctx.Config().EmbeddedInMake() && !Bool(stripper.StripProperties.Strip.None)
}
func (stripper *stripper) strip(ctx ModuleContext, in, out android.ModuleOutPath,
@@ -36,7 +38,7 @@
if ctx.Darwin() {
TransformDarwinStrip(ctx, in, out)
} else {
- flags.stripKeepSymbols = stripper.StripProperties.Strip.Keep_symbols
+ flags.stripKeepSymbols = Bool(stripper.StripProperties.Strip.Keep_symbols)
// TODO(ccross): don't add gnu debuglink for user builds
flags.stripAddGnuDebuglink = true
TransformStrip(ctx, in, out, flags)
diff --git a/cc/test.go b/cc/test.go
index a52e94a..7b9dcb9 100644
--- a/cc/test.go
+++ b/cc/test.go
@@ -43,43 +43,43 @@
// list of compatibility suites (for example "cts", "vts") that the module should be
// installed into.
- Test_suites []string
+ Test_suites []string `android:"arch_variant"`
}
func init() {
- android.RegisterModuleType("cc_test", testFactory)
- android.RegisterModuleType("cc_test_library", testLibraryFactory)
- android.RegisterModuleType("cc_benchmark", benchmarkFactory)
- android.RegisterModuleType("cc_test_host", testHostFactory)
- android.RegisterModuleType("cc_benchmark_host", benchmarkHostFactory)
+ android.RegisterModuleType("cc_test", TestFactory)
+ android.RegisterModuleType("cc_test_library", TestLibraryFactory)
+ android.RegisterModuleType("cc_benchmark", BenchmarkFactory)
+ android.RegisterModuleType("cc_test_host", TestHostFactory)
+ android.RegisterModuleType("cc_benchmark_host", BenchmarkHostFactory)
}
// Module factory for tests
-func testFactory() android.Module {
+func TestFactory() android.Module {
module := NewTest(android.HostAndDeviceSupported)
return module.Init()
}
// Module factory for test libraries
-func testLibraryFactory() android.Module {
+func TestLibraryFactory() android.Module {
module := NewTestLibrary(android.HostAndDeviceSupported)
return module.Init()
}
// Module factory for benchmarks
-func benchmarkFactory() android.Module {
+func BenchmarkFactory() android.Module {
module := NewBenchmark(android.HostAndDeviceSupported)
return module.Init()
}
// Module factory for host tests
-func testHostFactory() android.Module {
+func TestHostFactory() android.Module {
module := NewTest(android.HostSupported)
return module.Init()
}
// Module factory for host benchmarks
-func benchmarkHostFactory() android.Module {
+func BenchmarkHostFactory() android.Module {
module := NewBenchmark(android.HostSupported)
return module.Init()
}
@@ -100,7 +100,7 @@
func (test *testBinary) setSrc(name, src string) {
test.baseCompiler.Properties.Srcs = []string{src}
- test.binaryDecorator.Properties.Stem = name
+ test.binaryDecorator.Properties.Stem = StringPtr(name)
}
var _ testPerSrc = (*testBinary)(nil)
@@ -145,10 +145,8 @@
flags.CFlags = append(flags.CFlags, "-DGTEST_OS_WINDOWS")
case android.Linux:
flags.CFlags = append(flags.CFlags, "-DGTEST_OS_LINUX")
- flags.LdFlags = append(flags.LdFlags, "-lpthread")
case android.Darwin:
flags.CFlags = append(flags.CFlags, "-DGTEST_OS_MAC")
- flags.LdFlags = append(flags.LdFlags, "-lpthread")
}
} else {
flags.CFlags = append(flags.CFlags, "-DGTEST_OS_LINUX_ANDROID")
@@ -159,7 +157,7 @@
func (test *testDecorator) linkerDeps(ctx BaseModuleContext, deps Deps) Deps {
if test.gtest() {
- if ctx.sdk() && ctx.Device() {
+ if ctx.useSdk() && ctx.Device() {
switch ctx.selectedStl() {
case "ndk_libc++_shared", "ndk_libc++_static":
deps.StaticLibs = append(deps.StaticLibs, "libgtest_main_ndk_libcxx", "libgtest_ndk_libcxx")
@@ -177,11 +175,16 @@
}
func (test *testDecorator) linkerInit(ctx BaseModuleContext, linker *baseLinker) {
+ // add ../../lib[64] to rpath so that out/host/linux-x86/nativetest/<test dir>/<test> can
+ // find out/host/linux-x86/lib[64]/library.so
runpath := "../../lib"
if ctx.toolchain().Is64Bit() {
runpath += "64"
}
linker.dynamicProperties.RunPaths = append(linker.dynamicProperties.RunPaths, runpath)
+
+ // add "" to rpath so that test binaries can find libraries in their own test directory
+ linker.dynamicProperties.RunPaths = append(linker.dynamicProperties.RunPaths, "")
}
func (test *testDecorator) linkerProps() []interface{} {
@@ -233,7 +236,7 @@
if !Bool(test.Properties.No_named_install_directory) {
test.binaryDecorator.baseInstaller.relative = ctx.ModuleName()
- } else if test.binaryDecorator.baseInstaller.Properties.Relative_install_path == "" {
+ } else if String(test.binaryDecorator.baseInstaller.Properties.Relative_install_path) == "" {
ctx.PropertyErrorf("no_named_install_directory", "Module install directory may only be disabled if relative_install_path is set")
}
@@ -337,8 +340,8 @@
func (benchmark *benchmarkDecorator) install(ctx ModuleContext, file android.Path) {
benchmark.data = ctx.ExpandSources(benchmark.Properties.Data, nil)
- benchmark.binaryDecorator.baseInstaller.dir = filepath.Join("nativetest", ctx.ModuleName())
- benchmark.binaryDecorator.baseInstaller.dir64 = filepath.Join("nativetest64", ctx.ModuleName())
+ benchmark.binaryDecorator.baseInstaller.dir = filepath.Join("benchmarktest", ctx.ModuleName())
+ benchmark.binaryDecorator.baseInstaller.dir64 = filepath.Join("benchmarktest64", ctx.ModuleName())
benchmark.binaryDecorator.baseInstaller.install(ctx, file)
}
@@ -355,7 +358,7 @@
module, binary := NewBinary(hod)
module.multilib = android.MultilibBoth
- binary.baseInstaller = NewTestInstaller()
+ binary.baseInstaller = NewBaseInstaller("benchmarktest", "benchmarktest64", InstallInData)
benchmark := &benchmarkDecorator{
binaryDecorator: binary,
diff --git a/cc/test_data_test.go b/cc/test_data_test.go
index 962bde5..434edcd 100644
--- a/cc/test_data_test.go
+++ b/cc/test_data_test.go
@@ -117,7 +117,7 @@
}
defer os.RemoveAll(buildDir)
- config := android.TestConfig(buildDir)
+ config := android.TestConfig(buildDir, nil)
for _, test := range testDataTests {
t.Run(test.name, func(t *testing.T) {
diff --git a/cc/test_gen_stub_libs.py b/cc/test_gen_stub_libs.py
index 4df6cf8..b20a5c7 100755
--- a/cc/test_gen_stub_libs.py
+++ b/cc/test_gen_stub_libs.py
@@ -430,6 +430,8 @@
gsl.Version('VERSION_1', None, [], [
gsl.Symbol('foo', []),
gsl.Symbol('bar', ['var']),
+ gsl.Symbol('woodly', ['weak']),
+ gsl.Symbol('doodly', ['weak', 'var']),
]),
gsl.Version('VERSION_2', 'VERSION_1', [], [
gsl.Symbol('baz', []),
@@ -443,6 +445,8 @@
expected_src = textwrap.dedent("""\
void foo() {}
int bar = 0;
+ __attribute__((weak)) void woodly() {}
+ __attribute__((weak)) int doodly = 0;
void baz() {}
void qux() {}
""")
@@ -453,6 +457,8 @@
global:
foo;
bar;
+ woodly;
+ doodly;
};
VERSION_2 {
global:
diff --git a/cc/tidy.go b/cc/tidy.go
index c31f5ae..6d7c957 100644
--- a/cc/tidy.go
+++ b/cc/tidy.go
@@ -58,7 +58,7 @@
}
// If not explicitly set, check the global tidy flag
- if tidy.Properties.Tidy == nil && !ctx.AConfig().ClangTidy() {
+ if tidy.Properties.Tidy == nil && !ctx.Config().ClangTidy() {
return flags
}
@@ -82,7 +82,7 @@
flags.TidyFlags = append(flags.TidyFlags, "-extra-arg-before=-D__clang_analyzer__")
tidyChecks := "-checks="
- if checks := ctx.AConfig().TidyChecks(); len(checks) > 0 {
+ if checks := ctx.Config().TidyChecks(); len(checks) > 0 {
tidyChecks += checks
} else {
tidyChecks += config.TidyChecksForDir(ctx.ModuleDir())
diff --git a/cc/toolchain_library.go b/cc/toolchain_library.go
index 9b83f18..2bb4018 100644
--- a/cc/toolchain_library.go
+++ b/cc/toolchain_library.go
@@ -15,8 +15,6 @@
package cc
import (
- "github.com/google/blueprint/proptools"
-
"android/soong/android"
)
@@ -45,7 +43,7 @@
}
module.compiler = toolchainLibrary
module.linker = toolchainLibrary
- module.Properties.Clang = proptools.BoolPtr(false)
+ module.Properties.Clang = BoolPtr(false)
module.stl = nil
module.sanitize = nil
module.installer = nil
diff --git a/cc/util.go b/cc/util.go
index eeb64eb..cc89af6 100644
--- a/cc/util.go
+++ b/cc/util.go
@@ -131,3 +131,10 @@
}
return list
}
+
+func addSuffix(list []string, suffix string) []string {
+ for i := range list {
+ list[i] = list[i] + suffix
+ }
+ return list
+}
diff --git a/cc/vndk.go b/cc/vndk.go
index fd1bdcb..a61b74c 100644
--- a/cc/vndk.go
+++ b/cc/vndk.go
@@ -15,6 +15,10 @@
package cc
import (
+ "sort"
+ "strings"
+ "sync"
+
"android/soong/android"
)
@@ -23,8 +27,8 @@
// declared as a VNDK or VNDK-SP module. The vendor variant
// will be installed in /system instead of /vendor partition.
//
- // `vendor_available: true` must set to together for VNDK
- // modules.
+ // `vendor_vailable` must be explicitly set to either true or
+ // false together with `vndk: {enabled: true}`.
Enabled *bool
// declared as a VNDK-SP module, which is a subset of VNDK.
@@ -77,6 +81,24 @@
if to.linker == nil {
return
}
+ if !vndk.isVndk() {
+ // Non-VNDK modules (those installed to /vendor) can't depend on modules marked with
+ // vendor_available: false.
+ violation := false
+ if lib, ok := to.linker.(*llndkStubDecorator); ok && !Bool(lib.Properties.Vendor_available) {
+ violation = true
+ } else {
+ if _, ok := to.linker.(libraryInterface); ok && to.VendorProperties.Vendor_available != nil && !Bool(to.VendorProperties.Vendor_available) {
+ // Vendor_available == nil && !Bool(Vendor_available) should be okay since
+ // it means a vendor-only library which is a valid dependency for non-VNDK
+ // modules.
+ violation = true
+ }
+ }
+ if violation {
+ ctx.ModuleErrorf("Vendor module that is not VNDK should not link to %q which is marked as `vendor_available: false`", to.Name())
+ }
+ }
if lib, ok := to.linker.(*libraryDecorator); !ok || !lib.shared() {
// Check only shared libraries.
// Other (static and LL-NDK) libraries are allowed to link.
@@ -96,3 +118,59 @@
return
}
}
+
+var (
+ vndkCoreLibraries []string
+ vndkSpLibraries []string
+ llndkLibraries []string
+ vndkPrivateLibraries []string
+ vndkLibrariesLock sync.Mutex
+)
+
+// gather list of vndk-core, vndk-sp, and ll-ndk libs
+func vndkMutator(mctx android.BottomUpMutatorContext) {
+ if m, ok := mctx.Module().(*Module); ok {
+ if lib, ok := m.linker.(*llndkStubDecorator); ok {
+ vndkLibrariesLock.Lock()
+ defer vndkLibrariesLock.Unlock()
+ name := strings.TrimSuffix(m.Name(), llndkLibrarySuffix)
+ if !inList(name, llndkLibraries) {
+ llndkLibraries = append(llndkLibraries, name)
+ sort.Strings(llndkLibraries)
+ }
+ if !Bool(lib.Properties.Vendor_available) {
+ if !inList(name, vndkPrivateLibraries) {
+ vndkPrivateLibraries = append(vndkPrivateLibraries, name)
+ sort.Strings(vndkPrivateLibraries)
+ }
+ }
+ } else {
+ lib, is_lib := m.linker.(*libraryDecorator)
+ prebuilt_lib, is_prebuilt_lib := m.linker.(*prebuiltLibraryLinker)
+ if (is_lib && lib.shared()) || (is_prebuilt_lib && prebuilt_lib.shared()) {
+ name := strings.TrimPrefix(m.Name(), "prebuilt_")
+ if m.vndkdep.isVndk() {
+ vndkLibrariesLock.Lock()
+ defer vndkLibrariesLock.Unlock()
+ if m.vndkdep.isVndkSp() {
+ if !inList(name, vndkSpLibraries) {
+ vndkSpLibraries = append(vndkSpLibraries, name)
+ sort.Strings(vndkSpLibraries)
+ }
+ } else {
+ if !inList(name, vndkCoreLibraries) {
+ vndkCoreLibraries = append(vndkCoreLibraries, name)
+ sort.Strings(vndkCoreLibraries)
+ }
+ }
+ if !Bool(m.VendorProperties.Vendor_available) {
+ if !inList(name, vndkPrivateLibraries) {
+ vndkPrivateLibraries = append(vndkPrivateLibraries, name)
+ sort.Strings(vndkPrivateLibraries)
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/cc/vndk_prebuilt.go b/cc/vndk_prebuilt.go
new file mode 100644
index 0000000..9ccab03
--- /dev/null
+++ b/cc/vndk_prebuilt.go
@@ -0,0 +1,140 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "strings"
+
+ "android/soong/android"
+)
+
+var (
+ vndkSuffix = ".vndk."
+)
+
+// Creates vndk prebuilts that include the VNDK version.
+//
+// Example:
+//
+// vndk_prebuilt_shared {
+// name: "libfoo",
+// version: "27.1.0",
+// vendor_available: true,
+// vndk: {
+// enabled: true,
+// },
+// export_include_dirs: ["include/external/libfoo/vndk_include"],
+// arch: {
+// arm64: {
+// srcs: ["arm/lib64/libfoo.so"],
+// },
+// arm: {
+// srcs: ["arm/lib/libfoo.so"],
+// },
+// },
+// }
+//
+type vndkPrebuiltProperties struct {
+ // VNDK snapshot version that is formated as {SDK_ver}.{Major}.{Minor}.
+ Version string
+
+ // Prebuilt files for each arch.
+ Srcs []string `android:"arch_variant"`
+}
+
+type vndkPrebuiltLibraryDecorator struct {
+ *libraryDecorator
+ properties vndkPrebuiltProperties
+}
+
+func (p *vndkPrebuiltLibraryDecorator) Name(name string) string {
+ return name + vndkSuffix + p.version()
+}
+
+func (p *vndkPrebuiltLibraryDecorator) version() string {
+ return p.properties.Version
+}
+
+func (p *vndkPrebuiltLibraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags {
+ p.libraryDecorator.libName = strings.TrimSuffix(ctx.ModuleName(), vndkSuffix+p.version())
+ return p.libraryDecorator.linkerFlags(ctx, flags)
+}
+
+func (p *vndkPrebuiltLibraryDecorator) singleSourcePath(ctx ModuleContext) android.Path {
+ if len(p.properties.Srcs) == 0 {
+ ctx.PropertyErrorf("srcs", "missing prebuilt source file")
+ return nil
+ }
+
+ if len(p.properties.Srcs) > 1 {
+ ctx.PropertyErrorf("srcs", "multiple prebuilt source files")
+ return nil
+ }
+
+ return android.PathForModuleSrc(ctx, p.properties.Srcs[0])
+}
+
+func (p *vndkPrebuiltLibraryDecorator) link(ctx ModuleContext,
+ flags Flags, deps PathDeps, objs Objects) android.Path {
+ if len(p.properties.Srcs) > 0 && p.shared() {
+ // current VNDK prebuilts are only shared libs.
+ return p.singleSourcePath(ctx)
+ }
+ return nil
+}
+
+func (p *vndkPrebuiltLibraryDecorator) install(ctx ModuleContext, file android.Path) {
+ if p.shared() {
+ if ctx.Device() && ctx.useVndk() {
+ if ctx.isVndkSp() {
+ p.baseInstaller.subDir = "vndk-sp-" + p.version()
+ } else if ctx.isVndk() {
+ p.baseInstaller.subDir = "vndk-" + p.version()
+ }
+ }
+ p.baseInstaller.install(ctx, file)
+ }
+}
+
+func vndkPrebuiltSharedLibrary() *Module {
+ module, library := NewLibrary(android.DeviceSupported)
+ library.BuildOnlyShared()
+ module.stl = nil
+ module.sanitize = nil
+ library.StripProperties.Strip.None = BoolPtr(true)
+
+ prebuilt := &vndkPrebuiltLibraryDecorator{
+ libraryDecorator: library,
+ }
+
+ module.compiler = nil
+ module.linker = prebuilt
+ module.installer = prebuilt
+
+ module.AddProperties(
+ &prebuilt.properties,
+ )
+
+ return module
+}
+
+func vndkPrebuiltSharedFactory() android.Module {
+ module := vndkPrebuiltSharedLibrary()
+ return module.Init()
+}
+
+func init() {
+ android.RegisterModuleType("vndk_prebuilt_shared", vndkPrebuiltSharedFactory)
+}
diff --git a/cmd/soong_zip/Android.bp b/cmd/extract_linker/Android.bp
similarity index 76%
copy from cmd/soong_zip/Android.bp
copy to cmd/extract_linker/Android.bp
index 10896ce..fe76ae4 100644
--- a/cmd/soong_zip/Android.bp
+++ b/cmd/extract_linker/Android.bp
@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All rights reserved.
+// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -13,10 +13,8 @@
// limitations under the License.
blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
- srcs: [
- "soong_zip.go",
- "rate_limit.go",
- ],
+ name: "extract_linker",
+ srcs: ["main.go"],
+ testSrcs: ["main_test.go"],
}
+
diff --git a/cmd/extract_linker/main.go b/cmd/extract_linker/main.go
new file mode 100644
index 0000000..8530b4a
--- /dev/null
+++ b/cmd/extract_linker/main.go
@@ -0,0 +1,154 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This tool extracts ELF LOAD segments from our linker binary, and produces an
+// assembly file and linker script which will embed those segments as sections
+// in another binary.
+package main
+
+import (
+ "bytes"
+ "debug/elf"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "text/template"
+)
+
+var linkerScriptTemplate = template.Must(template.New("linker_script").Parse(`
+ENTRY(__dlwrap__start)
+SECTIONS {
+ __dlwrap_original_start = _start;
+ /DISCARD/ : { *(.interp) }
+
+{{range .}}
+ . = {{ printf "0x%x" .Vaddr }};
+ {{.Name}} : { KEEP(*({{.Name}})) }
+{{end}}
+
+ .text : { *(.text .text.*) }
+ .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) }
+ .data : { *(.data .data.* .gnu.linkonce.d.*) }
+ .bss : { *(.dynbss) *(.bss .bss.* .gnu.linkonce.b.*) *(COMMON) }
+}
+`))
+
+type LinkerSection struct {
+ Name string
+ Vaddr uint64
+}
+
+func main() {
+ var asmPath string
+ var scriptPath string
+
+ flag.StringVar(&asmPath, "s", "", "Path to save the assembly file")
+ flag.StringVar(&scriptPath, "T", "", "Path to save the linker script")
+ flag.Parse()
+
+ f, err := os.Open(flag.Arg(0))
+ if err != nil {
+ log.Fatalf("Error opening %q: %v", flag.Arg(0), err)
+ }
+ defer f.Close()
+
+ ef, err := elf.NewFile(f)
+ if err != nil {
+ log.Fatal("Unable to read elf file: %v", err)
+ }
+
+ asm := &bytes.Buffer{}
+
+ fmt.Fprintln(asm, ".globl __dlwrap_linker_entry")
+ fmt.Fprintf(asm, ".set __dlwrap_linker_entry, 0x%x\n\n", ef.Entry)
+
+ baseLoadAddr := uint64(0x1000)
+ sections := []LinkerSection{}
+ load := 0
+ for _, prog := range ef.Progs {
+ if prog.Type != elf.PT_LOAD {
+ continue
+ }
+
+ sectionName := fmt.Sprintf(".linker.sect%d", load)
+ flags := ""
+ if prog.Flags&elf.PF_W != 0 {
+ flags += "w"
+ }
+ if prog.Flags&elf.PF_X != 0 {
+ flags += "x"
+ }
+ fmt.Fprintf(asm, ".section %s, \"a%s\"\n", sectionName, flags)
+
+ if load == 0 {
+ fmt.Fprintln(asm, ".globl __dlwrap_linker_code_start")
+ fmt.Fprintln(asm, "__dlwrap_linker_code_start:")
+ }
+
+ buffer, _ := ioutil.ReadAll(prog.Open())
+ bytesToAsm(asm, buffer)
+
+ // Fill in zeros for any BSS sections. It would be nice to keep
+ // this as a true BSS, but ld/gold isn't preserving those,
+ // instead combining the segments with the following segment,
+ // and BSS only exists at the end of a LOAD segment. The
+ // linker doesn't use a lot of BSS, so this isn't a huge
+ // problem.
+ if prog.Memsz > prog.Filesz {
+ fmt.Fprintf(asm, ".fill 0x%x, 1, 0\n", prog.Memsz-prog.Filesz)
+ }
+ fmt.Fprintln(asm)
+
+ sections = append(sections, LinkerSection{
+ Name: sectionName,
+ Vaddr: baseLoadAddr + prog.Vaddr,
+ })
+
+ load += 1
+ }
+
+ if asmPath != "" {
+ if err := ioutil.WriteFile(asmPath, asm.Bytes(), 0777); err != nil {
+ log.Fatal("Unable to write %q: %v", asmPath, err)
+ }
+ }
+
+ if scriptPath != "" {
+ buf := &bytes.Buffer{}
+ if err := linkerScriptTemplate.Execute(buf, sections); err != nil {
+ log.Fatal("Failed to create linker script: %v", err)
+ }
+ if err := ioutil.WriteFile(scriptPath, buf.Bytes(), 0777); err != nil {
+ log.Fatal("Unable to write %q: %v", scriptPath, err)
+ }
+ }
+}
+
+func bytesToAsm(asm io.Writer, buf []byte) {
+ for i, b := range buf {
+ if i%64 == 0 {
+ if i != 0 {
+ fmt.Fprint(asm, "\n")
+ }
+ fmt.Fprint(asm, ".byte ")
+ } else {
+ fmt.Fprint(asm, ",")
+ }
+ fmt.Fprintf(asm, "%d", b)
+ }
+ fmt.Fprintln(asm)
+}
diff --git a/cmd/extract_linker/main_test.go b/cmd/extract_linker/main_test.go
new file mode 100644
index 0000000..6ac4ec6
--- /dev/null
+++ b/cmd/extract_linker/main_test.go
@@ -0,0 +1,63 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "testing"
+)
+
+var bytesToAsmTestCases = []struct {
+ name string
+ in []byte
+ out string
+}{
+ {
+ name: "empty",
+ in: []byte{},
+ out: "\n",
+ },
+ {
+ name: "short",
+ in: []byte{0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01},
+ out: ".byte 127,69,76,70,2,1\n",
+ },
+ {
+ name: "multiline",
+ in: []byte{0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x30, 0x00, 0x3e, 0x00, 0x01, 0x00, 0x00, 0x00,
+ 0x50, 0x98, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x88, 0xd1, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x38, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x38, 0x00,
+ 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x38, 0x00},
+ out: ".byte 127,69,76,70,2,1,1,0,0,0,0,0,0,0,0,0,48,0,62,0,1,0,0,0,80,152,2,0,0,0,0,0,64,0,0,0,0,0,0,0,136,209,18,0,0,0,0,0,0,0,0,0,64,0,56,0,1,0,0,0,64,0,56,0\n" +
+ ".byte 2,0,0,0,64,0,56,0\n",
+ },
+}
+
+func TestBytesToAsm(t *testing.T) {
+ for _, testcase := range bytesToAsmTestCases {
+ t.Run(testcase.name, func(t *testing.T) {
+ buf := bytes.Buffer{}
+ bytesToAsm(&buf, testcase.in)
+ if buf.String() != testcase.out {
+ t.Errorf("input: %#v\n want: %q\n got: %q\n", testcase.in, testcase.out, buf.String())
+ }
+ })
+ }
+}
diff --git a/cmd/javac_wrapper/javac_wrapper.go b/cmd/javac_wrapper/javac_wrapper.go
index ab4d23f..4df4938 100644
--- a/cmd/javac_wrapper/javac_wrapper.go
+++ b/cmd/javac_wrapper/javac_wrapper.go
@@ -17,6 +17,11 @@
//
// It also hides the unhelpful and unhideable "warning there is a warning"
// messages.
+//
+// Each javac build statement has an order-only dependency on the
+// soong_javac_wrapper tool, which means the javac command will not be rerun
+// if soong_javac_wrapper changes. That means that soong_javac_wrapper must
+// not do anything that will affect the results of the build.
package main
import (
@@ -170,4 +175,5 @@
regexp.MustCompile(`Note: Recompile with -Xlint:deprecation for details.`),
regexp.MustCompile(`Note: (Some input files|.*\.java) uses? unchecked or unsafe operations.`),
regexp.MustCompile(`Note: Recompile with -Xlint:unchecked for details.`),
+ regexp.MustCompile(`bootstrap class path not set in conjunction with -source`),
}
diff --git a/cmd/javac_wrapper/javac_wrapper_test.go b/cmd/javac_wrapper/javac_wrapper_test.go
index c345479..d76793f 100644
--- a/cmd/javac_wrapper/javac_wrapper_test.go
+++ b/cmd/javac_wrapper/javac_wrapper_test.go
@@ -41,8 +41,8 @@
out: "\x1b[1mFile.java:398: \x1b[35mwarning:\x1b[0m\x1b[1m [RectIntersectReturnValueIgnored] Return value of com.blah.function() must be checked\x1b[0m\n",
},
{
- in: "warning: [options] bootstrap class path not set in conjunction with -source 1.7\n",
- out: "\x1b[1m\x1b[35mwarning:\x1b[0m\x1b[1m [options] bootstrap class path not set in conjunction with -source 1.7\x1b[0m\n",
+ in: "warning: [options] blah\n",
+ out: "\x1b[1m\x1b[35mwarning:\x1b[0m\x1b[1m [options] blah\x1b[0m\n",
},
{
in: " (see http://go/errorprone/bugpattern/RectIntersectReturnValueIgnored.md)\n",
@@ -56,6 +56,7 @@
Note: Recompile with -Xlint:unchecked for details.
Note: dir/file.java uses or overrides a deprecated API.
Note: dir/file.java uses unchecked or unsafe operations.
+warning: [options] bootstrap class path not set in conjunction with -source 1.7
`,
out: "\n",
},
diff --git a/cmd/soong_zip/Android.bp b/cmd/merge_zips/Android.bp
similarity index 77%
copy from cmd/soong_zip/Android.bp
copy to cmd/merge_zips/Android.bp
index 10896ce..ace079d 100644
--- a/cmd/soong_zip/Android.bp
+++ b/cmd/merge_zips/Android.bp
@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All rights reserved.
+// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -13,10 +13,13 @@
// limitations under the License.
blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
+ name: "merge_zips",
+ deps: [
+ "android-archive-zip",
+ "soong-jar",
+ ],
srcs: [
- "soong_zip.go",
- "rate_limit.go",
+ "merge_zips.go",
],
}
+
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go
new file mode 100644
index 0000000..556dad0
--- /dev/null
+++ b/cmd/merge_zips/merge_zips.go
@@ -0,0 +1,340 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "errors"
+ "flag"
+ "fmt"
+ "hash/crc32"
+ "log"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "android/soong/jar"
+ "android/soong/third_party/zip"
+)
+
+type fileList []string
+
+func (f *fileList) String() string {
+ return `""`
+}
+
+func (f *fileList) Set(name string) error {
+ *f = append(*f, filepath.Clean(name))
+
+ return nil
+}
+
+type zipsToNotStripSet map[string]bool
+
+func (s zipsToNotStripSet) String() string {
+ return `""`
+}
+
+func (s zipsToNotStripSet) Set(zip_path string) error {
+ s[zip_path] = true
+
+ return nil
+}
+
+var (
+ sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)")
+ emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)")
+ stripDirs fileList
+ stripFiles fileList
+ zipsToNotStrip = make(zipsToNotStripSet)
+ stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file")
+ manifest = flag.String("m", "", "manifest file to insert in jar")
+ ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn")
+)
+
+func init() {
+ flag.Var(&stripDirs, "stripDir", "the prefix of file path to be excluded from the output zip")
+ flag.Var(&stripFiles, "stripFile", "filenames to be excluded from the output zip, accepts wildcards")
+ flag.Var(&zipsToNotStrip, "zipToNotStrip", "the input zip file which is not applicable for stripping")
+}
+
+func main() {
+ flag.Usage = func() {
+ fmt.Fprintln(os.Stderr, "usage: merge_zips [-jsD] [-m manifest] output [inputs...]")
+ flag.PrintDefaults()
+ }
+
+ // parse args
+ flag.Parse()
+ args := flag.Args()
+ if len(args) < 1 {
+ flag.Usage()
+ os.Exit(1)
+ }
+ outputPath := args[0]
+ inputs := args[1:]
+
+ log.SetFlags(log.Lshortfile)
+
+ // make writer
+ output, err := os.Create(outputPath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer output.Close()
+ writer := zip.NewWriter(output)
+ defer func() {
+ err := writer.Close()
+ if err != nil {
+ log.Fatal(err)
+ }
+ }()
+
+ // make readers
+ readers := []namedZipReader{}
+ for _, input := range inputs {
+ reader, err := zip.OpenReader(input)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer reader.Close()
+ namedReader := namedZipReader{path: input, reader: reader}
+ readers = append(readers, namedReader)
+ }
+
+ if *manifest != "" && !*emulateJar {
+ log.Fatal(errors.New("must specify -j when specifying a manifest via -m"))
+ }
+
+ // do merge
+ err = mergeZips(readers, writer, *manifest, *sortEntries, *emulateJar, *stripDirEntries, *ignoreDuplicates)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+// a namedZipReader reads a .zip file and can say which file it's reading
+type namedZipReader struct {
+ path string
+ reader *zip.ReadCloser
+}
+
+// a zipEntryPath refers to a file contained in a zip
+type zipEntryPath struct {
+ zipName string
+ entryName string
+}
+
+func (p zipEntryPath) String() string {
+ return p.zipName + "/" + p.entryName
+}
+
+// a zipEntry is a zipSource that pulls its content from another zip
+type zipEntry struct {
+ path zipEntryPath
+ content *zip.File
+}
+
+func (ze zipEntry) String() string {
+ return ze.path.String()
+}
+
+func (ze zipEntry) IsDir() bool {
+ return ze.content.FileInfo().IsDir()
+}
+
+func (ze zipEntry) CRC32() uint32 {
+ return ze.content.FileHeader.CRC32
+}
+
+func (ze zipEntry) WriteToZip(dest string, zw *zip.Writer) error {
+ return zw.CopyFrom(ze.content, dest)
+}
+
+// a bufferEntry is a zipSource that pulls its content from a []byte
+type bufferEntry struct {
+ fh *zip.FileHeader
+ content []byte
+}
+
+func (be bufferEntry) String() string {
+ return "internal buffer"
+}
+
+func (be bufferEntry) IsDir() bool {
+ return be.fh.FileInfo().IsDir()
+}
+
+func (be bufferEntry) CRC32() uint32 {
+ return crc32.ChecksumIEEE(be.content)
+}
+
+func (be bufferEntry) WriteToZip(dest string, zw *zip.Writer) error {
+ w, err := zw.CreateHeader(be.fh)
+ if err != nil {
+ return err
+ }
+
+ if !be.IsDir() {
+ _, err = w.Write(be.content)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+type zipSource interface {
+ String() string
+ IsDir() bool
+ CRC32() uint32
+ WriteToZip(dest string, zw *zip.Writer) error
+}
+
+// a fileMapping specifies to copy a zip entry from one place to another
+type fileMapping struct {
+ dest string
+ source zipSource
+}
+
+func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest string,
+ sortEntries, emulateJar, stripDirEntries, ignoreDuplicates bool) error {
+
+ sourceByDest := make(map[string]zipSource, 0)
+ orderedMappings := []fileMapping{}
+
+ // if dest already exists returns a non-null zipSource for the existing source
+ addMapping := func(dest string, source zipSource) zipSource {
+ mapKey := filepath.Clean(dest)
+ if existingSource, exists := sourceByDest[mapKey]; exists {
+ return existingSource
+ }
+
+ sourceByDest[mapKey] = source
+ orderedMappings = append(orderedMappings, fileMapping{source: source, dest: dest})
+ return nil
+ }
+
+ if manifest != "" {
+ if !stripDirEntries {
+ dirHeader := jar.MetaDirFileHeader()
+ dirSource := bufferEntry{dirHeader, nil}
+ addMapping(jar.MetaDir, dirSource)
+ }
+
+ fh, buf, err := jar.ManifestFileContents(manifest)
+ if err != nil {
+ return err
+ }
+
+ fileSource := bufferEntry{fh, buf}
+ addMapping(jar.ManifestFile, fileSource)
+ }
+
+ for _, namedReader := range readers {
+ _, skipStripThisZip := zipsToNotStrip[namedReader.path]
+ for _, file := range namedReader.reader.File {
+ if !skipStripThisZip && shouldStripFile(emulateJar, file.Name) {
+ continue
+ }
+
+ if stripDirEntries && file.FileInfo().IsDir() {
+ continue
+ }
+
+ // check for other files or directories destined for the same path
+ dest := file.Name
+
+ // make a new entry to add
+ source := zipEntry{path: zipEntryPath{zipName: namedReader.path, entryName: file.Name}, content: file}
+
+ if existingSource := addMapping(dest, source); existingSource != nil {
+ // handle duplicates
+ if existingSource.IsDir() != source.IsDir() {
+ return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n",
+ dest, existingSource, source)
+ }
+ if ignoreDuplicates {
+ continue
+ }
+ if emulateJar &&
+ file.Name == jar.ManifestFile || file.Name == jar.ModuleInfoClass {
+ // Skip manifest and module info files that are not from the first input file
+ continue
+ }
+ if !source.IsDir() {
+ if emulateJar {
+ if existingSource.CRC32() != source.CRC32() {
+ fmt.Fprintf(os.Stdout, "WARNING: Duplicate path %v found in %v and %v\n",
+ dest, existingSource, source)
+ }
+ } else {
+ return fmt.Errorf("Duplicate path %v found in %v and %v\n",
+ dest, existingSource, source)
+ }
+ }
+ }
+ }
+ }
+
+ if emulateJar {
+ jarSort(orderedMappings)
+ } else if sortEntries {
+ alphanumericSort(orderedMappings)
+ }
+
+ for _, entry := range orderedMappings {
+ if err := entry.source.WriteToZip(entry.dest, writer); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func shouldStripFile(emulateJar bool, name string) bool {
+ for _, dir := range stripDirs {
+ if strings.HasPrefix(name, dir+"/") {
+ if emulateJar {
+ if name != jar.MetaDir && name != jar.ManifestFile {
+ return true
+ }
+ } else {
+ return true
+ }
+ }
+ }
+ for _, pattern := range stripFiles {
+ if match, err := filepath.Match(pattern, filepath.Base(name)); err != nil {
+ panic(fmt.Errorf("%s: %s", err.Error(), pattern))
+ } else if match {
+ return true
+ }
+ }
+ return false
+}
+
+func jarSort(files []fileMapping) {
+ sort.SliceStable(files, func(i, j int) bool {
+ return jar.EntryNamesLess(files[i].dest, files[j].dest)
+ })
+}
+
+func alphanumericSort(files []fileMapping) {
+ sort.SliceStable(files, func(i, j int) bool {
+ return files[i].dest < files[j].dest
+ })
+}
diff --git a/cmd/multiproduct_kati/Android.bp b/cmd/multiproduct_kati/Android.bp
index b264c35..04a5802 100644
--- a/cmd/multiproduct_kati/Android.bp
+++ b/cmd/multiproduct_kati/Android.bp
@@ -18,6 +18,7 @@
"soong-ui-build",
"soong-ui-logger",
"soong-ui-tracer",
+ "soong-zip",
],
srcs: [
"main.go",
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go
index fb1c890..06c5626 100644
--- a/cmd/multiproduct_kati/main.go
+++ b/cmd/multiproduct_kati/main.go
@@ -15,7 +15,6 @@
package main
import (
- "bytes"
"context"
"flag"
"fmt"
@@ -25,11 +24,13 @@
"runtime"
"strings"
"sync"
+ "syscall"
"time"
"android/soong/ui/build"
"android/soong/ui/logger"
"android/soong/ui/tracer"
+ "android/soong/zip"
)
// We default to number of cpus / 4, which seems to be the sweet spot for my
@@ -45,7 +46,7 @@
var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs")
-var keep = flag.Bool("keep", false, "keep successful output files")
+var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts")
var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or <out>/dist when empty)")
@@ -55,6 +56,9 @@
var buildVariant = flag.String("variant", "eng", "build variant to use")
+var skipProducts = flag.String("skip-products", "", "comma-separated list of products to skip (known failures, etc)")
+var includeProducts = flag.String("products", "", "comma-separated list of products to build")
+
const errorLeadingLines = 20
const errorTrailingLines = 20
@@ -156,6 +160,39 @@
return s.failed
}
+// TODO(b/70370883): This tool uses a lot of open files -- over the default
+// soft limit of 1024 on some systems. So bump up to the hard limit until I fix
+// the algorithm.
+func setMaxFiles(log logger.Logger) {
+ var limits syscall.Rlimit
+
+ err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits)
+ if err != nil {
+ log.Println("Failed to get file limit:", err)
+ return
+ }
+
+ log.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max)
+ if limits.Cur == limits.Max {
+ return
+ }
+
+ limits.Cur = limits.Max
+ err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
+ if err != nil {
+ log.Println("Failed to increase file limit:", err)
+ }
+}
+
+func inList(str string, list []string) bool {
+ for _, other := range list {
+ if str == other {
+ return true
+ }
+ }
+ return false
+}
+
func main() {
log := logger.New(os.Stderr)
defer log.Cleanup()
@@ -198,41 +235,77 @@
if err := os.MkdirAll(*outDir, 0777); err != nil {
log.Fatalf("Failed to create tempdir: %v", err)
}
-
- if !*keep {
- defer func() {
- if status.Finished() == 0 {
- os.RemoveAll(*outDir)
- }
- }()
- }
}
config.Environment().Set("OUT_DIR", *outDir)
log.Println("Output directory:", *outDir)
+ logsDir := filepath.Join(config.OutDir(), "logs")
+ os.MkdirAll(logsDir, 0777)
+
build.SetupOutDir(buildCtx, config)
if *alternateResultDir {
- logsDir := filepath.Join(config.DistDir(), "logs")
- os.MkdirAll(logsDir, 0777)
- log.SetOutput(filepath.Join(logsDir, "soong.log"))
- trace.SetOutput(filepath.Join(logsDir, "build.trace"))
+ distLogsDir := filepath.Join(config.DistDir(), "logs")
+ os.MkdirAll(distLogsDir, 0777)
+ log.SetOutput(filepath.Join(distLogsDir, "soong.log"))
+ trace.SetOutput(filepath.Join(distLogsDir, "build.trace"))
} else {
log.SetOutput(filepath.Join(config.OutDir(), "soong.log"))
trace.SetOutput(filepath.Join(config.OutDir(), "build.trace"))
}
- vars, err := build.DumpMakeVars(buildCtx, config, nil, nil, []string{"all_named_products"})
+ setMaxFiles(log)
+
+ vars, err := build.DumpMakeVars(buildCtx, config, nil, []string{"all_named_products"})
if err != nil {
log.Fatal(err)
}
- products := strings.Fields(vars["all_named_products"])
- log.Verbose("Got product list:", products)
+ var productsList []string
+ allProducts := strings.Fields(vars["all_named_products"])
+
+ if *includeProducts != "" {
+ missingProducts := []string{}
+ for _, product := range strings.Split(*includeProducts, ",") {
+ if inList(product, allProducts) {
+ productsList = append(productsList, product)
+ } else {
+ missingProducts = append(missingProducts, product)
+ }
+ }
+ if len(missingProducts) > 0 {
+ log.Fatalf("Products don't exist: %s\n", missingProducts)
+ }
+ } else {
+ productsList = allProducts
+ }
+
+ products := make([]string, 0, len(productsList))
+ skipList := strings.Split(*skipProducts, ",")
+ skipProduct := func(p string) bool {
+ for _, s := range skipList {
+ if p == s {
+ return true
+ }
+ }
+ return false
+ }
+ for _, product := range productsList {
+ if !skipProduct(product) {
+ products = append(products, product)
+ } else {
+ log.Verbose("Skipping: ", product)
+ }
+ }
+
+ log.Verbose("Got product list: ", products)
status.SetTotal(len(products))
var wg sync.WaitGroup
productConfigs := make(chan Product, len(products))
+ finder := build.NewSourceFinder(buildCtx, config)
+ defer finder.Shutdown()
+
// Run the product config for every product in parallel
for _, product := range products {
wg.Add(1)
@@ -245,17 +318,14 @@
})
productOutDir := filepath.Join(config.OutDir(), product)
- productLogDir := productOutDir
- if *alternateResultDir {
- productLogDir = filepath.Join(config.DistDir(), product)
- if err := os.MkdirAll(productLogDir, 0777); err != nil {
- log.Fatalf("Error creating log directory: %v", err)
- }
- }
+ productLogDir := filepath.Join(logsDir, product)
if err := os.MkdirAll(productOutDir, 0777); err != nil {
log.Fatalf("Error creating out directory: %v", err)
}
+ if err := os.MkdirAll(productLogDir, 0777); err != nil {
+ log.Fatalf("Error creating log directory: %v", err)
+ }
stdLog = filepath.Join(productLogDir, "std.log")
f, err := os.Create(stdLog)
@@ -263,7 +333,7 @@
log.Fatalf("Error creating std.log: %v", err)
}
- productLog := logger.New(&bytes.Buffer{})
+ productLog := logger.New(f)
productLog.SetOutput(filepath.Join(productLogDir, "soong.log"))
productCtx := build.Context{&build.ContextImpl{
@@ -276,6 +346,7 @@
productConfig := build.NewConfig(productCtx)
productConfig.Environment().Set("OUT_DIR", productOutDir)
+ build.FindSources(productCtx, productConfig, finder)
productConfig.Lunch(productCtx, product, *buildVariant)
build.Build(productCtx, productConfig, build.BuildProductConfig)
@@ -299,6 +370,26 @@
status.Fail(product.config.TargetProduct(), err, product.logFile)
})
+ defer func() {
+ if *keepArtifacts {
+ args := zip.ZipArgs{
+ FileArgs: []zip.FileArg{
+ {
+ GlobDir: product.config.OutDir(),
+ SourcePrefixToStrip: product.config.OutDir(),
+ },
+ },
+ OutputFilePath: filepath.Join(config.OutDir(), product.config.TargetProduct()+".zip"),
+ NumParallelJobs: runtime.NumCPU(),
+ CompressionLevel: 5,
+ }
+ if err := zip.Run(args); err != nil {
+ log.Fatalf("Error zipping artifacts: %v", err)
+ }
+ }
+ os.RemoveAll(product.config.OutDir())
+ }()
+
buildWhat := 0
if !*onlyConfig {
buildWhat |= build.BuildSoong
@@ -307,9 +398,6 @@
}
}
build.Build(product.ctx, product.config, buildWhat)
- if !*keep {
- os.RemoveAll(product.config.OutDir())
- }
status.Finish(product.config.TargetProduct())
}()
}
@@ -317,6 +405,20 @@
}
wg2.Wait()
+ if *alternateResultDir {
+ args := zip.ZipArgs{
+ FileArgs: []zip.FileArg{
+ {GlobDir: logsDir, SourcePrefixToStrip: logsDir},
+ },
+ OutputFilePath: filepath.Join(config.DistDir(), "logs.zip"),
+ NumParallelJobs: runtime.NumCPU(),
+ CompressionLevel: 5,
+ }
+ if err := zip.Run(args); err != nil {
+ log.Fatalf("Error zipping logs: %v", err)
+ }
+ }
+
if count := status.Finished(); count > 0 {
log.Fatalln(count, "products failed")
}
diff --git a/cmd/pom2mk/pom2mk.go b/cmd/pom2mk/pom2mk.go
index e6144a5..33d7ff9 100644
--- a/cmd/pom2mk/pom2mk.go
+++ b/cmd/pom2mk/pom2mk.go
@@ -18,7 +18,6 @@
"encoding/xml"
"flag"
"fmt"
- "io"
"io/ioutil"
"os"
"path/filepath"
@@ -56,13 +55,15 @@
return nil
}
-func (r *RewriteNames) Rewrite(name string) string {
+func (r *RewriteNames) MavenToMk(groupId string, artifactId string) string {
for _, r := range *r {
- if r.regexp.MatchString(name) {
- return r.regexp.ReplaceAllString(name, r.repl)
+ if r.regexp.MatchString(groupId + ":" + artifactId) {
+ return r.regexp.ReplaceAllString(groupId+":"+artifactId, r.repl)
+ } else if r.regexp.MatchString(artifactId) {
+ return r.regexp.ReplaceAllString(artifactId, r.repl)
}
}
- return name
+ return artifactId
}
var rewriteNames = RewriteNames{}
@@ -84,6 +85,9 @@
var extraDeps = make(ExtraDeps)
+var sdkVersion string
+var useVersion string
+
type Dependency struct {
XMLName xml.Name `xml:"dependency"`
@@ -98,18 +102,23 @@
type Pom struct {
XMLName xml.Name `xml:"http://maven.apache.org/POM/4.0.0 project"`
+ PomFile string `xml:"-"`
ArtifactFile string `xml:"-"`
+ MakeTarget string `xml:"-"`
GroupId string `xml:"groupId"`
ArtifactId string `xml:"artifactId"`
Version string `xml:"version"`
Packaging string `xml:"packaging"`
- Dependencies []Dependency `xml:"dependencies>dependency"`
+ Dependencies []*Dependency `xml:"dependencies>dependency"`
}
func (p Pom) MkName() string {
- return rewriteNames.Rewrite(p.ArtifactId)
+ if p.MakeTarget == "" {
+ p.MakeTarget = rewriteNames.MavenToMk(p.GroupId, p.ArtifactId)
+ }
+ return p.MakeTarget
}
func (p Pom) MkDeps() []string {
@@ -118,13 +127,28 @@
if d.Type != "aar" {
continue
}
- name := rewriteNames.Rewrite(d.ArtifactId)
+ name := rewriteNames.MavenToMk(d.GroupId, d.ArtifactId)
ret = append(ret, name)
ret = append(ret, extraDeps[name]...)
}
return ret
}
+func (p Pom) SdkVersion() string {
+ return sdkVersion
+}
+
+func (p *Pom) FixDepTypes(modules map[string]*Pom) {
+ for _, d := range p.Dependencies {
+ if d.Type != "" {
+ continue
+ }
+ if depPom, ok := modules[p.MkName()]; ok {
+ d.Type = depPom.Packaging
+ }
+ }
+}
+
var mkTemplate = template.Must(template.New("mk").Parse(`
include $(CLEAR_VARS)
LOCAL_MODULE := {{.MkName}}
@@ -134,31 +158,37 @@
LOCAL_BUILT_MODULE_STEM := javalib.jar
LOCAL_MODULE_SUFFIX := .{{.Packaging}}
LOCAL_USE_AAPT2 := true
+LOCAL_SDK_VERSION := {{.SdkVersion}}
LOCAL_STATIC_ANDROID_LIBRARIES := \
{{range .MkDeps}} {{.}} \
{{end}}
include $(BUILD_PREBUILT)
`))
-func convert(filename string, out io.Writer) error {
+func parse(filename string) (*Pom, error) {
data, err := ioutil.ReadFile(filename)
if err != nil {
- return err
+ return nil, err
}
var pom Pom
err = xml.Unmarshal(data, &pom)
if err != nil {
- return err
+ return nil, err
+ }
+
+ if useVersion != "" && pom.Version != useVersion {
+ return nil, nil
}
if pom.Packaging == "" {
pom.Packaging = "jar"
}
+ pom.PomFile = filename
pom.ArtifactFile = strings.TrimSuffix(filename, ".pom") + "." + pom.Packaging
- return mkTemplate.Execute(out, pom)
+ return &pom, nil
}
func main() {
@@ -171,13 +201,20 @@
Usage: %s [--rewrite <regex>=<replace>] [--extra-deps <module>=<module>[,<module>]] <dir>
-rewrite <regex>=<replace>
- rewrite can be used to specify mappings between the artifactId in the pom files and module
- names in the Android.mk files. This can be specified multiple times, the first matching
- regex will be used.
+ rewrite can be used to specify mappings between Maven projects and Make modules. The -rewrite
+ option can be specified multiple times. When determining the Make module for a given Maven
+ project, mappings are searched in the order they were specified. The first <regex> matching
+ either the Maven project's <groupId>:<artifactId> or <artifactId> will be used to generate
+ the Make module name using <replace>. If no matches are found, <artifactId> is used.
-extra-deps <module>=<module>[,<module>]
Some Android.mk modules have transitive dependencies that must be specified when they are
depended upon (like android-support-v7-mediarouter requires android-support-v7-appcompat).
This may be specified multiple times to declare these dependencies.
+ -sdk-version <version>
+ Sets LOCAL_SDK_VERSION := <version> for all modules.
+ -use-version <version>
+ If the maven directory contains multiple versions of artifacts and their pom files,
+ -use-version can be used to only write makefiles for a specific version of those artifacts.
<dir>
The directory to search for *.pom files under.
@@ -187,6 +224,8 @@
flag.Var(&extraDeps, "extra-deps", "Extra dependencies needed when depending on a module")
flag.Var(&rewriteNames, "rewrite", "Regex(es) to rewrite artifact names")
+ flag.StringVar(&sdkVersion, "sdk-version", "", "What to write to LOCAL_SDK_VERSION")
+ flag.StringVar(&useVersion, "use-version", "", "Only read artifacts of a specific version")
flag.Parse()
if flag.NArg() != 1 {
@@ -240,14 +279,40 @@
sort.Strings(filenames)
+ poms := []*Pom{}
+ modules := make(map[string]*Pom)
+ for _, filename := range filenames {
+ pom, err := parse(filename)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, "Error converting", filename, err)
+ os.Exit(1)
+ }
+
+ if pom != nil {
+ poms = append(poms, pom)
+ key := pom.MkName()
+
+ if old, ok := modules[key]; ok {
+ fmt.Fprintln(os.Stderr, "Module", key, "defined twice:", old.PomFile, pom.PomFile)
+ os.Exit(1)
+ }
+
+ modules[key] = pom
+ }
+ }
+
+ for _, pom := range poms {
+ pom.FixDepTypes(modules)
+ }
+
fmt.Println("# Automatically generated with:")
fmt.Println("# pom2mk", strings.Join(proptools.ShellEscape(os.Args[1:]), " "))
fmt.Println("LOCAL_PATH := $(call my-dir)")
- for _, filename := range filenames {
- err := convert(filename, os.Stdout)
+ for _, pom := range poms {
+ err := mkTemplate.Execute(os.Stdout, pom)
if err != nil {
- fmt.Fprintln(os.Stderr, "Error converting", filename, err)
+ fmt.Fprintln(os.Stderr, "Error writing", pom.PomFile, pom.MkName(), err)
os.Exit(1)
}
}
diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go
index 5064626..3b41c90 100644
--- a/cmd/sbox/sbox.go
+++ b/cmd/sbox/sbox.go
@@ -15,6 +15,8 @@
package main
import (
+ "errors"
+ "flag"
"fmt"
"io/ioutil"
"os"
@@ -24,7 +26,52 @@
"strings"
)
+var (
+ sandboxesRoot string
+ rawCommand string
+ outputRoot string
+ keepOutDir bool
+ depfileOut string
+)
+
+func init() {
+ flag.StringVar(&sandboxesRoot, "sandbox-path", "",
+ "root of temp directory to put the sandbox into")
+ flag.StringVar(&rawCommand, "c", "",
+ "command to run")
+ flag.StringVar(&outputRoot, "output-root", "",
+ "root of directory to copy outputs into")
+ flag.BoolVar(&keepOutDir, "keep-out-dir", false,
+ "whether to keep the sandbox directory when done")
+
+ flag.StringVar(&depfileOut, "depfile-out", "",
+ "file path of the depfile to generate. This value will replace '__SBOX_DEPFILE__' in the command and will be treated as an output but won't be added to __SBOX_OUT_FILES__")
+
+}
+
+func usageViolation(violation string) {
+ if violation != "" {
+ fmt.Fprintf(os.Stderr, "Usage error: %s.\n\n", violation)
+ }
+
+ fmt.Fprintf(os.Stderr,
+ "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> --overwrite [--depfile-out depFile] <outputFile> [<outputFile>...]\n"+
+ "\n"+
+ "Deletes <outputRoot>,"+
+ "runs <commandToRun>,"+
+ "and moves each <outputFile> out of <sandboxPath> and into <outputRoot>\n")
+
+ flag.PrintDefaults()
+
+ os.Exit(1)
+}
+
func main() {
+ flag.Usage = func() {
+ usageViolation("")
+ }
+ flag.Parse()
+
error := run()
if error != nil {
fmt.Fprintln(os.Stderr, error)
@@ -32,75 +79,88 @@
}
}
-var usage = "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> <outputFile> [<outputFile>...]\n" +
- "\n" +
- "Runs <commandToRun> and moves each <outputFile> out of <sandboxPath>\n" +
- "If any file in <outputFiles> is specified by absolute path, then <outputRoot> must be specified as well,\n" +
- "to enable sbox to compute the relative path within the sandbox of the specified output files"
-
-func usageError(violation string) error {
- return fmt.Errorf("Usage error: %s.\n\n%s", violation, usage)
+func findAllFilesUnder(root string) (paths []string) {
+ paths = []string{}
+ filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+ if !info.IsDir() {
+ relPath, err := filepath.Rel(root, path)
+ if err != nil {
+ // couldn't find relative path from ancestor?
+ panic(err)
+ }
+ paths = append(paths, relPath)
+ }
+ return nil
+ })
+ return paths
}
func run() error {
- var outFiles []string
- args := os.Args[1:]
-
- var rawCommand string
- var sandboxesRoot string
- removeTempDir := true
- var outputRoot string
-
- for i := 0; i < len(args); i++ {
- arg := args[i]
- if arg == "--sandbox-path" {
- sandboxesRoot = args[i+1]
- i++
- } else if arg == "-c" {
- rawCommand = args[i+1]
- i++
- } else if arg == "--output-root" {
- outputRoot = args[i+1]
- i++
- } else if arg == "--keep-out-dir" {
- removeTempDir = false
- } else {
- outFiles = append(outFiles, arg)
- }
+ if rawCommand == "" {
+ usageViolation("-c <commandToRun> is required and must be non-empty")
}
- if len(rawCommand) == 0 {
- return usageError("-c <commandToRun> is required and must be non-empty")
- }
- if outFiles == nil {
- return usageError("at least one output file must be given")
- }
- if len(sandboxesRoot) == 0 {
+ if sandboxesRoot == "" {
// In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
// and the sbox executable will most likely be at a fixed location relative to OUT_DIR too, so
// the value of sandboxesRoot will most likely be at a fixed location relative to the sbox executable
// However, Soong also needs to be able to separately remove the sandbox directory on startup (if it has anything left in it)
// and by passing it as a parameter we don't need to duplicate its value
- return usageError("--sandbox-path <sandboxPath> is required and must be non-empty")
+ usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
}
+ if len(outputRoot) == 0 {
+ usageViolation("--output-root <outputRoot> is required and must be non-empty")
+ }
+
+ // the contents of the __SBOX_OUT_FILES__ variable
+ outputsVarEntries := flag.Args()
+ if len(outputsVarEntries) == 0 {
+ usageViolation("at least one output file must be given")
+ }
+
+ // all outputs
+ var allOutputs []string
+
+ // setup directories
+ err := os.MkdirAll(sandboxesRoot, 0777)
+ if err != nil {
+ return err
+ }
+ err = os.RemoveAll(outputRoot)
+ if err != nil {
+ return err
+ }
+ err = os.MkdirAll(outputRoot, 0777)
+ if err != nil {
+ return err
+ }
+
+ tempDir, err := ioutil.TempDir(sandboxesRoot, "sbox")
// Rewrite output file paths to be relative to output root
// This facilitates matching them up against the corresponding paths in the temporary directory in case they're absolute
- for i, filePath := range outFiles {
- if path.IsAbs(filePath) {
- if len(outputRoot) == 0 {
- return fmt.Errorf("Absolute path %s requires nonempty value for --output-root", filePath)
- }
- }
+ for i, filePath := range outputsVarEntries {
relativePath, err := filepath.Rel(outputRoot, filePath)
if err != nil {
return err
}
- outFiles[i] = relativePath
+ outputsVarEntries[i] = relativePath
}
- os.MkdirAll(sandboxesRoot, 0777)
+ allOutputs = append([]string(nil), outputsVarEntries...)
- tempDir, err := ioutil.TempDir(sandboxesRoot, "sbox")
+ if depfileOut != "" {
+ sandboxedDepfile, err := filepath.Rel(outputRoot, depfileOut)
+ if err != nil {
+ return err
+ }
+ allOutputs = append(allOutputs, sandboxedDepfile)
+ if !strings.Contains(rawCommand, "__SBOX_DEPFILE__") {
+ return fmt.Errorf("the --depfile-out argument only makes sense if the command contains the text __SBOX_DEPFILE__")
+ }
+ rawCommand = strings.Replace(rawCommand, "__SBOX_DEPFILE__", filepath.Join(tempDir, sandboxedDepfile), -1)
+
+ }
+
if err != nil {
return fmt.Errorf("Failed to create temp dir: %s", err)
}
@@ -110,7 +170,7 @@
// then at the beginning of the next build, Soong will retry the cleanup
defer func() {
// in some cases we decline to remove the temp dir, to facilitate debugging
- if removeTempDir {
+ if !keepOutDir {
os.RemoveAll(tempDir)
}
}()
@@ -122,7 +182,7 @@
if strings.Contains(rawCommand, "__SBOX_OUT_FILES__") {
// expands into a space-separated list of output files to be generated into the sandbox directory
tempOutPaths := []string{}
- for _, outputPath := range outFiles {
+ for _, outputPath := range outputsVarEntries {
tempOutPath := path.Join(tempDir, outputPath)
tempOutPaths = append(tempOutPaths, tempOutPath)
}
@@ -130,8 +190,12 @@
rawCommand = strings.Replace(rawCommand, "__SBOX_OUT_FILES__", pathsText, -1)
}
- for _, filePath := range outFiles {
- os.MkdirAll(path.Join(tempDir, filepath.Dir(filePath)), 0777)
+ for _, filePath := range allOutputs {
+ dir := path.Join(tempDir, filepath.Dir(filePath))
+ err = os.MkdirAll(dir, 0777)
+ if err != nil {
+ return err
+ }
}
commandDescription := rawCommand
@@ -149,32 +213,62 @@
}
// validate that all files are created properly
- var outputErrors []error
- for _, filePath := range outFiles {
+ var missingOutputErrors []string
+ for _, filePath := range allOutputs {
tempPath := filepath.Join(tempDir, filePath)
fileInfo, err := os.Stat(tempPath)
if err != nil {
- outputErrors = append(outputErrors, fmt.Errorf("failed to create expected output file: %s\n", tempPath))
+ missingOutputErrors = append(missingOutputErrors, fmt.Sprintf("%s: does not exist", filePath))
continue
}
if fileInfo.IsDir() {
- outputErrors = append(outputErrors, fmt.Errorf("Output path %s refers to a directory, not a file. This is not permitted because it prevents robust up-to-date checks\n", filePath))
+ missingOutputErrors = append(missingOutputErrors, fmt.Sprintf("%s: not a file", filePath))
}
}
- if len(outputErrors) > 0 {
+ if len(missingOutputErrors) > 0 {
+ // find all created files for making a more informative error message
+ createdFiles := findAllFilesUnder(tempDir)
+
+ // build error message
+ errorMessage := "mismatch between declared and actual outputs\n"
+ errorMessage += "in sbox command(" + commandDescription + ")\n\n"
+ errorMessage += "in sandbox " + tempDir + ",\n"
+ errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
+ for _, missingOutputError := range missingOutputErrors {
+ errorMessage += " " + missingOutputError + "\n"
+ }
+ if len(createdFiles) < 1 {
+ errorMessage += "created 0 files."
+ } else {
+ errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
+ creationMessages := createdFiles
+ maxNumCreationLines := 10
+ if len(creationMessages) > maxNumCreationLines {
+ creationMessages = creationMessages[:maxNumCreationLines]
+ creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxNumCreationLines))
+ }
+ for _, creationMessage := range creationMessages {
+ errorMessage += " " + creationMessage + "\n"
+ }
+ }
+
// Keep the temporary output directory around in case a user wants to inspect it for debugging purposes.
// Soong will delete it later anyway.
- removeTempDir = false
- return fmt.Errorf("mismatch between declared and actual outputs in sbox command (%s):\n%v", commandDescription, outputErrors)
+ keepOutDir = true
+ return errors.New(errorMessage)
}
// the created files match the declared files; now move them
- for _, filePath := range outFiles {
+ for _, filePath := range allOutputs {
tempPath := filepath.Join(tempDir, filePath)
destPath := filePath
if len(outputRoot) != 0 {
destPath = filepath.Join(outputRoot, filePath)
}
- err := os.Rename(tempPath, destPath)
+ err := os.MkdirAll(filepath.Dir(destPath), 0777)
+ if err != nil {
+ return err
+ }
+ err = os.Rename(tempPath, destPath)
if err != nil {
return err
}
diff --git a/cmd/soong_build/Android.bp b/cmd/soong_build/Android.bp
index d9daafc..2536a53 100644
--- a/cmd/soong_build/Android.bp
+++ b/cmd/soong_build/Android.bp
@@ -23,6 +23,7 @@
],
srcs: [
"main.go",
+ "writedocs.go",
],
primaryBuilder: true,
}
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index e15a6bd..40beab8 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -25,6 +25,30 @@
"android/soong/android"
)
+var (
+ docFile string
+)
+
+func init() {
+ flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
+}
+
+func newNameResolver(config android.Config) *android.NameResolver {
+ namespacePathsToExport := make(map[string]bool)
+
+ for _, namespaceName := range config.ProductVariables.NamespacesToExport {
+ namespacePathsToExport[namespaceName] = true
+ }
+
+ namespacePathsToExport["."] = true // always export the root namespace
+
+ exportFilter := func(namespace *android.Namespace) bool {
+ return namespacePathsToExport[namespace.Path]
+ }
+
+ return android.NewNameResolver(exportFilter)
+}
+
func main() {
flag.Parse()
@@ -40,10 +64,17 @@
os.Exit(1)
}
- // Temporary hack
- //ctx.SetIgnoreUnknownModuleTypes(true)
+ if docFile != "" {
+ configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions)
+ }
+
+ ctx.SetNameInterface(newNameResolver(configuration))
ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
bootstrap.Main(ctx.Context, configuration, configuration.ConfigFileName, configuration.ProductVariablesFileName)
+
+ if docFile != "" {
+ writeDocs(ctx, docFile)
+ }
}
diff --git a/cmd/soong_build/writedocs.go b/cmd/soong_build/writedocs.go
new file mode 100644
index 0000000..a6686c0
--- /dev/null
+++ b/cmd/soong_build/writedocs.go
@@ -0,0 +1,129 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "android/soong/android"
+ "bytes"
+ "html/template"
+ "io/ioutil"
+
+ "github.com/google/blueprint/bootstrap"
+)
+
+func writeDocs(ctx *android.Context, filename string) error {
+ moduleTypeList, err := bootstrap.ModuleTypeDocs(ctx.Context)
+ if err != nil {
+ return err
+ }
+
+ buf := &bytes.Buffer{}
+
+ unique := 0
+
+ tmpl, err := template.New("file").Funcs(map[string]interface{}{
+ "unique": func() int {
+ unique++
+ return unique
+ }}).Parse(fileTemplate)
+ if err != nil {
+ return err
+ }
+
+ err = tmpl.Execute(buf, moduleTypeList)
+ if err != nil {
+ return err
+ }
+
+ err = ioutil.WriteFile(filename, buf.Bytes(), 0666)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+const (
+ fileTemplate = `
+<html>
+<head>
+<title>Build Docs</title>
+<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css">
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+</head>
+<body>
+<h1>Build Docs</h1>
+<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+ {{range .}}
+ {{ $collapseIndex := unique }}
+ <div class="panel panel-default">
+ <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}">
+ <h2 class="panel-title">
+ <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}">
+ {{.Name}}
+ </a>
+ </h2>
+ </div>
+ </div>
+ <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}">
+ <div class="panel-body">
+ <p>{{.Text}}</p>
+ {{range .PropertyStructs}}
+ <p>{{.Text}}</p>
+ {{template "properties" .Properties}}
+ {{end}}
+ </div>
+ </div>
+ {{end}}
+</div>
+</body>
+</html>
+
+{{define "properties"}}
+ <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+ {{range .}}
+ {{$collapseIndex := unique}}
+ {{if .Properties}}
+ <div class="panel panel-default">
+ <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}">
+ <h4 class="panel-title">
+ <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}">
+ {{.Name}}{{range .OtherNames}}, {{.}}{{end}}
+ </a>
+ </h4>
+ </div>
+ </div>
+ <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}">
+ <div class="panel-body">
+ <p>{{.Text}}</p>
+ {{range .OtherTexts}}<p>{{.}}</p>{{end}}
+ {{template "properties" .Properties}}
+ </div>
+ </div>
+ {{else}}
+ <div>
+ <h4>{{.Name}}{{range .OtherNames}}, {{.}}{{end}}</h4>
+ <p>{{.Text}}</p>
+ {{range .OtherTexts}}<p>{{.}}</p>{{end}}
+ <p><i>Type: {{.Type}}</i></p>
+ {{if .Default}}<p><i>Default: {{.Default}}</i></p>{{end}}
+ </div>
+ {{end}}
+ {{end}}
+ </div>
+{{end}}
+`
+)
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 94d6d5c..2ca7ebf 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -16,6 +16,8 @@
import (
"context"
+ "flag"
+ "fmt"
"os"
"path/filepath"
"strconv"
@@ -45,7 +47,10 @@
log := logger.New(os.Stderr)
defer log.Cleanup()
- if len(os.Args) < 2 || !inList("--make-mode", os.Args) {
+ if len(os.Args) < 2 || !(inList("--make-mode", os.Args) ||
+ os.Args[1] == "--dumpvars-mode" ||
+ os.Args[1] == "--dumpvar-mode") {
+
log.Fatalln("The `soong` native UI is not yet available.")
}
@@ -66,7 +71,12 @@
Tracer: trace,
StdioInterface: build.StdioImpl{},
}}
- config := build.NewConfig(buildCtx, os.Args[1:]...)
+ var config build.Config
+ if os.Args[1] == "--dumpvars-mode" || os.Args[1] == "--dumpvar-mode" {
+ config = build.NewConfig(buildCtx)
+ } else {
+ config = build.NewConfig(buildCtx, os.Args[1:]...)
+ }
log.SetVerbose(config.IsVerbose())
build.SetupOutDir(buildCtx, config)
@@ -95,5 +105,137 @@
}
}
- build.Build(buildCtx, config, build.BuildAll)
+ f := build.NewSourceFinder(buildCtx, config)
+ defer f.Shutdown()
+ build.FindSources(buildCtx, config, f)
+
+ if os.Args[1] == "--dumpvar-mode" {
+ dumpVar(buildCtx, config, os.Args[2:])
+ } else if os.Args[1] == "--dumpvars-mode" {
+ dumpVars(buildCtx, config, os.Args[2:])
+ } else {
+ toBuild := build.BuildAll
+ if config.Checkbuild() {
+ toBuild |= build.RunBuildTests
+ }
+ build.Build(buildCtx, config, toBuild)
+ }
+}
+
+func dumpVar(ctx build.Context, config build.Config, args []string) {
+ flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
+ flags.Usage = func() {
+ fmt.Fprintf(os.Stderr, "usage: %s --dumpvar-mode [--abs] <VAR>\n\n", os.Args[0])
+ fmt.Fprintln(os.Stderr, "In dumpvar mode, print the value of the legacy make variable VAR to stdout")
+ fmt.Fprintln(os.Stderr, "")
+
+ fmt.Fprintln(os.Stderr, "'report_config' is a special case that prints the human-readable config banner")
+ fmt.Fprintln(os.Stderr, "from the beginning of the build.")
+ fmt.Fprintln(os.Stderr, "")
+ flags.PrintDefaults()
+ }
+ abs := flags.Bool("abs", false, "Print the absolute path of the value")
+ flags.Parse(args)
+
+ if flags.NArg() != 1 {
+ flags.Usage()
+ os.Exit(1)
+ }
+
+ varName := flags.Arg(0)
+ if varName == "report_config" {
+ varData, err := build.DumpMakeVars(ctx, config, nil, build.BannerVars)
+ if err != nil {
+ ctx.Fatal(err)
+ }
+
+ fmt.Println(build.Banner(varData))
+ } else {
+ varData, err := build.DumpMakeVars(ctx, config, nil, []string{varName})
+ if err != nil {
+ ctx.Fatal(err)
+ }
+
+ if *abs {
+ var res []string
+ for _, path := range strings.Fields(varData[varName]) {
+ if abs, err := filepath.Abs(path); err == nil {
+ res = append(res, abs)
+ } else {
+ ctx.Fatalln("Failed to get absolute path of", path, err)
+ }
+ }
+ fmt.Println(strings.Join(res, " "))
+ } else {
+ fmt.Println(varData[varName])
+ }
+ }
+}
+
+func dumpVars(ctx build.Context, config build.Config, args []string) {
+ flags := flag.NewFlagSet("dumpvars", flag.ExitOnError)
+ flags.Usage = func() {
+ fmt.Fprintf(os.Stderr, "usage: %s --dumpvars-mode [--vars=\"VAR VAR ...\"]\n\n", os.Args[0])
+ fmt.Fprintln(os.Stderr, "In dumpvars mode, dump the values of one or more legacy make variables, in")
+ fmt.Fprintln(os.Stderr, "shell syntax. The resulting output may be sourced directly into a shell to")
+ fmt.Fprintln(os.Stderr, "set corresponding shell variables.")
+ fmt.Fprintln(os.Stderr, "")
+
+ fmt.Fprintln(os.Stderr, "'report_config' is a special case that dumps a variable containing the")
+ fmt.Fprintln(os.Stderr, "human-readable config banner from the beginning of the build.")
+ fmt.Fprintln(os.Stderr, "")
+ flags.PrintDefaults()
+ }
+
+ varsStr := flags.String("vars", "", "Space-separated list of variables to dump")
+ absVarsStr := flags.String("abs-vars", "", "Space-separated list of variables to dump (using absolute paths)")
+
+ varPrefix := flags.String("var-prefix", "", "String to prepend to all variable names when dumping")
+ absVarPrefix := flags.String("abs-var-prefix", "", "String to prepent to all absolute path variable names when dumping")
+
+ flags.Parse(args)
+
+ if flags.NArg() != 0 {
+ flags.Usage()
+ os.Exit(1)
+ }
+
+ vars := strings.Fields(*varsStr)
+ absVars := strings.Fields(*absVarsStr)
+
+ allVars := append([]string{}, vars...)
+ allVars = append(allVars, absVars...)
+
+ if i := indexList("report_config", allVars); i != -1 {
+ allVars = append(allVars[:i], allVars[i+1:]...)
+ allVars = append(allVars, build.BannerVars...)
+ }
+
+ if len(allVars) == 0 {
+ return
+ }
+
+ varData, err := build.DumpMakeVars(ctx, config, nil, allVars)
+ if err != nil {
+ ctx.Fatal(err)
+ }
+
+ for _, name := range vars {
+ if name == "report_config" {
+ fmt.Printf("%sreport_config='%s'\n", *varPrefix, build.Banner(varData))
+ } else {
+ fmt.Printf("%s%s='%s'\n", *varPrefix, name, varData[name])
+ }
+ }
+ for _, name := range absVars {
+ var res []string
+ for _, path := range strings.Fields(varData[name]) {
+ abs, err := filepath.Abs(path)
+ if err != nil {
+ ctx.Fatalln("Failed to get absolute path of", path, err)
+ }
+ res = append(res, abs)
+ }
+ fmt.Printf("%s%s='%s'\n", *absVarPrefix, name, strings.Join(res, " "))
+ }
}
diff --git a/cmd/soong_zip/rate_limit.go b/cmd/soong_zip/rate_limit.go
deleted file mode 100644
index 9e95bc1..0000000
--- a/cmd/soong_zip/rate_limit.go
+++ /dev/null
@@ -1,115 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
- "runtime"
-)
-
-type RateLimit struct {
- requests chan struct{}
- finished chan int
- released chan int
- stop chan struct{}
-}
-
-// NewRateLimit starts a new rate limiter with maxExecs number of executions
-// allowed to happen at a time. If maxExecs is <= 0, it will default to the
-// number of logical CPUs on the system.
-//
-// With Finish and Release, we'll keep track of outstanding buffer sizes to be
-// written. If that size goes above maxMem, we'll prevent starting new
-// executions.
-//
-// The total memory use may be higher due to current executions. This just
-// prevents runaway memory use due to slower writes.
-func NewRateLimit(maxExecs int, maxMem int64) *RateLimit {
- if maxExecs <= 0 {
- maxExecs = runtime.NumCPU()
- }
- if maxMem <= 0 {
- // Default to 512MB
- maxMem = 512 * 1024 * 1024
- }
-
- ret := &RateLimit{
- requests: make(chan struct{}),
-
- // Let all of the pending executions to mark themselves as finished,
- // even if our goroutine isn't processing input.
- finished: make(chan int, maxExecs),
-
- released: make(chan int),
- stop: make(chan struct{}),
- }
-
- go ret.goFunc(maxExecs, maxMem)
-
- return ret
-}
-
-// RequestExecution blocks until another execution can be allowed to run.
-func (r *RateLimit) RequestExecution() Execution {
- <-r.requests
- return r.finished
-}
-
-type Execution chan<- int
-
-// Finish will mark your execution as finished, and allow another request to be
-// approved.
-//
-// bufferSize may be specified to count memory buffer sizes, and must be
-// matched with calls to RateLimit.Release to mark the buffers as released.
-func (e Execution) Finish(bufferSize int) {
- e <- bufferSize
-}
-
-// Call Release when finished with a buffer recorded with Finish.
-func (r *RateLimit) Release(bufferSize int) {
- r.released <- bufferSize
-}
-
-// Stop the background goroutine
-func (r *RateLimit) Stop() {
- close(r.stop)
-}
-
-func (r *RateLimit) goFunc(maxExecs int, maxMem int64) {
- var curExecs int
- var curMemory int64
-
- for {
- var requests chan struct{}
- if curExecs < maxExecs && curMemory < maxMem {
- requests = r.requests
- }
-
- select {
- case requests <- struct{}{}:
- curExecs++
- case amount := <-r.finished:
- curExecs--
- curMemory += int64(amount)
- if curExecs < 0 {
- panic("curExecs < 0")
- }
- case amount := <-r.released:
- curMemory -= int64(amount)
- case <-r.stop:
- return
- }
- }
-}
diff --git a/cmd/soong_zip/soong_zip.go b/cmd/soong_zip/soong_zip.go
deleted file mode 100644
index 8407788..0000000
--- a/cmd/soong_zip/soong_zip.go
+++ /dev/null
@@ -1,715 +0,0 @@
-// Copyright 2015 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
- "bytes"
- "compress/flate"
- "flag"
- "fmt"
- "hash/crc32"
- "io"
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "runtime"
- "runtime/pprof"
- "runtime/trace"
- "strings"
- "sync"
- "time"
-
- "android/soong/third_party/zip"
-)
-
-// Block size used during parallel compression of a single file.
-const parallelBlockSize = 1 * 1024 * 1024 // 1MB
-
-// Minimum file size to use parallel compression. It requires more
-// flate.Writer allocations, since we can't change the dictionary
-// during Reset
-const minParallelFileSize = parallelBlockSize * 6
-
-// Size of the ZIP compression window (32KB)
-const windowSize = 32 * 1024
-
-type nopCloser struct {
- io.Writer
-}
-
-func (nopCloser) Close() error {
- return nil
-}
-
-type fileArg struct {
- pathPrefixInZip, sourcePrefixToStrip string
- sourceFiles []string
-}
-
-type pathMapping struct {
- dest, src string
- zipMethod uint16
-}
-
-type uniqueSet map[string]bool
-
-func (u *uniqueSet) String() string {
- return `""`
-}
-
-func (u *uniqueSet) Set(s string) error {
- if _, found := (*u)[s]; found {
- return fmt.Errorf("File %q was specified twice as a file to not deflate", s)
- } else {
- (*u)[s] = true
- }
-
- return nil
-}
-
-type fileArgs []fileArg
-
-type file struct{}
-
-type listFiles struct{}
-
-func (f *file) String() string {
- return `""`
-}
-
-func (f *file) Set(s string) error {
- if *relativeRoot == "" {
- return fmt.Errorf("must pass -C before -f or -l")
- }
-
- fArgs = append(fArgs, fileArg{
- pathPrefixInZip: filepath.Clean(*rootPrefix),
- sourcePrefixToStrip: filepath.Clean(*relativeRoot),
- sourceFiles: []string{s},
- })
-
- return nil
-}
-
-func (l *listFiles) String() string {
- return `""`
-}
-
-func (l *listFiles) Set(s string) error {
- if *relativeRoot == "" {
- return fmt.Errorf("must pass -C before -f or -l")
- }
-
- list, err := ioutil.ReadFile(s)
- if err != nil {
- return err
- }
-
- fArgs = append(fArgs, fileArg{
- pathPrefixInZip: filepath.Clean(*rootPrefix),
- sourcePrefixToStrip: filepath.Clean(*relativeRoot),
- sourceFiles: strings.Split(string(list), "\n"),
- })
-
- return nil
-}
-
-var (
- out = flag.String("o", "", "file to write zip file to")
- manifest = flag.String("m", "", "input jar manifest file name")
- directories = flag.Bool("d", false, "include directories in zip")
- rootPrefix = flag.String("P", "", "path prefix within the zip at which to place files")
- relativeRoot = flag.String("C", "", "path to use as relative root of files in next -f or -l argument")
- parallelJobs = flag.Int("j", runtime.NumCPU(), "number of parallel threads to use")
- compLevel = flag.Int("L", 5, "deflate compression level (0-9)")
-
- fArgs fileArgs
- nonDeflatedFiles = make(uniqueSet)
-
- cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file")
- traceFile = flag.String("trace", "", "write trace to file")
-)
-
-func init() {
- flag.Var(&listFiles{}, "l", "file containing list of .class files")
- flag.Var(&file{}, "f", "file to include in zip")
- flag.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression")
-}
-
-func usage() {
- fmt.Fprintf(os.Stderr, "usage: soong_zip -o zipfile [-m manifest] -C dir [-f|-l file]...\n")
- flag.PrintDefaults()
- os.Exit(2)
-}
-
-type zipWriter struct {
- time time.Time
- createdDirs map[string]bool
- directories bool
-
- errors chan error
- writeOps chan chan *zipEntry
-
- rateLimit *RateLimit
-
- compressorPool sync.Pool
- compLevel int
-}
-
-type zipEntry struct {
- fh *zip.FileHeader
-
- // List of delayed io.Reader
- futureReaders chan chan io.Reader
-}
-
-func main() {
- flag.Parse()
-
- if *cpuProfile != "" {
- f, err := os.Create(*cpuProfile)
- if err != nil {
- fmt.Fprintln(os.Stderr, err.Error())
- os.Exit(1)
- }
- defer f.Close()
- pprof.StartCPUProfile(f)
- defer pprof.StopCPUProfile()
- }
-
- if *traceFile != "" {
- f, err := os.Create(*traceFile)
- if err != nil {
- fmt.Fprintln(os.Stderr, err.Error())
- os.Exit(1)
- }
- defer f.Close()
- err = trace.Start(f)
- if err != nil {
- fmt.Fprintln(os.Stderr, err.Error())
- os.Exit(1)
- }
- defer trace.Stop()
- }
-
- if *out == "" {
- fmt.Fprintf(os.Stderr, "error: -o is required\n")
- usage()
- }
-
- w := &zipWriter{
- time: time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC),
- createdDirs: make(map[string]bool),
- directories: *directories,
- compLevel: *compLevel,
- }
-
- pathMappings := []pathMapping{}
- set := make(map[string]string)
-
- for _, fa := range fArgs {
- for _, src := range fa.sourceFiles {
- if err := fillPathPairs(fa.pathPrefixInZip,
- fa.sourcePrefixToStrip, src, set, &pathMappings); err != nil {
- log.Fatal(err)
- }
- }
- }
-
- err := w.write(*out, pathMappings, *manifest)
- if err != nil {
- fmt.Fprintln(os.Stderr, err.Error())
- os.Exit(1)
- }
-}
-
-func fillPathPairs(prefix, rel, src string, set map[string]string, pathMappings *[]pathMapping) error {
- src = strings.TrimSpace(src)
- if src == "" {
- return nil
- }
- src = filepath.Clean(src)
- dest, err := filepath.Rel(rel, src)
- if err != nil {
- return err
- }
- dest = filepath.Join(prefix, dest)
-
- if _, found := set[dest]; found {
- return fmt.Errorf("found two file paths to be copied into dest path: %q,"+
- " both [%q]%q and [%q]%q!",
- dest, dest, src, dest, set[dest])
- } else {
- set[dest] = src
- }
-
- zipMethod := zip.Deflate
- if _, found := nonDeflatedFiles[dest]; found {
- zipMethod = zip.Store
- }
- *pathMappings = append(*pathMappings,
- pathMapping{dest: dest, src: src, zipMethod: zipMethod})
-
- return nil
-}
-
-func (z *zipWriter) write(out string, pathMappings []pathMapping, manifest string) error {
- f, err := os.Create(out)
- if err != nil {
- return err
- }
-
- defer f.Close()
- defer func() {
- if err != nil {
- os.Remove(out)
- }
- }()
-
- z.errors = make(chan error)
- defer close(z.errors)
-
- // This channel size can be essentially unlimited -- it's used as a fifo
- // queue decouple the CPU and IO loads. Directories don't require any
- // compression time, but still cost some IO. Similar with small files that
- // can be very fast to compress. Some files that are more difficult to
- // compress won't take a corresponding longer time writing out.
- //
- // The optimum size here depends on your CPU and IO characteristics, and
- // the the layout of your zip file. 1000 was chosen mostly at random as
- // something that worked reasonably well for a test file.
- //
- // The RateLimit object will put the upper bounds on the number of
- // parallel compressions and outstanding buffers.
- z.writeOps = make(chan chan *zipEntry, 1000)
- z.rateLimit = NewRateLimit(*parallelJobs, 0)
- defer z.rateLimit.Stop()
-
- go func() {
- var err error
- defer close(z.writeOps)
-
- for _, ele := range pathMappings {
- err = z.writeFile(ele.dest, ele.src, ele.zipMethod)
- if err != nil {
- z.errors <- err
- return
- }
- }
-
- if manifest != "" {
- err = z.writeFile("META-INF/MANIFEST.MF", manifest, zip.Deflate)
- if err != nil {
- z.errors <- err
- return
- }
- }
- }()
-
- zipw := zip.NewWriter(f)
-
- var currentWriteOpChan chan *zipEntry
- var currentWriter io.WriteCloser
- var currentReaders chan chan io.Reader
- var currentReader chan io.Reader
- var done bool
-
- for !done {
- var writeOpsChan chan chan *zipEntry
- var writeOpChan chan *zipEntry
- var readersChan chan chan io.Reader
-
- if currentReader != nil {
- // Only read and process errors
- } else if currentReaders != nil {
- readersChan = currentReaders
- } else if currentWriteOpChan != nil {
- writeOpChan = currentWriteOpChan
- } else {
- writeOpsChan = z.writeOps
- }
-
- select {
- case writeOp, ok := <-writeOpsChan:
- if !ok {
- done = true
- }
-
- currentWriteOpChan = writeOp
-
- case op := <-writeOpChan:
- currentWriteOpChan = nil
-
- if op.fh.Method == zip.Deflate {
- currentWriter, err = zipw.CreateCompressedHeader(op.fh)
- } else {
- var zw io.Writer
- zw, err = zipw.CreateHeader(op.fh)
- currentWriter = nopCloser{zw}
- }
- if err != nil {
- return err
- }
-
- currentReaders = op.futureReaders
- if op.futureReaders == nil {
- currentWriter.Close()
- currentWriter = nil
- }
-
- case futureReader, ok := <-readersChan:
- if !ok {
- // Done with reading
- currentWriter.Close()
- currentWriter = nil
- currentReaders = nil
- }
-
- currentReader = futureReader
-
- case reader := <-currentReader:
- var count int64
- count, err = io.Copy(currentWriter, reader)
- if err != nil {
- return err
- }
- z.rateLimit.Release(int(count))
-
- currentReader = nil
-
- case err = <-z.errors:
- return err
- }
- }
-
- // One last chance to catch an error
- select {
- case err = <-z.errors:
- return err
- default:
- zipw.Close()
- return nil
- }
-}
-
-func (z *zipWriter) writeFile(dest, src string, method uint16) error {
- var fileSize int64
- var executable bool
-
- if s, err := os.Lstat(src); err != nil {
- return err
- } else if s.IsDir() {
- if z.directories {
- return z.writeDirectory(dest)
- }
- return nil
- } else if s.Mode()&os.ModeSymlink != 0 {
- return z.writeSymlink(dest, src)
- } else if !s.Mode().IsRegular() {
- return fmt.Errorf("%s is not a file, directory, or symlink", src)
- } else {
- fileSize = s.Size()
- executable = s.Mode()&0100 != 0
- }
-
- if z.directories {
- dir, _ := filepath.Split(dest)
- err := z.writeDirectory(dir)
- if err != nil {
- return err
- }
- }
-
- compressChan := make(chan *zipEntry, 1)
- z.writeOps <- compressChan
-
- // Pre-fill a zipEntry, it will be sent in the compressChan once
- // we're sure about the Method and CRC.
- ze := &zipEntry{
- fh: &zip.FileHeader{
- Name: dest,
- Method: method,
-
- UncompressedSize64: uint64(fileSize),
- },
- }
- ze.fh.SetModTime(z.time)
- if executable {
- ze.fh.SetMode(0700)
- }
-
- r, err := os.Open(src)
- if err != nil {
- return err
- }
-
- exec := z.rateLimit.RequestExecution()
-
- if method == zip.Deflate && fileSize >= minParallelFileSize {
- wg := new(sync.WaitGroup)
-
- // Allocate enough buffer to hold all readers. We'll limit
- // this based on actual buffer sizes in RateLimit.
- ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
-
- // Calculate the CRC in the background, since reading the entire
- // file could take a while.
- //
- // We could split this up into chuncks as well, but it's faster
- // than the compression. Due to the Go Zip API, we also need to
- // know the result before we can begin writing the compressed
- // data out to the zipfile.
- wg.Add(1)
- go z.crcFile(r, ze, exec, compressChan, wg)
-
- for start := int64(0); start < fileSize; start += parallelBlockSize {
- sr := io.NewSectionReader(r, start, parallelBlockSize)
- resultChan := make(chan io.Reader, 1)
- ze.futureReaders <- resultChan
-
- exec := z.rateLimit.RequestExecution()
-
- last := !(start+parallelBlockSize < fileSize)
- var dict []byte
- if start >= windowSize {
- dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
- }
-
- wg.Add(1)
- go z.compressPartialFile(sr, dict, last, exec, resultChan, wg)
- }
-
- close(ze.futureReaders)
-
- // Close the file handle after all readers are done
- go func(wg *sync.WaitGroup, f *os.File) {
- wg.Wait()
- f.Close()
- }(wg, r)
- } else {
- go z.compressWholeFile(ze, r, exec, compressChan)
- }
-
- return nil
-}
-
-func (z *zipWriter) crcFile(r io.Reader, ze *zipEntry, exec Execution, resultChan chan *zipEntry, wg *sync.WaitGroup) {
- defer wg.Done()
- defer exec.Finish(0)
-
- crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
- if err != nil {
- z.errors <- err
- return
- }
-
- ze.fh.CRC32 = crc.Sum32()
- resultChan <- ze
- close(resultChan)
-}
-
-func (z *zipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, exec Execution, resultChan chan io.Reader, wg *sync.WaitGroup) {
- defer wg.Done()
-
- result, err := z.compressBlock(r, dict, last)
- if err != nil {
- z.errors <- err
- return
- }
-
- exec.Finish(result.Len())
- resultChan <- result
-}
-
-func (z *zipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
- buf := new(bytes.Buffer)
- var fw *flate.Writer
- var err error
- if len(dict) > 0 {
- // There's no way to Reset a Writer with a new dictionary, so
- // don't use the Pool
- fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
- } else {
- var ok bool
- if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
- fw.Reset(buf)
- } else {
- fw, err = flate.NewWriter(buf, z.compLevel)
- }
- defer z.compressorPool.Put(fw)
- }
- if err != nil {
- return nil, err
- }
-
- _, err = io.Copy(fw, r)
- if err != nil {
- return nil, err
- }
- if last {
- fw.Close()
- } else {
- fw.Flush()
- }
-
- return buf, nil
-}
-
-func (z *zipWriter) compressWholeFile(ze *zipEntry, r *os.File, exec Execution, compressChan chan *zipEntry) {
- var bufSize int
-
- defer r.Close()
-
- crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
- if err != nil {
- z.errors <- err
- return
- }
-
- ze.fh.CRC32 = crc.Sum32()
-
- _, err = r.Seek(0, 0)
- if err != nil {
- z.errors <- err
- return
- }
-
- readFile := func(r *os.File) ([]byte, error) {
- _, err = r.Seek(0, 0)
- if err != nil {
- return nil, err
- }
-
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- return nil, err
- }
-
- return buf, nil
- }
-
- ze.futureReaders = make(chan chan io.Reader, 1)
- futureReader := make(chan io.Reader, 1)
- ze.futureReaders <- futureReader
- close(ze.futureReaders)
-
- if ze.fh.Method == zip.Deflate {
- compressed, err := z.compressBlock(r, nil, true)
- if err != nil {
- z.errors <- err
- return
- }
- if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
- futureReader <- compressed
- bufSize = compressed.Len()
- } else {
- buf, err := readFile(r)
- if err != nil {
- z.errors <- err
- return
- }
- ze.fh.Method = zip.Store
- futureReader <- bytes.NewReader(buf)
- bufSize = int(ze.fh.UncompressedSize64)
- }
- } else {
- buf, err := readFile(r)
- if err != nil {
- z.errors <- err
- return
- }
- ze.fh.Method = zip.Store
- futureReader <- bytes.NewReader(buf)
- bufSize = int(ze.fh.UncompressedSize64)
- }
-
- exec.Finish(bufSize)
- close(futureReader)
-
- compressChan <- ze
- close(compressChan)
-}
-
-func (z *zipWriter) writeDirectory(dir string) error {
- if dir != "" && !strings.HasSuffix(dir, "/") {
- dir = dir + "/"
- }
-
- for dir != "" && dir != "./" && !z.createdDirs[dir] {
- z.createdDirs[dir] = true
-
- dirHeader := &zip.FileHeader{
- Name: dir,
- }
- dirHeader.SetMode(0700 | os.ModeDir)
- dirHeader.SetModTime(z.time)
-
- ze := make(chan *zipEntry, 1)
- ze <- &zipEntry{
- fh: dirHeader,
- }
- close(ze)
- z.writeOps <- ze
-
- dir, _ = filepath.Split(dir)
- }
-
- return nil
-}
-
-func (z *zipWriter) writeSymlink(rel, file string) error {
- if z.directories {
- dir, _ := filepath.Split(rel)
- if err := z.writeDirectory(dir); err != nil {
- return err
- }
- }
-
- fileHeader := &zip.FileHeader{
- Name: rel,
- }
- fileHeader.SetModTime(z.time)
- fileHeader.SetMode(0700 | os.ModeSymlink)
-
- dest, err := os.Readlink(file)
- if err != nil {
- return err
- }
-
- ze := make(chan *zipEntry, 1)
- futureReaders := make(chan chan io.Reader, 1)
- futureReader := make(chan io.Reader, 1)
- futureReaders <- futureReader
- close(futureReaders)
- futureReader <- bytes.NewBufferString(dest)
- close(futureReader)
-
- // We didn't ask permission to execute, since this should be very short
- // but we still need to increment the outstanding buffer sizes, since
- // the read will decrement the buffer size.
- z.rateLimit.Release(-len(dest))
-
- ze <- &zipEntry{
- fh: fileHeader,
- futureReaders: futureReaders,
- }
- close(ze)
- z.writeOps <- ze
-
- return nil
-}
diff --git a/cmd/zip2zip/Android.bp b/cmd/zip2zip/Android.bp
index 476be4f..68d8bc7 100644
--- a/cmd/zip2zip/Android.bp
+++ b/cmd/zip2zip/Android.bp
@@ -14,7 +14,11 @@
blueprint_go_binary {
name: "zip2zip",
- deps: ["android-archive-zip"],
+ deps: [
+ "android-archive-zip",
+ "blueprint-pathtools",
+ "soong-jar",
+ ],
srcs: [
"zip2zip.go",
],
diff --git a/cmd/zip2zip/zip2zip.go b/cmd/zip2zip/zip2zip.go
index 815059c..e8ea9b9 100644
--- a/cmd/zip2zip/zip2zip.go
+++ b/cmd/zip2zip/zip2zip.go
@@ -24,6 +24,9 @@
"strings"
"time"
+ "github.com/google/blueprint/pathtools"
+
+ "android/soong/jar"
"android/soong/third_party/zip"
)
@@ -35,8 +38,14 @@
setTime = flag.Bool("t", false, "set timestamps to 2009-01-01 00:00:00")
staticTime = time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
+
+ excludes excludeArgs
)
+func init() {
+ flag.Var(&excludes, "x", "exclude a filespec from the output")
+}
+
func main() {
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [-s|-j] [-t] [filespec]...")
@@ -44,15 +53,14 @@
fmt.Fprintln(os.Stderr, " filespec:")
fmt.Fprintln(os.Stderr, " <name>")
fmt.Fprintln(os.Stderr, " <in_name>:<out_name>")
- fmt.Fprintln(os.Stderr, " <glob>:<out_dir>/")
+ fmt.Fprintln(os.Stderr, " <glob>[:<out_dir>]")
fmt.Fprintln(os.Stderr, "")
- fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://golang.org/pkg/path/filepath/#Match")
- fmt.Fprintln(os.Stderr, "As a special exception, '**' is supported to specify all files in the input zip.")
+ fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://godoc.org/github.com/google/blueprint/pathtools/#Match")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments.")
fmt.Fprintln(os.Stderr, "")
- fmt.Fprintln(os.Stderr, "If no filepsec is provided all files are copied (equivalent to '**').")
+ fmt.Fprintln(os.Stderr, "If no filepsec is provided all files and directories are copied.")
}
flag.Parse()
@@ -84,7 +92,9 @@
}
}()
- if err := zip2zip(&reader.Reader, writer, *sortGlobs, *sortJava, *setTime, flag.Args()); err != nil {
+ if err := zip2zip(&reader.Reader, writer, *sortGlobs, *sortJava, *setTime,
+ flag.Args(), excludes); err != nil {
+
log.Fatal(err)
}
}
@@ -94,121 +104,126 @@
newName string
}
-func zip2zip(reader *zip.Reader, writer *zip.Writer, sortGlobs, sortJava, setTime bool, args []string) error {
- if len(args) == 0 {
- // If no filespec is provided, default to copying everything
- args = []string{"**"}
- }
- for _, arg := range args {
- var input string
- var output string
+func zip2zip(reader *zip.Reader, writer *zip.Writer, sortOutput, sortJava, setTime bool,
+ includes []string, excludes []string) error {
+ matches := []pair{}
+
+ sortMatches := func(matches []pair) {
+ if sortJava {
+ sort.SliceStable(matches, func(i, j int) bool {
+ return jar.EntryNamesLess(matches[i].newName, matches[j].newName)
+ })
+ } else if sortOutput {
+ sort.SliceStable(matches, func(i, j int) bool {
+ return matches[i].newName < matches[j].newName
+ })
+ }
+ }
+
+ for _, include := range includes {
// Reserve escaping for future implementation, so make sure no
// one is using \ and expecting a certain behavior.
- if strings.Contains(arg, "\\") {
+ if strings.Contains(include, "\\") {
return fmt.Errorf("\\ characters are not currently supported")
}
- args := strings.SplitN(arg, ":", 2)
- input = args[0]
- if len(args) == 2 {
- output = args[1]
- }
+ input, output := includeSplit(include)
- matches := []pair{}
- if strings.IndexAny(input, "*?[") >= 0 {
- matchAll := input == "**"
- if !matchAll && strings.Contains(input, "**") {
- return fmt.Errorf("** is only supported on its own, not with other characters")
- }
+ var includeMatches []pair
- for _, file := range reader.File {
- match := matchAll
-
- if !match {
- var err error
- match, err = filepath.Match(input, file.Name)
- if err != nil {
- return err
- }
- }
-
- if match {
- var newName string
- if output == "" {
- newName = file.Name
- } else {
+ for _, file := range reader.File {
+ var newName string
+ if match, err := pathtools.Match(input, file.Name); err != nil {
+ return err
+ } else if match {
+ if output == "" {
+ newName = file.Name
+ } else {
+ if pathtools.IsGlob(input) {
+ // If the input is a glob then the output is a directory.
_, name := filepath.Split(file.Name)
newName = filepath.Join(output, name)
+ } else {
+ // Otherwise it is a file.
+ newName = output
}
- matches = append(matches, pair{file, newName})
}
- }
-
- if sortJava {
- jarSort(matches)
- } else if sortGlobs {
- sort.SliceStable(matches, func(i, j int) bool {
- return matches[i].newName < matches[j].newName
- })
- }
- } else {
- if output == "" {
- output = input
- }
- for _, file := range reader.File {
- if input == file.Name {
- matches = append(matches, pair{file, output})
- break
- }
+ includeMatches = append(includeMatches, pair{file, newName})
}
}
- for _, match := range matches {
- if setTime {
- match.File.SetModTime(staticTime)
- }
- if err := writer.CopyFrom(match.File, match.newName); err != nil {
+ sortMatches(includeMatches)
+ matches = append(matches, includeMatches...)
+ }
+
+ if len(includes) == 0 {
+ // implicitly match everything
+ for _, file := range reader.File {
+ matches = append(matches, pair{file, file.Name})
+ }
+ sortMatches(matches)
+ }
+
+ var matchesAfterExcludes []pair
+ seen := make(map[string]*zip.File)
+
+ for _, match := range matches {
+ // Filter out matches whose original file name matches an exclude filter
+ excluded := false
+ for _, exclude := range excludes {
+ if excludeMatch, err := pathtools.Match(exclude, match.File.Name); err != nil {
return err
+ } else if excludeMatch {
+ excluded = true
+ break
}
}
+
+ if excluded {
+ continue
+ }
+
+ // Check for duplicate output names, ignoring ones that come from the same input zip entry.
+ if prev, exists := seen[match.newName]; exists {
+ if prev != match.File {
+ return fmt.Errorf("multiple entries for %q with different contents", match.newName)
+ }
+ continue
+ }
+ seen[match.newName] = match.File
+
+ matchesAfterExcludes = append(matchesAfterExcludes, match)
+ }
+
+ for _, match := range matchesAfterExcludes {
+ if setTime {
+ match.File.SetModTime(staticTime)
+ }
+ if err := writer.CopyFrom(match.File, match.newName); err != nil {
+ return err
+ }
}
return nil
}
-func jarSort(files []pair) {
- // Treats trailing * as a prefix match
- match := func(pattern, name string) bool {
- if strings.HasSuffix(pattern, "*") {
- return strings.HasPrefix(name, strings.TrimSuffix(pattern, "*"))
- } else {
- return name == pattern
- }
+func includeSplit(s string) (string, string) {
+ split := strings.SplitN(s, ":", 2)
+ if len(split) == 2 {
+ return split[0], split[1]
+ } else {
+ return split[0], ""
}
+}
- var jarOrder = []string{
- "META-INF/",
- "META-INF/MANIFEST.MF",
- "META-INF/*",
- "*",
- }
+type excludeArgs []string
- index := func(name string) int {
- for i, pattern := range jarOrder {
- if match(pattern, name) {
- return i
- }
- }
- panic(fmt.Errorf("file %q did not match any pattern", name))
- }
+func (e *excludeArgs) String() string {
+ return strings.Join(*e, " ")
+}
- sort.SliceStable(files, func(i, j int) bool {
- diff := index(files[i].newName) - index(files[j].newName)
- if diff == 0 {
- return files[i].newName < files[j].newName
- } else {
- return diff < 0
- }
- })
+func (e *excludeArgs) Set(s string) error {
+ *e = append(*e, s)
+ return nil
}
diff --git a/cmd/zip2zip/zip2zip_test.go b/cmd/zip2zip/zip2zip_test.go
index 53c8ce2..212ab28 100644
--- a/cmd/zip2zip/zip2zip_test.go
+++ b/cmd/zip2zip/zip2zip_test.go
@@ -30,6 +30,7 @@
sortGlobs bool
sortJava bool
args []string
+ excludes []string
outputFiles []string
err error
@@ -41,13 +42,6 @@
err: fmt.Errorf("\\ characters are not currently supported"),
},
- {
- name: "unsupported **",
-
- args: []string{"a/**:b"},
-
- err: fmt.Errorf("** is only supported on its own, not with other characters"),
- },
{ // This is modelled after the update package build rules in build/make/core/Makefile
name: "filter globs",
@@ -95,16 +89,19 @@
name: "sort all",
inputFiles: []string{
+ "RADIO/",
"RADIO/a",
+ "IMAGES/",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
+ "OTA/",
"OTA/b",
"OTA/android-info.txt",
},
sortGlobs: true,
- args: []string{"**"},
+ args: []string{"**/*"},
outputFiles: []string{
"IMAGES/b.txt",
@@ -120,11 +117,14 @@
name: "sort all implicit",
inputFiles: []string{
+ "RADIO/",
"RADIO/a",
+ "IMAGES/",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
+ "OTA/",
"OTA/b",
"OTA/android-info.txt",
},
@@ -132,12 +132,15 @@
args: nil,
outputFiles: []string{
+ "IMAGES/",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/system.img",
"IMAGES/vendor.img",
+ "OTA/",
"OTA/android-info.txt",
"OTA/b",
+ "RADIO/",
"RADIO/a",
},
},
@@ -177,7 +180,7 @@
"b",
"a",
},
- args: []string{"a:a2", "**"},
+ args: []string{"a:a2", "**/*"},
outputFiles: []string{
"a2",
@@ -185,6 +188,69 @@
"a",
},
},
+ {
+ name: "multiple matches",
+
+ inputFiles: []string{
+ "a/a",
+ },
+ args: []string{"a/a", "a/*"},
+
+ outputFiles: []string{
+ "a/a",
+ },
+ },
+ {
+ name: "multiple conflicting matches",
+
+ inputFiles: []string{
+ "a/a",
+ "a/b",
+ },
+ args: []string{"a/b:a/a", "a/*"},
+
+ err: fmt.Errorf(`multiple entries for "a/a" with different contents`),
+ },
+ {
+ name: "excludes",
+
+ inputFiles: []string{
+ "a/a",
+ "a/b",
+ },
+ args: nil,
+ excludes: []string{"a/a"},
+
+ outputFiles: []string{
+ "a/b",
+ },
+ },
+ {
+ name: "excludes with include",
+
+ inputFiles: []string{
+ "a/a",
+ "a/b",
+ },
+ args: []string{"a/*"},
+ excludes: []string{"a/a"},
+
+ outputFiles: []string{
+ "a/b",
+ },
+ },
+ {
+ name: "excludes with glob",
+
+ inputFiles: []string{
+ "a/a",
+ "a/b",
+ },
+ args: []string{"a/*"},
+ excludes: []string{"a/*"},
+
+ outputFiles: nil,
+ },
}
func errorString(e error) string {
@@ -216,7 +282,7 @@
}
outputWriter := zip.NewWriter(outputBuf)
- err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, testCase.sortJava, false, testCase.args)
+ err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, testCase.sortJava, false, testCase.args, testCase.excludes)
if errorString(testCase.err) != errorString(err) {
t.Fatalf("Unexpected error:\n got: %q\nwant: %q", errorString(err), errorString(testCase.err))
}
diff --git a/cmd/soong_zip/Android.bp b/finder/Android.bp
similarity index 63%
copy from cmd/soong_zip/Android.bp
copy to finder/Android.bp
index 10896ce..b5c0e13 100644
--- a/cmd/soong_zip/Android.bp
+++ b/finder/Android.bp
@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All rights reserved.
+// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,11 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
+//
+// fast, parallel, caching implementation of `find`
+//
+
+subdirs = [
+ "cmd"
+]
+
+bootstrap_go_package {
+ name: "soong-finder",
+ pkgPath: "android/soong/finder",
srcs: [
- "soong_zip.go",
- "rate_limit.go",
+ "finder.go",
+ ],
+ testSrcs: [
+ "finder_test.go",
+ ],
+ deps: [
+ "soong-fs",
],
}
+
+
diff --git a/cmd/soong_zip/Android.bp b/finder/cmd/Android.bp
similarity index 75%
copy from cmd/soong_zip/Android.bp
copy to finder/cmd/Android.bp
index 10896ce..9dc84ae 100644
--- a/cmd/soong_zip/Android.bp
+++ b/finder/cmd/Android.bp
@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All rights reserved.
+// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,11 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//
+// fast, parallel, caching implementation of `find`
+//
+
blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
+ name: "finder",
srcs: [
- "soong_zip.go",
- "rate_limit.go",
+ "finder.go",
+ ],
+ deps: [
+ "soong-finder"
],
}
+
+
diff --git a/finder/cmd/finder.go b/finder/cmd/finder.go
new file mode 100644
index 0000000..70c1dc4
--- /dev/null
+++ b/finder/cmd/finder.go
@@ -0,0 +1,156 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "runtime/pprof"
+ "sort"
+ "strings"
+ "time"
+
+ "android/soong/finder"
+ "android/soong/fs"
+)
+
+var (
+ // configuration of what to find
+ excludeDirs string
+ filenamesToFind string
+ pruneFiles string
+
+ // other configuration
+ cpuprofile string
+ verbose bool
+ dbPath string
+ numIterations int
+)
+
+func init() {
+ flag.StringVar(&cpuprofile, "cpuprofile", "",
+ "filepath of profile file to write (optional)")
+ flag.BoolVar(&verbose, "v", false, "log additional information")
+ flag.StringVar(&dbPath, "db", "", "filepath of cache db")
+
+ flag.StringVar(&excludeDirs, "exclude-dirs", "",
+ "comma-separated list of directory names to exclude from search")
+ flag.StringVar(&filenamesToFind, "names", "",
+ "comma-separated list of filenames to find")
+ flag.StringVar(&pruneFiles, "prune-files", "",
+ "filenames that if discovered will exclude their entire directory "+
+ "(including sibling files and directories)")
+ flag.IntVar(&numIterations, "count", 1,
+ "number of times to run. This is intended for use with --cpuprofile"+
+ " , to increase profile accuracy")
+}
+
+var usage = func() {
+ fmt.Printf("usage: finder -name <fileName> --db <dbPath> <searchDirectory> [<searchDirectory>...]\n")
+ flag.PrintDefaults()
+}
+
+func main() {
+ err := run()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "%v\n", err.Error())
+ os.Exit(1)
+ }
+}
+
+func stringToList(input string) []string {
+ return strings.Split(input, ",")
+}
+
+func run() error {
+ startTime := time.Now()
+ flag.Parse()
+
+ if cpuprofile != "" {
+ f, err := os.Create(cpuprofile)
+ if err != nil {
+ return fmt.Errorf("Error opening cpuprofile: %s", err)
+ }
+ pprof.StartCPUProfile(f)
+ defer f.Close()
+ defer pprof.StopCPUProfile()
+ }
+
+ var writer io.Writer
+ if verbose {
+ writer = os.Stderr
+ } else {
+ writer = ioutil.Discard
+ }
+
+ // TODO: replace Lshortfile with Llongfile when bug 63821638 is done
+ logger := log.New(writer, "", log.Ldate|log.Lmicroseconds|log.Lshortfile)
+
+ logger.Printf("Finder starting at %v\n", startTime)
+
+ rootPaths := flag.Args()
+ if len(rootPaths) < 1 {
+ usage()
+ return fmt.Errorf(
+ "Must give at least one <searchDirectory>")
+ }
+
+ workingDir, err := os.Getwd()
+ if err != nil {
+ return err
+ }
+ params := finder.CacheParams{
+ WorkingDirectory: workingDir,
+ RootDirs: rootPaths,
+ ExcludeDirs: stringToList(excludeDirs),
+ PruneFiles: stringToList(pruneFiles),
+ IncludeFiles: stringToList(filenamesToFind),
+ }
+ if dbPath == "" {
+ usage()
+ return errors.New("Param 'db' must be nonempty")
+ }
+
+ matches := []string{}
+ for i := 0; i < numIterations; i++ {
+ matches, err = runFind(params, logger)
+ if err != nil {
+ return err
+ }
+ }
+ findDuration := time.Since(startTime)
+ logger.Printf("Found these %v inodes in %v :\n", len(matches), findDuration)
+ sort.Strings(matches)
+ for _, match := range matches {
+ fmt.Println(match)
+ }
+ logger.Printf("End of %v inodes\n", len(matches))
+ logger.Printf("Finder completed in %v\n", time.Since(startTime))
+ return nil
+}
+
+func runFind(params finder.CacheParams, logger *log.Logger) (paths []string, err error) {
+ service, err := finder.New(params, fs.OsFs, logger, dbPath)
+ if err != nil {
+ return []string{}, err
+ }
+ defer service.Shutdown()
+ return service.FindAll(), nil
+}
diff --git a/finder/finder.go b/finder/finder.go
new file mode 100644
index 0000000..2dd8e0b
--- /dev/null
+++ b/finder/finder.go
@@ -0,0 +1,1533 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package finder
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "android/soong/fs"
+)
+
+// This file provides a Finder struct that can quickly search for files satisfying
+// certain criteria.
+// This Finder gets its speed partially from parallelism and partially from caching.
+// If a Stat call returns the same result as last time, then it means Finder
+// can skip the ReadDir call for that dir.
+
+// The primary data structure used by the finder is the field Finder.nodes ,
+// which is a tree of nodes of type *pathMap .
+// Each node represents a directory on disk, along with its stats, subdirectories,
+// and contained files.
+
+// The common use case for the Finder is that the caller creates a Finder and gives
+// it the same query that was given to it in the previous execution.
+// In this situation, the major events that take place are:
+// 1. The Finder begins to load its db
+// 2. The Finder begins to stat the directories mentioned in its db (using multiple threads)
+// Calling Stat on each of these directories is generally a large fraction of the total time
+// 3. The Finder begins to construct a separate tree of nodes in each of its threads
+// 4. The Finder merges the individual node trees into the main node tree
+// 5. The Finder may call ReadDir a few times if there are a few directories that are out-of-date
+// These ReadDir calls might prompt additional Stat calls, etc
+// 6. The Finder waits for all loading to complete
+// 7. The Finder searches the cache for files matching the user's query (using multiple threads)
+
+// These are the invariants regarding concurrency:
+// 1. The public methods of Finder are threadsafe.
+// The public methods are only performance-optimized for one caller at a time, however.
+// For the moment, multiple concurrent callers shouldn't expect any better performance than
+// multiple serial callers.
+// 2. While building the node tree, only one thread may ever access the <children> collection of a
+// *pathMap at once.
+// a) The thread that accesses the <children> collection is the thread that discovers the
+// children (by reading them from the cache or by having received a response to ReadDir).
+// 1) Consequently, the thread that discovers the children also spawns requests to stat
+// subdirs.
+// b) Consequently, while building the node tree, no thread may do a lookup of its
+// *pathMap via filepath because another thread may be adding children to the
+// <children> collection of an ancestor node. Additionally, in rare cases, another thread
+// may be removing children from an ancestor node if the children were only discovered to
+// be irrelevant after calling ReadDir (which happens if a prune-file was just added).
+// 3. No query will begin to be serviced until all loading (both reading the db
+// and scanning the filesystem) is complete.
+// Tests indicate that it only takes about 10% as long to search the in-memory cache as to
+// generate it, making this not a huge loss in performance.
+// 4. The parsing of the db and the initial setup of the pathMap tree must complete before
+// beginning to call listDirSync (because listDirSync can create new entries in the pathMap)
+
+// see cmd/finder.go or finder_test.go for usage examples
+
+// Update versionString whenever making a backwards-incompatible change to the cache file format
+const versionString = "Android finder version 1"
+
+// a CacheParams specifies which files and directories the user wishes be scanned and
+// potentially added to the cache
+type CacheParams struct {
+ // WorkingDirectory is used as a base for any relative file paths given to the Finder
+ WorkingDirectory string
+
+ // RootDirs are the root directories used to initiate the search
+ RootDirs []string
+
+ // ExcludeDirs are directory names that if encountered are removed from the search
+ ExcludeDirs []string
+
+ // PruneFiles are file names that if encountered prune their entire directory
+ // (including siblings)
+ PruneFiles []string
+
+ // IncludeFiles are file names to include as matches
+ IncludeFiles []string
+}
+
+// a cacheConfig stores the inputs that determine what should be included in the cache
+type cacheConfig struct {
+ CacheParams
+
+ // FilesystemView is a unique identifier telling which parts of which file systems
+ // are readable by the Finder. In practice its value is essentially username@hostname.
+ // FilesystemView is set to ensure that a cache file copied to another host or
+ // found by another user doesn't inadvertently get reused.
+ FilesystemView string
+}
+
+func (p *cacheConfig) Dump() ([]byte, error) {
+ bytes, err := json.Marshal(p)
+ return bytes, err
+}
+
+// a cacheMetadata stores version information about the cache
+type cacheMetadata struct {
+ // The Version enables the Finder to determine whether it can even parse the file
+ // If the version changes, the entire cache file must be regenerated
+ Version string
+
+ // The CacheParams enables the Finder to determine whether the parameters match
+ // If the CacheParams change, the Finder can choose how much of the cache file to reuse
+ // (although in practice, the Finder will probably choose to ignore the entire file anyway)
+ Config cacheConfig
+}
+
+type Logger interface {
+ Output(calldepth int, s string) error
+}
+
+// the Finder is the main struct that callers will want to use
+type Finder struct {
+ // configuration
+ DbPath string
+ numDbLoadingThreads int
+ numSearchingThreads int
+ cacheMetadata cacheMetadata
+ logger Logger
+ filesystem fs.FileSystem
+
+ // temporary state
+ threadPool *threadPool
+ mutex sync.Mutex
+ fsErrs []fsErr
+ errlock sync.Mutex
+ shutdownWaitgroup sync.WaitGroup
+
+ // non-temporary state
+ modifiedFlag int32
+ nodes pathMap
+}
+
+var defaultNumThreads = runtime.NumCPU() * 2
+
+// New creates a new Finder for use
+func New(cacheParams CacheParams, filesystem fs.FileSystem,
+ logger Logger, dbPath string) (f *Finder, err error) {
+ return newImpl(cacheParams, filesystem, logger, dbPath, defaultNumThreads)
+}
+
+// newImpl is like New but accepts more params
+func newImpl(cacheParams CacheParams, filesystem fs.FileSystem,
+ logger Logger, dbPath string, numThreads int) (f *Finder, err error) {
+ numDbLoadingThreads := numThreads
+ numSearchingThreads := numThreads
+
+ metadata := cacheMetadata{
+ Version: versionString,
+ Config: cacheConfig{
+ CacheParams: cacheParams,
+ FilesystemView: filesystem.ViewId(),
+ },
+ }
+
+ f = &Finder{
+ numDbLoadingThreads: numDbLoadingThreads,
+ numSearchingThreads: numSearchingThreads,
+ cacheMetadata: metadata,
+ logger: logger,
+ filesystem: filesystem,
+
+ nodes: *newPathMap("/"),
+ DbPath: dbPath,
+
+ shutdownWaitgroup: sync.WaitGroup{},
+ }
+
+ f.loadFromFilesystem()
+
+ // check for any filesystem errors
+ err = f.getErr()
+ if err != nil {
+ return nil, err
+ }
+
+ // confirm that every path mentioned in the CacheConfig exists
+ for _, path := range cacheParams.RootDirs {
+ if !filepath.IsAbs(path) {
+ path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
+ }
+ node := f.nodes.GetNode(filepath.Clean(path), false)
+ if node == nil || node.ModTime == 0 {
+ return nil, fmt.Errorf("path %v was specified to be included in the cache but does not exist\n", path)
+ }
+ }
+
+ return f, nil
+}
+
+// FindNamed searches for every cached file
+func (f *Finder) FindAll() []string {
+ return f.FindAt("/")
+}
+
+// FindNamed searches for every cached file under <rootDir>
+func (f *Finder) FindAt(rootDir string) []string {
+ filter := func(entries DirEntries) (dirNames []string, fileNames []string) {
+ return entries.DirNames, entries.FileNames
+ }
+ return f.FindMatching(rootDir, filter)
+}
+
+// FindNamed searches for every cached file named <fileName>
+func (f *Finder) FindNamed(fileName string) []string {
+ return f.FindNamedAt("/", fileName)
+}
+
+// FindNamedAt searches under <rootPath> for every file named <fileName>
+// The reason a caller might use FindNamedAt instead of FindNamed is if they want
+// to limit their search to a subset of the cache
+func (f *Finder) FindNamedAt(rootPath string, fileName string) []string {
+ filter := func(entries DirEntries) (dirNames []string, fileNames []string) {
+ matches := []string{}
+ for _, foundName := range entries.FileNames {
+ if foundName == fileName {
+ matches = append(matches, foundName)
+ }
+ }
+ return entries.DirNames, matches
+
+ }
+ return f.FindMatching(rootPath, filter)
+}
+
+// FindFirstNamed searches for every file named <fileName>
+// Whenever it finds a match, it stops search subdirectories
+func (f *Finder) FindFirstNamed(fileName string) []string {
+ return f.FindFirstNamedAt("/", fileName)
+}
+
+// FindFirstNamedAt searches for every file named <fileName>
+// Whenever it finds a match, it stops search subdirectories
+func (f *Finder) FindFirstNamedAt(rootPath string, fileName string) []string {
+ filter := func(entries DirEntries) (dirNames []string, fileNames []string) {
+ matches := []string{}
+ for _, foundName := range entries.FileNames {
+ if foundName == fileName {
+ matches = append(matches, foundName)
+ }
+ }
+
+ if len(matches) > 0 {
+ return []string{}, matches
+ }
+ return entries.DirNames, matches
+ }
+ return f.FindMatching(rootPath, filter)
+}
+
+// FindMatching is the most general exported function for searching for files in the cache
+// The WalkFunc will be invoked repeatedly and is expected to modify the provided DirEntries
+// in place, removing file paths and directories as desired.
+// WalkFunc will be invoked potentially many times in parallel, and must be threadsafe.
+func (f *Finder) FindMatching(rootPath string, filter WalkFunc) []string {
+ // set up some parameters
+ scanStart := time.Now()
+ var isRel bool
+ workingDir := f.cacheMetadata.Config.WorkingDirectory
+
+ isRel = !filepath.IsAbs(rootPath)
+ if isRel {
+ rootPath = filepath.Join(workingDir, rootPath)
+ }
+
+ rootPath = filepath.Clean(rootPath)
+
+ // ensure nothing else is using the Finder
+ f.verbosef("FindMatching waiting for finder to be idle\n")
+ f.lock()
+ defer f.unlock()
+
+ node := f.nodes.GetNode(rootPath, false)
+ if node == nil {
+ f.verbosef("No data for path %v ; apparently not included in cache params: %v\n",
+ rootPath, f.cacheMetadata.Config.CacheParams)
+ // path is not found; don't do a search
+ return []string{}
+ }
+
+ // search for matching files
+ f.verbosef("Finder finding %v using cache\n", rootPath)
+ results := f.findInCacheMultithreaded(node, filter, f.numSearchingThreads)
+
+ // format and return results
+ if isRel {
+ for i := 0; i < len(results); i++ {
+ results[i] = strings.Replace(results[i], workingDir+"/", "", 1)
+ }
+ }
+ sort.Strings(results)
+ f.verbosef("Found %v files under %v in %v using cache\n",
+ len(results), rootPath, time.Since(scanStart))
+ return results
+}
+
+// Shutdown declares that the finder is no longer needed and waits for its cleanup to complete
+// Currently, that only entails waiting for the database dump to complete.
+func (f *Finder) Shutdown() {
+ f.waitForDbDump()
+}
+
+// End of public api
+
+func (f *Finder) goDumpDb() {
+ if f.wasModified() {
+ f.shutdownWaitgroup.Add(1)
+ go func() {
+ err := f.dumpDb()
+ if err != nil {
+ f.verbosef("%v\n", err)
+ }
+ f.shutdownWaitgroup.Done()
+ }()
+ } else {
+ f.verbosef("Skipping dumping unmodified db\n")
+ }
+}
+
+func (f *Finder) waitForDbDump() {
+ f.shutdownWaitgroup.Wait()
+}
+
+// joinCleanPaths is like filepath.Join but is faster because
+// joinCleanPaths doesn't have to support paths ending in "/" or containing ".."
+func joinCleanPaths(base string, leaf string) string {
+ if base == "" {
+ return leaf
+ }
+ if base == "/" {
+ return base + leaf
+ }
+ if leaf == "" {
+ return base
+ }
+ return base + "/" + leaf
+}
+
+func (f *Finder) verbosef(format string, args ...interface{}) {
+ f.logger.Output(2, fmt.Sprintf(format, args...))
+}
+
+// loadFromFilesystem populates the in-memory cache based on the contents of the filesystem
+func (f *Finder) loadFromFilesystem() {
+ f.threadPool = newThreadPool(f.numDbLoadingThreads)
+
+ err := f.startFromExternalCache()
+ if err != nil {
+ f.startWithoutExternalCache()
+ }
+
+ f.goDumpDb()
+
+ f.threadPool = nil
+}
+
+func (f *Finder) startFind(path string) {
+ if !filepath.IsAbs(path) {
+ path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
+ }
+ node := f.nodes.GetNode(path, true)
+ f.statDirAsync(node)
+}
+
+func (f *Finder) lock() {
+ f.mutex.Lock()
+}
+
+func (f *Finder) unlock() {
+ f.mutex.Unlock()
+}
+
+// a statResponse is the relevant portion of the response from the filesystem to a Stat call
+type statResponse struct {
+ ModTime int64
+ Inode uint64
+ Device uint64
+}
+
+// a pathAndStats stores a path and its stats
+type pathAndStats struct {
+ statResponse
+
+ Path string
+}
+
+// a dirFullInfo stores all of the relevant information we know about a directory
+type dirFullInfo struct {
+ pathAndStats
+
+ FileNames []string
+}
+
+// a PersistedDirInfo is the information about a dir that we save to our cache on disk
+type PersistedDirInfo struct {
+ // These field names are short because they are repeated many times in the output json file
+ P string // path
+ T int64 // modification time
+ I uint64 // inode number
+ F []string // relevant filenames contained
+}
+
+// a PersistedDirs is the information that we persist for a group of dirs
+type PersistedDirs struct {
+ // the device on which each directory is stored
+ Device uint64
+ // the common root path to which all contained dirs are relative
+ Root string
+ // the directories themselves
+ Dirs []PersistedDirInfo
+}
+
+// a CacheEntry is the smallest unit that can be read and parsed from the cache (on disk) at a time
+type CacheEntry []PersistedDirs
+
+// a DirEntries lists the files and directories contained directly within a specific directory
+type DirEntries struct {
+ Path string
+
+ // elements of DirNames are just the dir names; they don't include any '/' character
+ DirNames []string
+ // elements of FileNames are just the file names; they don't include '/' character
+ FileNames []string
+}
+
+// a WalkFunc is the type that is passed into various Find functions for determining which
+// directories the caller wishes be walked. The WalkFunc is expected to decide which
+// directories to walk and which files to consider as matches to the original query.
+type WalkFunc func(DirEntries) (dirs []string, files []string)
+
+// a mapNode stores the relevant stats about a directory to be stored in a pathMap
+type mapNode struct {
+ statResponse
+ FileNames []string
+}
+
+// a pathMap implements the directory tree structure of nodes
+type pathMap struct {
+ mapNode
+
+ path string
+
+ children map[string]*pathMap
+
+ // number of descendent nodes, including self
+ approximateNumDescendents int
+}
+
+func newPathMap(path string) *pathMap {
+ result := &pathMap{path: path, children: make(map[string]*pathMap, 4),
+ approximateNumDescendents: 1}
+ return result
+}
+
+// GetNode returns the node at <path>
+func (m *pathMap) GetNode(path string, createIfNotFound bool) *pathMap {
+ if len(path) > 0 && path[0] == '/' {
+ path = path[1:]
+ }
+
+ node := m
+ for {
+ if path == "" {
+ return node
+ }
+
+ index := strings.Index(path, "/")
+ var firstComponent string
+ if index >= 0 {
+ firstComponent = path[:index]
+ path = path[index+1:]
+ } else {
+ firstComponent = path
+ path = ""
+ }
+
+ child, found := node.children[firstComponent]
+
+ if !found {
+ if createIfNotFound {
+ child = node.newChild(firstComponent)
+ } else {
+ return nil
+ }
+ }
+
+ node = child
+ }
+}
+
+func (m *pathMap) newChild(name string) (child *pathMap) {
+ path := joinCleanPaths(m.path, name)
+ newChild := newPathMap(path)
+ m.children[name] = newChild
+
+ return m.children[name]
+}
+
+func (m *pathMap) UpdateNumDescendents() int {
+ count := 1
+ for _, child := range m.children {
+ count += child.approximateNumDescendents
+ }
+ m.approximateNumDescendents = count
+ return count
+}
+
+func (m *pathMap) UpdateNumDescendentsRecursive() {
+ for _, child := range m.children {
+ child.UpdateNumDescendentsRecursive()
+ }
+ m.UpdateNumDescendents()
+}
+
+func (m *pathMap) MergeIn(other *pathMap) {
+ for key, theirs := range other.children {
+ ours, found := m.children[key]
+ if found {
+ ours.MergeIn(theirs)
+ } else {
+ m.children[key] = theirs
+ }
+ }
+ if other.ModTime != 0 {
+ m.mapNode = other.mapNode
+ }
+ m.UpdateNumDescendents()
+}
+
+func (m *pathMap) DumpAll() []dirFullInfo {
+ results := []dirFullInfo{}
+ m.dumpInto("", &results)
+ return results
+}
+
+func (m *pathMap) dumpInto(path string, results *[]dirFullInfo) {
+ *results = append(*results,
+ dirFullInfo{
+ pathAndStats{statResponse: m.statResponse, Path: path},
+ m.FileNames},
+ )
+ for key, child := range m.children {
+ childPath := joinCleanPaths(path, key)
+ if len(childPath) == 0 || childPath[0] != '/' {
+ childPath = "/" + childPath
+ }
+ child.dumpInto(childPath, results)
+ }
+}
+
+// a semaphore can be locked by up to <capacity> callers at once
+type semaphore struct {
+ pool chan bool
+}
+
+func newSemaphore(capacity int) *semaphore {
+ return &semaphore{pool: make(chan bool, capacity)}
+}
+
+func (l *semaphore) Lock() {
+ l.pool <- true
+}
+
+func (l *semaphore) Unlock() {
+ <-l.pool
+}
+
+// A threadPool runs goroutines and supports throttling and waiting.
+// Without throttling, Go may exhaust the maximum number of various resources, such as
+// threads or file descriptors, and crash the program.
+type threadPool struct {
+ receivedRequests sync.WaitGroup
+ activeRequests semaphore
+}
+
+func newThreadPool(maxNumConcurrentThreads int) *threadPool {
+ return &threadPool{
+ receivedRequests: sync.WaitGroup{},
+ activeRequests: *newSemaphore(maxNumConcurrentThreads),
+ }
+}
+
+// Run requests to run the given function in its own goroutine
+func (p *threadPool) Run(function func()) {
+ p.receivedRequests.Add(1)
+ // If Run() was called from within a goroutine spawned by this threadPool,
+ // then we may need to return from Run() before having capacity to actually
+ // run <function>.
+ //
+ // It's possible that the body of <function> contains a statement (such as a syscall)
+ // that will cause Go to pin it to a thread, or will contain a statement that uses
+ // another resource that is in short supply (such as a file descriptor), so we can't
+ // actually run <function> until we have capacity.
+ //
+ // However, the semaphore used for synchronization is implemented via a channel and
+ // shouldn't require a new thread for each access.
+ go func() {
+ p.activeRequests.Lock()
+ function()
+ p.activeRequests.Unlock()
+ p.receivedRequests.Done()
+ }()
+}
+
+// Wait waits until all goroutines are done, just like sync.WaitGroup's Wait
+func (p *threadPool) Wait() {
+ p.receivedRequests.Wait()
+}
+
+type fsErr struct {
+ path string
+ err error
+}
+
+func (e fsErr) String() string {
+ return e.path + ": " + e.err.Error()
+}
+
+func (f *Finder) serializeCacheEntry(dirInfos []dirFullInfo) ([]byte, error) {
+ // group each dirFullInfo by its Device, to avoid having to repeat it in the output
+ dirsByDevice := map[uint64][]PersistedDirInfo{}
+ for _, entry := range dirInfos {
+ _, found := dirsByDevice[entry.Device]
+ if !found {
+ dirsByDevice[entry.Device] = []PersistedDirInfo{}
+ }
+ dirsByDevice[entry.Device] = append(dirsByDevice[entry.Device],
+ PersistedDirInfo{P: entry.Path, T: entry.ModTime, I: entry.Inode, F: entry.FileNames})
+ }
+
+ cacheEntry := CacheEntry{}
+
+ for device, infos := range dirsByDevice {
+ // find common prefix
+ prefix := ""
+ if len(infos) > 0 {
+ prefix = infos[0].P
+ }
+ for _, info := range infos {
+ for !strings.HasPrefix(info.P+"/", prefix+"/") {
+ prefix = filepath.Dir(prefix)
+ if prefix == "/" {
+ break
+ }
+ }
+ }
+ // remove common prefix
+ for i := range infos {
+ suffix := strings.Replace(infos[i].P, prefix, "", 1)
+ if len(suffix) > 0 && suffix[0] == '/' {
+ suffix = suffix[1:]
+ }
+ infos[i].P = suffix
+ }
+
+ // turn the map (keyed by device) into a list of structs with labeled fields
+ // this is to improve readability of the output
+ cacheEntry = append(cacheEntry, PersistedDirs{Device: device, Root: prefix, Dirs: infos})
+ }
+
+ // convert to json.
+ // it would save some space to use a different format than json for the db file,
+ // but the space and time savings are small, and json is easy for humans to read
+ bytes, err := json.Marshal(cacheEntry)
+ return bytes, err
+}
+
+func (f *Finder) parseCacheEntry(bytes []byte) ([]dirFullInfo, error) {
+ var cacheEntry CacheEntry
+ err := json.Unmarshal(bytes, &cacheEntry)
+ if err != nil {
+ return nil, err
+ }
+
+ // convert from a CacheEntry to a []dirFullInfo (by copying a few fields)
+ capacity := 0
+ for _, element := range cacheEntry {
+ capacity += len(element.Dirs)
+ }
+ nodes := make([]dirFullInfo, capacity)
+ count := 0
+ for _, element := range cacheEntry {
+ for _, dir := range element.Dirs {
+ path := joinCleanPaths(element.Root, dir.P)
+
+ nodes[count] = dirFullInfo{
+ pathAndStats: pathAndStats{
+ statResponse: statResponse{
+ ModTime: dir.T, Inode: dir.I, Device: element.Device,
+ },
+ Path: path},
+ FileNames: dir.F}
+ count++
+ }
+ }
+ return nodes, nil
+}
+
+// We use the following separator byte to distinguish individually parseable blocks of json
+// because we know this separator won't appear in the json that we're parsing.
+//
+// The newline byte can only appear in a UTF-8 stream if the newline character appears, because:
+// - The newline character is encoded as "0000 1010" in binary ("0a" in hex)
+// - UTF-8 dictates that bytes beginning with a "0" bit are never emitted as part of a multibyte
+// character.
+//
+// We know that the newline character will never appear in our json string, because:
+// - If a newline character appears as part of a data string, then json encoding will
+// emit two characters instead: '\' and 'n'.
+// - The json encoder that we use doesn't emit the optional newlines between any of its
+// other outputs.
+const lineSeparator = byte('\n')
+
+func (f *Finder) readLine(reader *bufio.Reader) ([]byte, error) {
+ return reader.ReadBytes(lineSeparator)
+}
+
+// validateCacheHeader reads the cache header from cacheReader and tells whether the cache is compatible with this Finder
+func (f *Finder) validateCacheHeader(cacheReader *bufio.Reader) bool {
+ cacheVersionBytes, err := f.readLine(cacheReader)
+ if err != nil {
+ f.verbosef("Failed to read database header; database is invalid\n")
+ return false
+ }
+ if len(cacheVersionBytes) > 0 && cacheVersionBytes[len(cacheVersionBytes)-1] == lineSeparator {
+ cacheVersionBytes = cacheVersionBytes[:len(cacheVersionBytes)-1]
+ }
+ cacheVersionString := string(cacheVersionBytes)
+ currentVersion := f.cacheMetadata.Version
+ if cacheVersionString != currentVersion {
+ f.verbosef("Version changed from %q to %q, database is not applicable\n", cacheVersionString, currentVersion)
+ return false
+ }
+
+ cacheParamBytes, err := f.readLine(cacheReader)
+ if err != nil {
+ f.verbosef("Failed to read database search params; database is invalid\n")
+ return false
+ }
+
+ if len(cacheParamBytes) > 0 && cacheParamBytes[len(cacheParamBytes)-1] == lineSeparator {
+ cacheParamBytes = cacheParamBytes[:len(cacheParamBytes)-1]
+ }
+
+ currentParamBytes, err := f.cacheMetadata.Config.Dump()
+ if err != nil {
+ panic("Finder failed to serialize its parameters")
+ }
+ cacheParamString := string(cacheParamBytes)
+ currentParamString := string(currentParamBytes)
+ if cacheParamString != currentParamString {
+ f.verbosef("Params changed from %q to %q, database is not applicable\n", cacheParamString, currentParamString)
+ return false
+ }
+ return true
+}
+
+// loadBytes compares the cache info in <data> to the state of the filesystem
+// loadBytes returns a map representing <data> and also a slice of dirs that need to be re-walked
+func (f *Finder) loadBytes(id int, data []byte) (m *pathMap, dirsToWalk []string, err error) {
+
+ helperStartTime := time.Now()
+
+ cachedNodes, err := f.parseCacheEntry(data)
+ if err != nil {
+ return nil, nil, fmt.Errorf("Failed to parse block %v: %v\n", id, err.Error())
+ }
+
+ unmarshalDate := time.Now()
+ f.verbosef("Unmarshaled %v objects for %v in %v\n",
+ len(cachedNodes), id, unmarshalDate.Sub(helperStartTime))
+
+ tempMap := newPathMap("/")
+ stats := make([]statResponse, len(cachedNodes))
+
+ for i, node := range cachedNodes {
+ // check the file system for an updated timestamp
+ stats[i] = f.statDirSync(node.Path)
+ }
+
+ dirsToWalk = []string{}
+ for i, cachedNode := range cachedNodes {
+ updated := stats[i]
+ // save the cached value
+ container := tempMap.GetNode(cachedNode.Path, true)
+ container.mapNode = mapNode{statResponse: updated}
+
+ // if the metadata changed and the directory still exists, then
+ // make a note to walk it later
+ if !f.isInfoUpToDate(cachedNode.statResponse, updated) && updated.ModTime != 0 {
+ f.setModified()
+ // make a note that the directory needs to be walked
+ dirsToWalk = append(dirsToWalk, cachedNode.Path)
+ } else {
+ container.mapNode.FileNames = cachedNode.FileNames
+ }
+ }
+ // count the number of nodes to improve our understanding of the shape of the tree,
+ // thereby improving parallelism of subsequent searches
+ tempMap.UpdateNumDescendentsRecursive()
+
+ f.verbosef("Statted inodes of block %v in %v\n", id, time.Now().Sub(unmarshalDate))
+ return tempMap, dirsToWalk, nil
+}
+
+// startFromExternalCache loads the cache database from disk
+// startFromExternalCache waits to return until the load of the cache db is complete, but
+// startFromExternalCache does not wait for all every listDir() or statDir() request to complete
+func (f *Finder) startFromExternalCache() (err error) {
+ startTime := time.Now()
+ dbPath := f.DbPath
+
+ // open cache file and validate its header
+ reader, err := f.filesystem.Open(dbPath)
+ if err != nil {
+ return errors.New("No data to load from database\n")
+ }
+ bufferedReader := bufio.NewReader(reader)
+ if !f.validateCacheHeader(bufferedReader) {
+ return errors.New("Cache header does not match")
+ }
+ f.verbosef("Database header matches, will attempt to use database %v\n", f.DbPath)
+
+ // read the file and spawn threads to process it
+ nodesToWalk := [][]*pathMap{}
+ mainTree := newPathMap("/")
+
+ // read the blocks and stream them into <blockChannel>
+ type dataBlock struct {
+ id int
+ err error
+ data []byte
+ }
+ blockChannel := make(chan dataBlock, f.numDbLoadingThreads)
+ readBlocks := func() {
+ index := 0
+ for {
+ // It takes some time to unmarshal the input from json, so we want
+ // to unmarshal it in parallel. In order to find valid places to
+ // break the input, we scan for the line separators that we inserted
+ // (for this purpose) when we dumped the database.
+ data, err := f.readLine(bufferedReader)
+ var response dataBlock
+ done := false
+ if err != nil && err != io.EOF {
+ response = dataBlock{id: index, err: err, data: nil}
+ done = true
+ } else {
+ done = (err == io.EOF)
+ response = dataBlock{id: index, err: nil, data: data}
+ }
+ blockChannel <- response
+ index++
+ duration := time.Since(startTime)
+ f.verbosef("Read block %v after %v\n", index, duration)
+ if done {
+ f.verbosef("Read %v blocks in %v\n", index, duration)
+ close(blockChannel)
+ return
+ }
+ }
+ }
+ go readBlocks()
+
+ // Read from <blockChannel> and stream the responses into <resultChannel>.
+ type workResponse struct {
+ id int
+ err error
+ tree *pathMap
+ updatedDirs []string
+ }
+ resultChannel := make(chan workResponse)
+ processBlocks := func() {
+ numProcessed := 0
+ threadPool := newThreadPool(f.numDbLoadingThreads)
+ for {
+ // get a block to process
+ block, received := <-blockChannel
+ if !received {
+ break
+ }
+
+ if block.err != nil {
+ resultChannel <- workResponse{err: block.err}
+ break
+ }
+ numProcessed++
+ // wait until there is CPU available to process it
+ threadPool.Run(
+ func() {
+ processStartTime := time.Now()
+ f.verbosef("Starting to process block %v after %v\n",
+ block.id, processStartTime.Sub(startTime))
+ tempMap, updatedDirs, err := f.loadBytes(block.id, block.data)
+ var response workResponse
+ if err != nil {
+ f.verbosef(
+ "Block %v failed to parse with error %v\n",
+ block.id, err)
+ response = workResponse{err: err}
+ } else {
+ response = workResponse{
+ id: block.id,
+ err: nil,
+ tree: tempMap,
+ updatedDirs: updatedDirs,
+ }
+ }
+ f.verbosef("Processed block %v in %v\n",
+ block.id, time.Since(processStartTime),
+ )
+ resultChannel <- response
+ },
+ )
+ }
+ threadPool.Wait()
+ f.verbosef("Finished processing %v blocks in %v\n",
+ numProcessed, time.Since(startTime))
+ close(resultChannel)
+ }
+ go processBlocks()
+
+ // Read from <resultChannel> and use the results
+ combineResults := func() (err error) {
+ for {
+ result, received := <-resultChannel
+ if !received {
+ break
+ }
+ if err != nil {
+ // In case of an error, wait for work to complete before
+ // returning the error. This ensures that any subsequent
+ // work doesn't need to compete for resources (and possibly
+ // fail due to, for example, a filesystem limit on the number of
+ // concurrently open files) with past work.
+ continue
+ }
+ if result.err != nil {
+ err = result.err
+ continue
+ }
+ // update main tree
+ mainTree.MergeIn(result.tree)
+ // record any new directories that we will need to Stat()
+ updatedNodes := make([]*pathMap, len(result.updatedDirs))
+ for j, dir := range result.updatedDirs {
+ node := mainTree.GetNode(dir, false)
+ updatedNodes[j] = node
+ }
+ nodesToWalk = append(nodesToWalk, updatedNodes)
+ }
+ return err
+ }
+ err = combineResults()
+ if err != nil {
+ return err
+ }
+
+ f.nodes = *mainTree
+
+ // after having loaded the entire db and therefore created entries for
+ // the directories we know of, now it's safe to start calling ReadDir on
+ // any updated directories
+ for i := range nodesToWalk {
+ f.listDirsAsync(nodesToWalk[i])
+ }
+ f.verbosef("Loaded db and statted known dirs in %v\n", time.Since(startTime))
+ f.threadPool.Wait()
+ f.verbosef("Loaded db and statted all dirs in %v\n", time.Now().Sub(startTime))
+
+ return err
+}
+
+// startWithoutExternalCache starts scanning the filesystem according to the cache config
+// startWithoutExternalCache should be called if startFromExternalCache is not applicable
+func (f *Finder) startWithoutExternalCache() {
+ startTime := time.Now()
+ configDirs := f.cacheMetadata.Config.RootDirs
+
+ // clean paths
+ candidates := make([]string, len(configDirs))
+ for i, dir := range configDirs {
+ candidates[i] = filepath.Clean(dir)
+ }
+ // remove duplicates
+ dirsToScan := make([]string, 0, len(configDirs))
+ for _, candidate := range candidates {
+ include := true
+ for _, included := range dirsToScan {
+ if included == "/" || strings.HasPrefix(candidate+"/", included+"/") {
+ include = false
+ break
+ }
+ }
+ if include {
+ dirsToScan = append(dirsToScan, candidate)
+ }
+ }
+
+ // start searching finally
+ for _, path := range dirsToScan {
+ f.verbosef("Starting find of %v\n", path)
+ f.startFind(path)
+ }
+
+ f.threadPool.Wait()
+
+ f.verbosef("Scanned filesystem (not using cache) in %v\n", time.Now().Sub(startTime))
+}
+
+// isInfoUpToDate tells whether <new> can confirm that results computed at <old> are still valid
+func (f *Finder) isInfoUpToDate(old statResponse, new statResponse) (equal bool) {
+ if old.Inode != new.Inode {
+ return false
+ }
+ if old.ModTime != new.ModTime {
+ return false
+ }
+ if old.Device != new.Device {
+ return false
+ }
+ return true
+}
+
+func (f *Finder) wasModified() bool {
+ return atomic.LoadInt32(&f.modifiedFlag) > 0
+}
+
+func (f *Finder) setModified() {
+ var newVal int32
+ newVal = 1
+ atomic.StoreInt32(&f.modifiedFlag, newVal)
+}
+
+// sortedDirEntries exports directory entries to facilitate dumping them to the external cache
+func (f *Finder) sortedDirEntries() []dirFullInfo {
+ startTime := time.Now()
+ nodes := make([]dirFullInfo, 0)
+ for _, node := range f.nodes.DumpAll() {
+ if node.ModTime != 0 {
+ nodes = append(nodes, node)
+ }
+ }
+ discoveryDate := time.Now()
+ f.verbosef("Generated %v cache entries in %v\n", len(nodes), discoveryDate.Sub(startTime))
+ less := func(i int, j int) bool {
+ return nodes[i].Path < nodes[j].Path
+ }
+ sort.Slice(nodes, less)
+ sortDate := time.Now()
+ f.verbosef("Sorted %v cache entries in %v\n", len(nodes), sortDate.Sub(discoveryDate))
+
+ return nodes
+}
+
+// serializeDb converts the cache database into a form to save to disk
+func (f *Finder) serializeDb() ([]byte, error) {
+ // sort dir entries
+ var entryList = f.sortedDirEntries()
+
+ // Generate an output file that can be conveniently loaded using the same number of threads
+ // as were used in this execution (because presumably that will be the number of threads
+ // used in the next execution too)
+
+ // generate header
+ header := []byte{}
+ header = append(header, []byte(f.cacheMetadata.Version)...)
+ header = append(header, lineSeparator)
+ configDump, err := f.cacheMetadata.Config.Dump()
+ if err != nil {
+ return nil, err
+ }
+ header = append(header, configDump...)
+
+ // serialize individual blocks in parallel
+ numBlocks := f.numDbLoadingThreads
+ if numBlocks > len(entryList) {
+ numBlocks = len(entryList)
+ }
+ blocks := make([][]byte, 1+numBlocks)
+ blocks[0] = header
+ blockMin := 0
+ wg := sync.WaitGroup{}
+ var errLock sync.Mutex
+
+ for i := 1; i <= numBlocks; i++ {
+ // identify next block
+ blockMax := len(entryList) * i / numBlocks
+ block := entryList[blockMin:blockMax]
+
+ // process block
+ wg.Add(1)
+ go func(index int, block []dirFullInfo) {
+ byteBlock, subErr := f.serializeCacheEntry(block)
+ f.verbosef("Serialized block %v into %v bytes\n", index, len(byteBlock))
+ if subErr != nil {
+ f.verbosef("%v\n", subErr.Error())
+ errLock.Lock()
+ err = subErr
+ errLock.Unlock()
+ } else {
+ blocks[index] = byteBlock
+ }
+ wg.Done()
+ }(i, block)
+
+ blockMin = blockMax
+ }
+
+ wg.Wait()
+
+ if err != nil {
+ return nil, err
+ }
+
+ content := bytes.Join(blocks, []byte{lineSeparator})
+
+ return content, nil
+}
+
+// dumpDb saves the cache database to disk
+func (f *Finder) dumpDb() error {
+ startTime := time.Now()
+ f.verbosef("Dumping db\n")
+
+ tempPath := f.DbPath + ".tmp"
+
+ bytes, err := f.serializeDb()
+ if err != nil {
+ return err
+ }
+ serializeDate := time.Now()
+ f.verbosef("Serialized db in %v\n", serializeDate.Sub(startTime))
+ // dump file and atomically move
+ err = f.filesystem.WriteFile(tempPath, bytes, 0777)
+ if err != nil {
+ return err
+ }
+ err = f.filesystem.Rename(tempPath, f.DbPath)
+ if err != nil {
+ return err
+ }
+
+ f.verbosef("Wrote db in %v\n", time.Now().Sub(serializeDate))
+ return nil
+
+}
+
+// canIgnoreFsErr checks for certain classes of filesystem errors that are safe to ignore
+func (f *Finder) canIgnoreFsErr(err error) bool {
+ pathErr, isPathErr := err.(*os.PathError)
+ if !isPathErr {
+ // Don't recognize this error
+ return false
+ }
+ if os.IsPermission(pathErr) {
+ // Permission errors are ignored:
+ // https://issuetracker.google.com/37553659
+ // https://github.com/google/kati/pull/116
+ return true
+ }
+ if pathErr.Err == os.ErrNotExist {
+ // If a directory doesn't exist, that generally means the cache is out-of-date
+ return true
+ }
+ // Don't recognize this error
+ return false
+}
+
+// onFsError should be called whenever a potentially fatal error is returned from a filesystem call
+func (f *Finder) onFsError(path string, err error) {
+ if !f.canIgnoreFsErr(err) {
+ // We could send the errors through a channel instead, although that would cause this call
+ // to block unless we preallocated a sufficient buffer or spawned a reader thread.
+ // Although it wouldn't be too complicated to spawn a reader thread, it's still slightly
+ // more convenient to use a lock. Only in an unusual situation should this code be
+ // invoked anyway.
+ f.errlock.Lock()
+ f.fsErrs = append(f.fsErrs, fsErr{path: path, err: err})
+ f.errlock.Unlock()
+ }
+}
+
+// discardErrsForPrunedPaths removes any errors for paths that are no longer included in the cache
+func (f *Finder) discardErrsForPrunedPaths() {
+ // This function could be somewhat inefficient due to being single-threaded,
+ // but the length of f.fsErrs should be approximately 0, so it shouldn't take long anyway.
+ relevantErrs := make([]fsErr, 0, len(f.fsErrs))
+ for _, fsErr := range f.fsErrs {
+ path := fsErr.path
+ node := f.nodes.GetNode(path, false)
+ if node != nil {
+ // The path in question wasn't pruned due to a failure to process a parent directory.
+ // So, the failure to process this path is important
+ relevantErrs = append(relevantErrs, fsErr)
+ }
+ }
+ f.fsErrs = relevantErrs
+}
+
+// getErr returns an error based on previous calls to onFsErr, if any
+func (f *Finder) getErr() error {
+ f.discardErrsForPrunedPaths()
+
+ numErrs := len(f.fsErrs)
+ if numErrs < 1 {
+ return nil
+ }
+
+ maxNumErrsToInclude := 10
+ message := ""
+ if numErrs > maxNumErrsToInclude {
+ message = fmt.Sprintf("finder encountered %v errors: %v...", numErrs, f.fsErrs[:maxNumErrsToInclude])
+ } else {
+ message = fmt.Sprintf("finder encountered %v errors: %v", numErrs, f.fsErrs)
+ }
+
+ return errors.New(message)
+}
+
+func (f *Finder) statDirAsync(dir *pathMap) {
+ node := dir
+ path := dir.path
+ f.threadPool.Run(
+ func() {
+ updatedStats := f.statDirSync(path)
+
+ if !f.isInfoUpToDate(node.statResponse, updatedStats) {
+ node.mapNode = mapNode{
+ statResponse: updatedStats,
+ FileNames: []string{},
+ }
+ f.setModified()
+ if node.statResponse.ModTime != 0 {
+ // modification time was updated, so re-scan for
+ // child directories
+ f.listDirAsync(dir)
+ }
+ }
+ },
+ )
+}
+
+func (f *Finder) statDirSync(path string) statResponse {
+
+ fileInfo, err := f.filesystem.Lstat(path)
+
+ var stats statResponse
+ if err != nil {
+ // possibly record this error
+ f.onFsError(path, err)
+ // in case of a failure to stat the directory, treat the directory as missing (modTime = 0)
+ return stats
+ }
+ modTime := fileInfo.ModTime()
+ stats = statResponse{}
+ inode, err := f.filesystem.InodeNumber(fileInfo)
+ if err != nil {
+ panic(fmt.Sprintf("Could not get inode number of %v: %v\n", path, err.Error()))
+ }
+ stats.Inode = inode
+ device, err := f.filesystem.DeviceNumber(fileInfo)
+ if err != nil {
+ panic(fmt.Sprintf("Could not get device number of %v: %v\n", path, err.Error()))
+ }
+ stats.Device = device
+ permissionsChangeTime, err := f.filesystem.PermTime(fileInfo)
+
+ if err != nil {
+ panic(fmt.Sprintf("Could not get permissions modification time (CTime) of %v: %v\n", path, err.Error()))
+ }
+ // We're only interested in knowing whether anything about the directory
+ // has changed since last check, so we use the latest of the two
+ // modification times (content modification (mtime) and
+ // permission modification (ctime))
+ if permissionsChangeTime.After(modTime) {
+ modTime = permissionsChangeTime
+ }
+ stats.ModTime = modTime.UnixNano()
+
+ return stats
+}
+
+// pruneCacheCandidates removes the items that we don't want to include in our persistent cache
+func (f *Finder) pruneCacheCandidates(items *DirEntries) {
+
+ for _, fileName := range items.FileNames {
+ for _, abortedName := range f.cacheMetadata.Config.PruneFiles {
+ if fileName == abortedName {
+ items.FileNames = []string{}
+ items.DirNames = []string{}
+ return
+ }
+ }
+ }
+
+ // remove any files that aren't the ones we want to include
+ writeIndex := 0
+ for _, fileName := range items.FileNames {
+ // include only these files
+ for _, includedName := range f.cacheMetadata.Config.IncludeFiles {
+ if fileName == includedName {
+ items.FileNames[writeIndex] = fileName
+ writeIndex++
+ break
+ }
+ }
+ }
+ // resize
+ items.FileNames = items.FileNames[:writeIndex]
+
+ writeIndex = 0
+ for _, dirName := range items.DirNames {
+ items.DirNames[writeIndex] = dirName
+ // ignore other dirs that are known to not be inputs to the build process
+ include := true
+ for _, excludedName := range f.cacheMetadata.Config.ExcludeDirs {
+ if dirName == excludedName {
+ // don't include
+ include = false
+ break
+ }
+ }
+ if include {
+ writeIndex++
+ }
+ }
+ // resize
+ items.DirNames = items.DirNames[:writeIndex]
+}
+
+func (f *Finder) listDirsAsync(nodes []*pathMap) {
+ f.threadPool.Run(
+ func() {
+ for i := range nodes {
+ f.listDirSync(nodes[i])
+ }
+ },
+ )
+}
+
+func (f *Finder) listDirAsync(node *pathMap) {
+ f.threadPool.Run(
+ func() {
+ f.listDirSync(node)
+ },
+ )
+}
+
+func (f *Finder) listDirSync(dir *pathMap) {
+ path := dir.path
+ children, err := f.filesystem.ReadDir(path)
+
+ if err != nil {
+ // possibly record this error
+ f.onFsError(path, err)
+ // if listing the contents of the directory fails (presumably due to
+ // permission denied), then treat the directory as empty
+ children = []os.FileInfo{}
+ }
+
+ var subdirs []string
+ var subfiles []string
+
+ for _, child := range children {
+ linkBits := child.Mode() & os.ModeSymlink
+ isLink := linkBits != 0
+ if child.IsDir() {
+ if !isLink {
+ // Skip symlink dirs.
+ // We don't have to support symlink dirs because
+ // that would cause duplicates.
+ subdirs = append(subdirs, child.Name())
+ }
+ } else {
+ // We do have to support symlink files because the link name might be
+ // different than the target name
+ // (for example, Android.bp -> build/soong/root.bp)
+ subfiles = append(subfiles, child.Name())
+ }
+
+ }
+ parentNode := dir
+
+ entry := &DirEntries{Path: path, DirNames: subdirs, FileNames: subfiles}
+ f.pruneCacheCandidates(entry)
+
+ // create a pathMap node for each relevant subdirectory
+ relevantChildren := map[string]*pathMap{}
+ for _, subdirName := range entry.DirNames {
+ childNode, found := parentNode.children[subdirName]
+ // if we already knew of this directory, then we already have a request pending to Stat it
+ // if we didn't already know of this directory, then we must Stat it now
+ if !found {
+ childNode = parentNode.newChild(subdirName)
+ f.statDirAsync(childNode)
+ }
+ relevantChildren[subdirName] = childNode
+ }
+ // Note that in rare cases, it's possible that we're reducing the set of
+ // children via this statement, if these are all true:
+ // 1. we previously had a cache that knew about subdirectories of parentNode
+ // 2. the user created a prune-file (described in pruneCacheCandidates)
+ // inside <parentNode>, which specifies that the contents of parentNode
+ // are to be ignored.
+ // The fact that it's possible to remove children here means that *pathMap structs
+ // must not be looked up from f.nodes by filepath (and instead must be accessed by
+ // direct pointer) until after every listDirSync completes
+ parentNode.FileNames = entry.FileNames
+ parentNode.children = relevantChildren
+
+}
+
+// listMatches takes a node and a function that specifies which subdirectories and
+// files to include, and listMatches returns the matches
+func (f *Finder) listMatches(node *pathMap,
+ filter WalkFunc) (subDirs []*pathMap, filePaths []string) {
+ entries := DirEntries{
+ FileNames: node.FileNames,
+ }
+ entries.DirNames = make([]string, 0, len(node.children))
+ for childName := range node.children {
+ entries.DirNames = append(entries.DirNames, childName)
+ }
+
+ dirNames, fileNames := filter(entries)
+
+ subDirs = []*pathMap{}
+ filePaths = make([]string, 0, len(fileNames))
+ for _, fileName := range fileNames {
+ filePaths = append(filePaths, joinCleanPaths(node.path, fileName))
+ }
+ subDirs = make([]*pathMap, 0, len(dirNames))
+ for _, childName := range dirNames {
+ child, ok := node.children[childName]
+ if ok {
+ subDirs = append(subDirs, child)
+ }
+ }
+
+ return subDirs, filePaths
+}
+
+// findInCacheMultithreaded spawns potentially multiple goroutines with which to search the cache.
+func (f *Finder) findInCacheMultithreaded(node *pathMap, filter WalkFunc,
+ approxNumThreads int) []string {
+
+ if approxNumThreads < 2 {
+ // Done spawning threads; process remaining directories
+ return f.findInCacheSinglethreaded(node, filter)
+ }
+
+ totalWork := 0
+ for _, child := range node.children {
+ totalWork += child.approximateNumDescendents
+ }
+ childrenResults := make(chan []string, len(node.children))
+
+ subDirs, filePaths := f.listMatches(node, filter)
+
+ // process child directories
+ for _, child := range subDirs {
+ numChildThreads := approxNumThreads * child.approximateNumDescendents / totalWork
+ childProcessor := func(child *pathMap) {
+ childResults := f.findInCacheMultithreaded(child, filter, numChildThreads)
+ childrenResults <- childResults
+ }
+ // If we're allowed to use more than 1 thread to process this directory,
+ // then instead we use 1 thread for each subdirectory.
+ // It would be strange to spawn threads for only some subdirectories.
+ go childProcessor(child)
+ }
+
+ // collect results
+ for i := 0; i < len(subDirs); i++ {
+ childResults := <-childrenResults
+ filePaths = append(filePaths, childResults...)
+ }
+ close(childrenResults)
+
+ return filePaths
+}
+
+// findInCacheSinglethreaded synchronously searches the cache for all matching file paths
+// note findInCacheSinglethreaded runs 2X to 4X as fast by being iterative rather than recursive
+func (f *Finder) findInCacheSinglethreaded(node *pathMap, filter WalkFunc) []string {
+ if node == nil {
+ return []string{}
+ }
+
+ nodes := []*pathMap{node}
+ matches := []string{}
+
+ for len(nodes) > 0 {
+ currentNode := nodes[0]
+ nodes = nodes[1:]
+
+ subDirs, filePaths := f.listMatches(currentNode, filter)
+
+ nodes = append(nodes, subDirs...)
+
+ matches = append(matches, filePaths...)
+ }
+ return matches
+}
diff --git a/finder/finder_test.go b/finder/finder_test.go
new file mode 100644
index 0000000..1522c68
--- /dev/null
+++ b/finder/finder_test.go
@@ -0,0 +1,1667 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package finder
+
+import (
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "testing"
+ "time"
+
+ "android/soong/fs"
+ "runtime/debug"
+)
+
+// some utils for tests to use
+func newFs() *fs.MockFs {
+ return fs.NewMockFs(map[string][]byte{})
+}
+
+func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Finder {
+ return newFinderWithNumThreads(t, filesystem, cacheParams, 2)
+}
+
+func newFinderWithNumThreads(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) *Finder {
+ f, err := newFinderAndErr(t, filesystem, cacheParams, numThreads)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+ return f
+}
+
+func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) (*Finder, error) {
+ cachePath := "/finder/finder-db"
+ cacheDir := filepath.Dir(cachePath)
+ filesystem.MkDirs(cacheDir)
+ if cacheParams.WorkingDirectory == "" {
+ cacheParams.WorkingDirectory = "/cwd"
+ }
+
+ logger := log.New(ioutil.Discard, "", 0)
+ f, err := newImpl(cacheParams, filesystem, logger, cachePath, numThreads)
+ return f, err
+}
+
+func finderWithSameParams(t *testing.T, original *Finder) *Finder {
+ f, err := finderAndErrorWithSameParams(t, original)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+ return f
+}
+
+func finderAndErrorWithSameParams(t *testing.T, original *Finder) (*Finder, error) {
+ f, err := newImpl(
+ original.cacheMetadata.Config.CacheParams,
+ original.filesystem,
+ original.logger,
+ original.DbPath,
+ original.numDbLoadingThreads,
+ )
+ return f, err
+}
+
+func write(t *testing.T, path string, content string, filesystem *fs.MockFs) {
+ parent := filepath.Dir(path)
+ filesystem.MkDirs(parent)
+ err := filesystem.WriteFile(path, []byte(content), 0777)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+}
+
+func create(t *testing.T, path string, filesystem *fs.MockFs) {
+ write(t, path, "hi", filesystem)
+}
+
+func delete(t *testing.T, path string, filesystem *fs.MockFs) {
+ err := filesystem.Remove(path)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+}
+
+func removeAll(t *testing.T, path string, filesystem *fs.MockFs) {
+ err := filesystem.RemoveAll(path)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+}
+
+func move(t *testing.T, oldPath string, newPath string, filesystem *fs.MockFs) {
+ err := filesystem.Rename(oldPath, newPath)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+}
+
+func link(t *testing.T, newPath string, oldPath string, filesystem *fs.MockFs) {
+ parentPath := filepath.Dir(newPath)
+ err := filesystem.MkDirs(parentPath)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ err = filesystem.Symlink(oldPath, newPath)
+ if err != nil {
+ fatal(t, err.Error())
+ }
+}
+func read(t *testing.T, path string, filesystem *fs.MockFs) string {
+ reader, err := filesystem.Open(path)
+ if err != nil {
+ t.Fatalf(err.Error())
+ }
+ bytes, err := ioutil.ReadAll(reader)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ return string(bytes)
+}
+func modTime(t *testing.T, path string, filesystem *fs.MockFs) time.Time {
+ stats, err := filesystem.Lstat(path)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ return stats.ModTime()
+}
+func setReadable(t *testing.T, path string, readable bool, filesystem *fs.MockFs) {
+ err := filesystem.SetReadable(path, readable)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+}
+
+func setReadErr(t *testing.T, path string, readErr error, filesystem *fs.MockFs) {
+ err := filesystem.SetReadErr(path, readErr)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+}
+
+func fatal(t *testing.T, message string) {
+ t.Error(message)
+ debug.PrintStack()
+ t.FailNow()
+}
+
+func assertSameResponse(t *testing.T, actual []string, expected []string) {
+ sort.Strings(actual)
+ sort.Strings(expected)
+ if !reflect.DeepEqual(actual, expected) {
+ fatal(
+ t,
+ fmt.Sprintf(
+ "Expected Finder to return these %v paths:\n %v,\ninstead returned these %v paths: %v\n",
+ len(expected), expected, len(actual), actual),
+ )
+ }
+}
+
+func assertSameStatCalls(t *testing.T, actual []string, expected []string) {
+ sort.Strings(actual)
+ sort.Strings(expected)
+
+ if !reflect.DeepEqual(actual, expected) {
+ fatal(
+ t,
+ fmt.Sprintf(
+ "Finder made incorrect Stat calls.\n"+
+ "Actual:\n"+
+ "%v\n"+
+ "Expected:\n"+
+ "%v\n"+
+ "\n",
+ actual, expected),
+ )
+ }
+}
+func assertSameReadDirCalls(t *testing.T, actual []string, expected []string) {
+ sort.Strings(actual)
+ sort.Strings(expected)
+
+ if !reflect.DeepEqual(actual, expected) {
+ fatal(
+ t,
+ fmt.Sprintf(
+ "Finder made incorrect ReadDir calls.\n"+
+ "Actual:\n"+
+ "%v\n"+
+ "Expected:\n"+
+ "%v\n"+
+ "\n",
+ actual, expected),
+ )
+ }
+}
+
+// runSimpleTests creates a few files, searches for findme.txt, and checks for the expected matches
+func runSimpleTest(t *testing.T, existentPaths []string, expectedMatches []string) {
+ filesystem := newFs()
+ root := "/tmp"
+ filesystem.MkDirs(root)
+ for _, path := range existentPaths {
+ create(t, filepath.Join(root, path), filesystem)
+ }
+
+ finder := newFinder(t,
+ filesystem,
+ CacheParams{
+ "/cwd",
+ []string{root},
+ nil,
+ nil,
+ []string{"findme.txt", "skipme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt(root, "findme.txt")
+ absoluteMatches := []string{}
+ for i := range expectedMatches {
+ absoluteMatches = append(absoluteMatches, filepath.Join(root, expectedMatches[i]))
+ }
+ assertSameResponse(t, foundPaths, absoluteMatches)
+}
+
+// testAgainstSeveralThreadcounts runs the given test for each threadcount that we care to test
+func testAgainstSeveralThreadcounts(t *testing.T, tester func(t *testing.T, numThreads int)) {
+ // test singlethreaded, multithreaded, and also using the same number of threads as
+ // will be used on the current system
+ threadCounts := []int{1, 2, defaultNumThreads}
+ for _, numThreads := range threadCounts {
+ testName := fmt.Sprintf("%v threads", numThreads)
+ // store numThreads in a new variable to prevent numThreads from changing in each loop
+ localNumThreads := numThreads
+ t.Run(testName, func(t *testing.T) {
+ tester(t, localNumThreads)
+ })
+ }
+}
+
+// end of utils, start of individual tests
+
+func TestSingleFile(t *testing.T) {
+ runSimpleTest(t,
+ []string{"findme.txt"},
+ []string{"findme.txt"},
+ )
+}
+
+func TestIncludeFiles(t *testing.T) {
+ runSimpleTest(t,
+ []string{"findme.txt", "skipme.txt"},
+ []string{"findme.txt"},
+ )
+}
+
+func TestNestedDirectories(t *testing.T) {
+ runSimpleTest(t,
+ []string{"findme.txt", "skipme.txt", "subdir/findme.txt", "subdir/skipme.txt"},
+ []string{"findme.txt", "subdir/findme.txt"},
+ )
+}
+
+func TestEmptyDirectory(t *testing.T) {
+ runSimpleTest(t,
+ []string{},
+ []string{},
+ )
+}
+
+func TestEmptyPath(t *testing.T) {
+ filesystem := newFs()
+ root := "/tmp"
+ create(t, filepath.Join(root, "findme.txt"), filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{root},
+ IncludeFiles: []string{"findme.txt", "skipme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("", "findme.txt")
+
+ assertSameResponse(t, foundPaths, []string{})
+}
+
+func TestFilesystemRoot(t *testing.T) {
+
+ testWithNumThreads := func(t *testing.T, numThreads int) {
+ filesystem := newFs()
+ root := "/"
+ createdPath := "/findme.txt"
+ create(t, createdPath, filesystem)
+
+ finder := newFinderWithNumThreads(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{root},
+ IncludeFiles: []string{"findme.txt", "skipme.txt"},
+ },
+ numThreads,
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt(root, "findme.txt")
+
+ assertSameResponse(t, foundPaths, []string{createdPath})
+ }
+
+ testAgainstSeveralThreadcounts(t, testWithNumThreads)
+}
+
+func TestNonexistentDir(t *testing.T) {
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+
+ _, err := newFinderAndErr(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp/IDontExist"},
+ IncludeFiles: []string{"findme.txt", "skipme.txt"},
+ },
+ 1,
+ )
+ if err == nil {
+ fatal(t, "Did not fail when given a nonexistent root directory")
+ }
+}
+
+func TestExcludeDirs(t *testing.T) {
+ filesystem := newFs()
+ create(t, "/tmp/exclude/findme.txt", filesystem)
+ create(t, "/tmp/exclude/subdir/findme.txt", filesystem)
+ create(t, "/tmp/subdir/exclude/findme.txt", filesystem)
+ create(t, "/tmp/subdir/subdir/findme.txt", filesystem)
+ create(t, "/tmp/subdir/findme.txt", filesystem)
+ create(t, "/tmp/findme.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ ExcludeDirs: []string{"exclude"},
+ IncludeFiles: []string{"findme.txt", "skipme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/subdir/findme.txt",
+ "/tmp/subdir/subdir/findme.txt"})
+}
+
+func TestPruneFiles(t *testing.T) {
+ filesystem := newFs()
+ create(t, "/tmp/out/findme.txt", filesystem)
+ create(t, "/tmp/out/.ignore-out-dir", filesystem)
+ create(t, "/tmp/out/child/findme.txt", filesystem)
+
+ create(t, "/tmp/out2/.ignore-out-dir", filesystem)
+ create(t, "/tmp/out2/sub/findme.txt", filesystem)
+
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/include/findme.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ PruneFiles: []string{".ignore-out-dir"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/include/findme.txt"})
+}
+
+// TestRootDir tests that the value of RootDirs is used
+// tests of the filesystem root are in TestFilesystemRoot
+func TestRootDir(t *testing.T) {
+ filesystem := newFs()
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/subdir/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+ create(t, "/tmp/b/subdir/findme.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp/a"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("/tmp/a", "findme.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/a/findme.txt",
+ "/tmp/a/subdir/findme.txt"})
+}
+
+func TestUncachedDir(t *testing.T) {
+ filesystem := newFs()
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/subdir/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+ create(t, "/tmp/b/subdir/findme.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp/b"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+
+ foundPaths := finder.FindNamedAt("/tmp/a", "findme.txt")
+ // If the caller queries for a file that is in the cache, then computing the
+ // correct answer won't be fast, and it would be easy for the caller to
+ // fail to notice its slowness. Instead, we only ever search the cache for files
+ // to return, which enforces that we can determine which files will be
+ // interesting upfront.
+ assertSameResponse(t, foundPaths, []string{})
+
+ finder.Shutdown()
+}
+
+func TestSearchingForFilesExcludedFromCache(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/misc.txt", filesystem)
+
+ // set up the finder and run it
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "misc.txt")
+ // If the caller queries for a file that is in the cache, then computing the
+ // correct answer won't be fast, and it would be easy for the caller to
+ // fail to notice its slowness. Instead, we only ever search the cache for files
+ // to return, which enforces that we can determine which files will be
+ // interesting upfront.
+ assertSameResponse(t, foundPaths, []string{})
+
+ finder.Shutdown()
+}
+
+func TestRelativeFilePaths(t *testing.T) {
+ filesystem := newFs()
+
+ create(t, "/tmp/ignore/hi.txt", filesystem)
+ create(t, "/tmp/include/hi.txt", filesystem)
+ create(t, "/cwd/hi.txt", filesystem)
+ create(t, "/cwd/a/hi.txt", filesystem)
+ create(t, "/cwd/a/a/hi.txt", filesystem)
+ create(t, "/rel/a/hi.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/cwd", "../rel", "/tmp/include"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("a", "hi.txt")
+ assertSameResponse(t, foundPaths,
+ []string{"a/hi.txt",
+ "a/a/hi.txt"})
+
+ foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt")
+ assertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"})
+
+ foundPaths = finder.FindNamedAt(".", "hi.txt")
+ assertSameResponse(t, foundPaths,
+ []string{"hi.txt",
+ "a/hi.txt",
+ "a/a/hi.txt"})
+
+ foundPaths = finder.FindNamedAt("/rel", "hi.txt")
+ assertSameResponse(t, foundPaths,
+ []string{"/rel/a/hi.txt"})
+
+ foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt")
+ assertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"})
+}
+
+// have to run this test with the race-detector (`go test -race src/android/soong/finder/*.go`)
+// for there to be much chance of the test actually detecting any error that may be present
+func TestRootDirsContainedInOtherRootDirs(t *testing.T) {
+ filesystem := newFs()
+
+ create(t, "/tmp/a/b/c/d/e/f/g/h/i/j/findme.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/", "/tmp/a/b/c", "/tmp/a/b/c/d/e/f", "/tmp/a/b/c/d/e/f/g/h/i"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("/tmp/a", "findme.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/a/b/c/d/e/f/g/h/i/j/findme.txt"})
+}
+
+func TestFindFirst(t *testing.T) {
+ filesystem := newFs()
+ create(t, "/tmp/a/hi.txt", filesystem)
+ create(t, "/tmp/b/hi.txt", filesystem)
+ create(t, "/tmp/b/a/hi.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindFirstNamed("hi.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/a/hi.txt",
+ "/tmp/b/hi.txt"},
+ )
+}
+
+func TestConcurrentFindSameDirectory(t *testing.T) {
+
+ testWithNumThreads := func(t *testing.T, numThreads int) {
+ filesystem := newFs()
+
+ // create a bunch of files and directories
+ paths := []string{}
+ for i := 0; i < 10; i++ {
+ parentDir := fmt.Sprintf("/tmp/%v", i)
+ for j := 0; j < 10; j++ {
+ filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
+ paths = append(paths, filePath)
+ }
+ }
+ sort.Strings(paths)
+ for _, path := range paths {
+ create(t, path, filesystem)
+ }
+
+ // set up a finder
+ finder := newFinderWithNumThreads(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ numThreads,
+ )
+ defer finder.Shutdown()
+
+ numTests := 20
+ results := make(chan []string, numTests)
+ // make several parallel calls to the finder
+ for i := 0; i < numTests; i++ {
+ go func() {
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ results <- foundPaths
+ }()
+ }
+
+ // check that each response was correct
+ for i := 0; i < numTests; i++ {
+ foundPaths := <-results
+ assertSameResponse(t, foundPaths, paths)
+ }
+ }
+
+ testAgainstSeveralThreadcounts(t, testWithNumThreads)
+}
+
+func TestConcurrentFindDifferentDirectories(t *testing.T) {
+ filesystem := newFs()
+
+ // create a bunch of files and directories
+ allFiles := []string{}
+ numSubdirs := 10
+ rootPaths := []string{}
+ queryAnswers := [][]string{}
+ for i := 0; i < numSubdirs; i++ {
+ parentDir := fmt.Sprintf("/tmp/%v", i)
+ rootPaths = append(rootPaths, parentDir)
+ queryAnswers = append(queryAnswers, []string{})
+ for j := 0; j < 10; j++ {
+ filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
+ queryAnswers[i] = append(queryAnswers[i], filePath)
+ allFiles = append(allFiles, filePath)
+ }
+ sort.Strings(queryAnswers[i])
+ }
+ sort.Strings(allFiles)
+ for _, path := range allFiles {
+ create(t, path, filesystem)
+ }
+
+ // set up a finder
+ finder := newFinder(
+ t,
+ filesystem,
+
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ type testRun struct {
+ path string
+ foundMatches []string
+ correctMatches []string
+ }
+
+ numTests := numSubdirs + 1
+ testRuns := make(chan testRun, numTests)
+
+ searchAt := func(path string, correctMatches []string) {
+ foundPaths := finder.FindNamedAt(path, "findme.txt")
+ testRuns <- testRun{path, foundPaths, correctMatches}
+ }
+
+ // make several parallel calls to the finder
+ go searchAt("/tmp", allFiles)
+ for i := 0; i < len(rootPaths); i++ {
+ go searchAt(rootPaths[i], queryAnswers[i])
+ }
+
+ // check that each response was correct
+ for i := 0; i < numTests; i++ {
+ testRun := <-testRuns
+ assertSameResponse(t, testRun.foundMatches, testRun.correctMatches)
+ }
+}
+
+func TestStrangelyFormattedPaths(t *testing.T) {
+ filesystem := newFs()
+
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"//tmp//a//.."},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("//tmp//a//..", "findme.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/a/findme.txt",
+ "/tmp/b/findme.txt",
+ "/tmp/findme.txt"})
+}
+
+func TestCorruptedCacheHeader(t *testing.T) {
+ filesystem := newFs()
+
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ write(t, "/finder/finder-db", "sample header", filesystem)
+
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ defer finder.Shutdown()
+
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/a/findme.txt",
+ "/tmp/findme.txt"})
+}
+
+func TestCanUseCache(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ // check the response of the first finder
+ correctResponse := []string{"/tmp/a/findme.txt",
+ "/tmp/findme.txt"}
+ assertSameResponse(t, foundPaths, correctResponse)
+ finder.Shutdown()
+
+ // check results
+ cacheText := read(t, finder.DbPath, filesystem)
+ if len(cacheText) < 1 {
+ t.Fatalf("saved cache db is empty\n")
+ }
+ if len(filesystem.StatCalls) == 0 {
+ t.Fatal("No Stat calls recorded by mock filesystem")
+ }
+ if len(filesystem.ReadDirCalls) == 0 {
+ t.Fatal("No ReadDir calls recorded by filesystem")
+ }
+ statCalls := filesystem.StatCalls
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+ // check results
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
+ assertSameReadDirCalls(t, filesystem.StatCalls, statCalls)
+
+ finder2.Shutdown()
+}
+
+func TestCorruptedCacheBody(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+
+ // check the response of the first finder
+ correctResponse := []string{"/tmp/a/findme.txt",
+ "/tmp/findme.txt"}
+ assertSameResponse(t, foundPaths, correctResponse)
+ numStatCalls := len(filesystem.StatCalls)
+ numReadDirCalls := len(filesystem.ReadDirCalls)
+
+ // load the cache file, corrupt it, and save it
+ cacheReader, err := filesystem.Open(finder.DbPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cacheData, err := ioutil.ReadAll(cacheReader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cacheData = append(cacheData, []byte("DontMindMe")...)
+ filesystem.WriteFile(finder.DbPath, cacheData, 0777)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+ // check results
+ assertSameResponse(t, foundPaths, correctResponse)
+ numNewStatCalls := len(filesystem.StatCalls)
+ numNewReadDirCalls := len(filesystem.ReadDirCalls)
+ // It's permissable to make more Stat calls with a corrupted cache because
+ // the Finder may restart once it detects corruption.
+ // However, it may have already issued many Stat calls.
+ // Because a corrupted db is not expected to be a common (or even a supported case),
+ // we don't care to optimize it and don't cache the already-issued Stat calls
+ if numNewReadDirCalls < numReadDirCalls {
+ t.Fatalf(
+ "Finder made fewer ReadDir calls with a corrupted cache (%v calls) than with no cache"+
+ " (%v calls)",
+ numNewReadDirCalls, numReadDirCalls)
+ }
+ if numNewStatCalls < numStatCalls {
+ t.Fatalf(
+ "Finder made fewer Stat calls with a corrupted cache (%v calls) than with no cache (%v calls)",
+ numNewStatCalls, numStatCalls)
+ }
+ finder2.Shutdown()
+}
+
+func TestStatCalls(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/a/findme.txt", filesystem)
+
+ // run finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+
+ // check response
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
+ assertSameStatCalls(t, filesystem.StatCalls, []string{"/tmp", "/tmp/a"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/a"})
+}
+
+func TestFileAdded(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/ignoreme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/b/ignore.txt", filesystem)
+ create(t, "/tmp/b/c/nope.txt", filesystem)
+ create(t, "/tmp/b/c/d/irrelevant.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ filesystem.Clock.Tick()
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ create(t, "/tmp/b/c/findme.txt", filesystem)
+ filesystem.Clock.Tick()
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt", "/tmp/b/c/findme.txt"})
+ assertSameStatCalls(t, filesystem.StatCalls, []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/c", "/tmp/b/c/d"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b/c"})
+ finder2.Shutdown()
+
+}
+
+func TestDirectoriesAdded(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/ignoreme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/b/ignore.txt", filesystem)
+ create(t, "/tmp/b/c/nope.txt", filesystem)
+ create(t, "/tmp/b/c/d/irrelevant.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ create(t, "/tmp/b/c/new/findme.txt", filesystem)
+ create(t, "/tmp/b/c/new/new2/findme.txt", filesystem)
+ create(t, "/tmp/b/c/new/new2/ignoreme.txt", filesystem)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/a/findme.txt", "/tmp/b/c/new/findme.txt", "/tmp/b/c/new/new2/findme.txt"})
+ assertSameStatCalls(t, filesystem.StatCalls,
+ []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/c", "/tmp/b/c/d", "/tmp/b/c/new", "/tmp/b/c/new/new2"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b/c", "/tmp/b/c/new", "/tmp/b/c/new/new2"})
+
+ finder2.Shutdown()
+}
+
+func TestDirectoryAndSubdirectoryBothUpdated(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/hi1.txt", filesystem)
+ create(t, "/tmp/a/hi1.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi1.txt", "hi2.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "hi1.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths, []string{"/tmp/hi1.txt", "/tmp/a/hi1.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ create(t, "/tmp/hi2.txt", filesystem)
+ create(t, "/tmp/a/hi2.txt", filesystem)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindAll()
+
+ // check results
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/hi1.txt", "/tmp/hi2.txt", "/tmp/a/hi1.txt", "/tmp/a/hi2.txt"})
+ assertSameStatCalls(t, filesystem.StatCalls,
+ []string{"/tmp", "/tmp/a"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/a"})
+
+ finder2.Shutdown()
+}
+
+func TestFileDeleted(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/ignoreme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+ create(t, "/tmp/b/c/nope.txt", filesystem)
+ create(t, "/tmp/b/c/d/irrelevant.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt", "/tmp/b/findme.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ delete(t, "/tmp/b/findme.txt", filesystem)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"})
+ assertSameStatCalls(t, filesystem.StatCalls, []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/c", "/tmp/b/c/d"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b"})
+
+ finder2.Shutdown()
+}
+
+func TestDirectoriesDeleted(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/1/findme.txt", filesystem)
+ create(t, "/tmp/a/1/2/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/a/findme.txt",
+ "/tmp/a/1/findme.txt",
+ "/tmp/a/1/2/findme.txt",
+ "/tmp/b/findme.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ removeAll(t, "/tmp/a/1", filesystem)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt", "/tmp/a/findme.txt", "/tmp/b/findme.txt"})
+ // Technically, we don't care whether /tmp/a/1/2 gets Statted or gets skipped
+ // if the Finder detects the nonexistence of /tmp/a/1
+ // However, when resuming from cache, we don't want the Finder to necessarily wait
+ // to stat a directory until after statting its parent.
+ // So here we just include /tmp/a/1/2 in the list.
+ // The Finder is currently implemented to always restat every dir and
+ // to not short-circuit due to nonexistence of parents (but it will remove
+ // missing dirs from the cache for next time)
+ assertSameStatCalls(t, filesystem.StatCalls,
+ []string{"/tmp", "/tmp/a", "/tmp/a/1", "/tmp/a/1/2", "/tmp/b"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/a"})
+
+ finder2.Shutdown()
+}
+
+func TestDirectoriesMoved(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/1/findme.txt", filesystem)
+ create(t, "/tmp/a/1/2/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/a/findme.txt",
+ "/tmp/a/1/findme.txt",
+ "/tmp/a/1/2/findme.txt",
+ "/tmp/b/findme.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ move(t, "/tmp/a", "/tmp/c", filesystem)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/b/findme.txt",
+ "/tmp/c/findme.txt",
+ "/tmp/c/1/findme.txt",
+ "/tmp/c/1/2/findme.txt"})
+ // Technically, we don't care whether /tmp/a/1/2 gets Statted or gets skipped
+ // if the Finder detects the nonexistence of /tmp/a/1
+ // However, when resuming from cache, we don't want the Finder to necessarily wait
+ // to stat a directory until after statting its parent.
+ // So here we just include /tmp/a/1/2 in the list.
+ // The Finder is currently implemented to always restat every dir and
+ // to not short-circuit due to nonexistence of parents (but it will remove
+ // missing dirs from the cache for next time)
+ assertSameStatCalls(t, filesystem.StatCalls,
+ []string{"/tmp", "/tmp/a", "/tmp/a/1", "/tmp/a/1/2", "/tmp/b", "/tmp/c", "/tmp/c/1", "/tmp/c/1/2"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/c", "/tmp/c/1", "/tmp/c/1/2"})
+ finder2.Shutdown()
+}
+
+func TestDirectoriesSwapped(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/1/findme.txt", filesystem)
+ create(t, "/tmp/a/1/2/findme.txt", filesystem)
+ create(t, "/tmp/b/findme.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/a/findme.txt",
+ "/tmp/a/1/findme.txt",
+ "/tmp/a/1/2/findme.txt",
+ "/tmp/b/findme.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ move(t, "/tmp/a", "/tmp/temp", filesystem)
+ move(t, "/tmp/b", "/tmp/a", filesystem)
+ move(t, "/tmp/temp", "/tmp/b", filesystem)
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt",
+ "/tmp/a/findme.txt",
+ "/tmp/b/findme.txt",
+ "/tmp/b/1/findme.txt",
+ "/tmp/b/1/2/findme.txt"})
+ // Technically, we don't care whether /tmp/a/1/2 gets Statted or gets skipped
+ // if the Finder detects the nonexistence of /tmp/a/1
+ // However, when resuming from cache, we don't want the Finder to necessarily wait
+ // to stat a directory until after statting its parent.
+ // So here we just include /tmp/a/1/2 in the list.
+ // The Finder is currently implemented to always restat every dir and
+ // to not short-circuit due to nonexistence of parents (but it will remove
+ // missing dirs from the cache for next time)
+ assertSameStatCalls(t, filesystem.StatCalls,
+ []string{"/tmp", "/tmp/a", "/tmp/a/1", "/tmp/a/1/2", "/tmp/b", "/tmp/b/1", "/tmp/b/1/2"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp", "/tmp/a", "/tmp/b", "/tmp/b/1", "/tmp/b/1/2"})
+ finder2.Shutdown()
+}
+
+// runFsReplacementTest tests a change modifying properties of the filesystem itself:
+// runFsReplacementTest tests changing the user, the hostname, or the device number
+// runFsReplacementTest is a helper method called by other tests
+func runFsReplacementTest(t *testing.T, fs1 *fs.MockFs, fs2 *fs.MockFs) {
+ // setup fs1
+ create(t, "/tmp/findme.txt", fs1)
+ create(t, "/tmp/a/findme.txt", fs1)
+ create(t, "/tmp/a/a/findme.txt", fs1)
+
+ // setup fs2 to have the same directories but different files
+ create(t, "/tmp/findme.txt", fs2)
+ create(t, "/tmp/a/findme.txt", fs2)
+ create(t, "/tmp/a/a/ignoreme.txt", fs2)
+ create(t, "/tmp/a/b/findme.txt", fs2)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ fs1,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+ // check the response of the first finder
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt", "/tmp/a/findme.txt", "/tmp/a/a/findme.txt"})
+
+ // copy the cache data from the first filesystem to the second
+ cacheContent := read(t, finder.DbPath, fs1)
+ write(t, finder.DbPath, cacheContent, fs2)
+
+ // run the second finder, with the same config and same cache contents but a different filesystem
+ finder2 := newFinder(
+ t,
+ fs2,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths = finder2.FindNamedAt("/tmp", "findme.txt")
+
+ // check results
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/findme.txt", "/tmp/a/findme.txt", "/tmp/a/b/findme.txt"})
+ assertSameStatCalls(t, fs2.StatCalls,
+ []string{"/tmp", "/tmp/a", "/tmp/a/a", "/tmp/a/b"})
+ assertSameReadDirCalls(t, fs2.ReadDirCalls,
+ []string{"/tmp", "/tmp/a", "/tmp/a/a", "/tmp/a/b"})
+ finder2.Shutdown()
+}
+
+func TestChangeOfDevice(t *testing.T) {
+ fs1 := newFs()
+ // not as fine-grained mounting controls as a real filesystem, but should be adequate
+ fs1.SetDeviceNumber(0)
+
+ fs2 := newFs()
+ fs2.SetDeviceNumber(1)
+
+ runFsReplacementTest(t, fs1, fs2)
+}
+
+func TestChangeOfUserOrHost(t *testing.T) {
+ fs1 := newFs()
+ fs1.SetViewId("me@here")
+
+ fs2 := newFs()
+ fs2.SetViewId("you@there")
+
+ runFsReplacementTest(t, fs1, fs2)
+}
+
+func TestConsistentCacheOrdering(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ for i := 0; i < 5; i++ {
+ create(t, fmt.Sprintf("/tmp/%v/findme.txt", i), filesystem)
+ }
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ finder.FindNamedAt("/tmp", "findme.txt")
+ finder.Shutdown()
+
+ // read db file
+ string1 := read(t, finder.DbPath, filesystem)
+
+ err := filesystem.Remove(finder.DbPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // run another finder
+ finder2 := finderWithSameParams(t, finder)
+ finder2.FindNamedAt("/tmp", "findme.txt")
+ finder2.Shutdown()
+
+ string2 := read(t, finder.DbPath, filesystem)
+
+ if string1 != string2 {
+ t.Errorf("Running Finder twice generated two dbs not having identical contents.\n"+
+ "Content of first file:\n"+
+ "\n"+
+ "%v"+
+ "\n"+
+ "\n"+
+ "Content of second file:\n"+
+ "\n"+
+ "%v\n"+
+ "\n",
+ string1,
+ string2,
+ )
+ }
+
+}
+
+func TestNumSyscallsOfSecondFind(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/misc.txt", filesystem)
+
+ // set up the finder and run it once
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ assertSameResponse(t, foundPaths, []string{"/tmp/findme.txt", "/tmp/a/findme.txt"})
+
+ filesystem.ClearMetrics()
+
+ // run the finder again and confirm it doesn't check the filesystem
+ refoundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ assertSameResponse(t, refoundPaths, foundPaths)
+ assertSameStatCalls(t, filesystem.StatCalls, []string{})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
+
+ finder.Shutdown()
+}
+
+func TestChangingParamsOfSecondFind(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/findme.txt", filesystem)
+ create(t, "/tmp/a/findme.txt", filesystem)
+ create(t, "/tmp/a/metoo.txt", filesystem)
+
+ // set up the finder and run it once
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt", "metoo.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ assertSameResponse(t, foundPaths, []string{"/tmp/findme.txt", "/tmp/a/findme.txt"})
+
+ filesystem.ClearMetrics()
+
+ // run the finder again and confirm it gets the right answer without asking the filesystem
+ refoundPaths := finder.FindNamedAt("/tmp", "metoo.txt")
+ assertSameResponse(t, refoundPaths, []string{"/tmp/a/metoo.txt"})
+ assertSameStatCalls(t, filesystem.StatCalls, []string{})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
+
+ finder.Shutdown()
+}
+
+func TestSymlinkPointingToFile(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/a/hi.txt", filesystem)
+ create(t, "/tmp/a/ignoreme.txt", filesystem)
+ link(t, "/tmp/hi.txt", "a/hi.txt", filesystem)
+ link(t, "/tmp/b/hi.txt", "../a/hi.txt", filesystem)
+ link(t, "/tmp/c/hi.txt", "/tmp/hi.txt", filesystem)
+ link(t, "/tmp/d/hi.txt", "../a/bye.txt", filesystem)
+ link(t, "/tmp/d/bye.txt", "../a/hi.txt", filesystem)
+ link(t, "/tmp/e/bye.txt", "../a/bye.txt", filesystem)
+ link(t, "/tmp/f/hi.txt", "somethingThatDoesntExist", filesystem)
+
+ // set up the finder and run it once
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ foundPaths := finder.FindNamedAt("/tmp", "hi.txt")
+ // should search based on the name of the link rather than the destination or validity of the link
+ correctResponse := []string{
+ "/tmp/a/hi.txt",
+ "/tmp/hi.txt",
+ "/tmp/b/hi.txt",
+ "/tmp/c/hi.txt",
+ "/tmp/d/hi.txt",
+ "/tmp/f/hi.txt",
+ }
+ assertSameResponse(t, foundPaths, correctResponse)
+
+}
+
+func TestSymlinkPointingToDirectory(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/dir/hi.txt", filesystem)
+ create(t, "/tmp/dir/ignoreme.txt", filesystem)
+
+ link(t, "/tmp/links/dir", "../dir", filesystem)
+ link(t, "/tmp/links/link", "../dir", filesystem)
+ link(t, "/tmp/links/broken", "nothingHere", filesystem)
+ link(t, "/tmp/links/recursive", "recursive", filesystem)
+
+ // set up the finder and run it once
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+
+ foundPaths := finder.FindNamedAt("/tmp", "hi.txt")
+
+ // should completely ignore symlinks that point to directories
+ correctResponse := []string{
+ "/tmp/dir/hi.txt",
+ }
+ assertSameResponse(t, foundPaths, correctResponse)
+
+}
+
+// TestAddPruneFile confirms that adding a prune-file (into a directory for which we
+// already had a cache) causes the directory to be ignored
+func TestAddPruneFile(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/out/hi.txt", filesystem)
+ create(t, "/tmp/out/a/hi.txt", filesystem)
+ create(t, "/tmp/hi.txt", filesystem)
+
+ // do find
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ PruneFiles: []string{".ignore-out-dir"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+
+ foundPaths := finder.FindNamedAt("/tmp", "hi.txt")
+
+ // check result
+ assertSameResponse(t, foundPaths,
+ []string{"/tmp/hi.txt",
+ "/tmp/out/hi.txt",
+ "/tmp/out/a/hi.txt"},
+ )
+ finder.Shutdown()
+
+ // modify filesystem
+ filesystem.Clock.Tick()
+ create(t, "/tmp/out/.ignore-out-dir", filesystem)
+ // run another find and check its result
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindNamedAt("/tmp", "hi.txt")
+ assertSameResponse(t, foundPaths, []string{"/tmp/hi.txt"})
+ finder2.Shutdown()
+}
+
+func TestUpdatingDbIffChanged(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/a/hi.txt", filesystem)
+ create(t, "/tmp/b/bye.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ foundPaths := finder.FindAll()
+ filesystem.Clock.Tick()
+ finder.Shutdown()
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"})
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+ create(t, "/tmp/b/hi.txt", filesystem)
+ filesystem.Clock.Tick()
+ filesystem.ClearMetrics()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindAll()
+ finder2.Shutdown()
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt", "/tmp/b/hi.txt"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{"/tmp/b"})
+ expectedDbWriteTime := filesystem.Clock.Time()
+ actualDbWriteTime := modTime(t, finder2.DbPath, filesystem)
+ if actualDbWriteTime != expectedDbWriteTime {
+ t.Fatalf("Expected to write db at %v, actually wrote db at %v\n",
+ expectedDbWriteTime, actualDbWriteTime)
+ }
+
+ // reset metrics
+ filesystem.ClearMetrics()
+
+ // run the third finder
+ finder3 := finderWithSameParams(t, finder2)
+ foundPaths = finder3.FindAll()
+
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt", "/tmp/b/hi.txt"})
+ assertSameReadDirCalls(t, filesystem.ReadDirCalls, []string{})
+ finder3.Shutdown()
+ actualDbWriteTime = modTime(t, finder3.DbPath, filesystem)
+ if actualDbWriteTime != expectedDbWriteTime {
+ t.Fatalf("Re-wrote db even when contents did not change")
+ }
+
+}
+
+func TestDirectoryNotPermitted(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/hi.txt", filesystem)
+ create(t, "/tmp/a/hi.txt", filesystem)
+ create(t, "/tmp/a/a/hi.txt", filesystem)
+ create(t, "/tmp/b/hi.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ foundPaths := finder.FindAll()
+ filesystem.Clock.Tick()
+ finder.Shutdown()
+ allPaths := []string{"/tmp/hi.txt", "/tmp/a/hi.txt", "/tmp/a/a/hi.txt", "/tmp/b/hi.txt"}
+ // check results
+ assertSameResponse(t, foundPaths, allPaths)
+
+ // modify the filesystem
+ filesystem.Clock.Tick()
+
+ setReadable(t, "/tmp/a", false, filesystem)
+ filesystem.Clock.Tick()
+
+ // run the second finder
+ finder2 := finderWithSameParams(t, finder)
+ foundPaths = finder2.FindAll()
+ finder2.Shutdown()
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/hi.txt", "/tmp/b/hi.txt"})
+
+ // modify the filesystem back
+ setReadable(t, "/tmp/a", true, filesystem)
+
+ // run the third finder
+ finder3 := finderWithSameParams(t, finder2)
+ foundPaths = finder3.FindAll()
+ finder3.Shutdown()
+ // check results
+ assertSameResponse(t, foundPaths, allPaths)
+}
+
+func TestFileNotPermitted(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/hi.txt", filesystem)
+ setReadable(t, "/tmp/hi.txt", false, filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ foundPaths := finder.FindAll()
+ filesystem.Clock.Tick()
+ finder.Shutdown()
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/hi.txt"})
+}
+
+func TestCacheEntryPathUnexpectedError(t *testing.T) {
+ // setup filesystem
+ filesystem := newFs()
+ create(t, "/tmp/a/hi.txt", filesystem)
+
+ // run the first finder
+ finder := newFinder(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"hi.txt"},
+ },
+ )
+ foundPaths := finder.FindAll()
+ filesystem.Clock.Tick()
+ finder.Shutdown()
+ // check results
+ assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"})
+
+ // make the directory not readable
+ setReadErr(t, "/tmp/a", os.ErrInvalid, filesystem)
+
+ // run the second finder
+ _, err := finderAndErrorWithSameParams(t, finder)
+ if err == nil {
+ fatal(t, "Failed to detect unexpected filesystem error")
+ }
+}
diff --git a/cmd/soong_zip/Android.bp b/fs/Android.bp
similarity index 64%
copy from cmd/soong_zip/Android.bp
copy to fs/Android.bp
index 10896ce..f4706ca 100644
--- a/cmd/soong_zip/Android.bp
+++ b/fs/Android.bp
@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All rights reserved.
+// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,11 +12,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
+//
+// mock filesystem
+//
+
+bootstrap_go_package {
+ name: "soong-fs",
+ pkgPath: "android/soong/fs",
srcs: [
- "soong_zip.go",
- "rate_limit.go",
+ "fs.go",
],
+ darwin: {
+ srcs: [
+ "fs_darwin.go",
+ ],
+ },
+ linux: {
+ srcs: [
+ "fs_linux.go",
+ ],
+ },
}
+
diff --git a/fs/fs.go b/fs/fs.go
new file mode 100644
index 0000000..eff8ad0
--- /dev/null
+++ b/fs/fs.go
@@ -0,0 +1,935 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/user"
+ "path/filepath"
+ "sync"
+ "time"
+)
+
+var OsFs FileSystem = osFs{}
+
+func NewMockFs(files map[string][]byte) *MockFs {
+ workDir := "/cwd"
+ fs := &MockFs{
+ Clock: NewClock(time.Unix(2, 2)),
+ workDir: workDir,
+ }
+ fs.root = *fs.newDir()
+ fs.MkDirs(workDir)
+
+ for path, bytes := range files {
+ dir := filepath.Dir(path)
+ fs.MkDirs(dir)
+ fs.WriteFile(path, bytes, 0777)
+ }
+
+ return fs
+}
+
+type FileSystem interface {
+ // getting information about files
+ Open(name string) (file io.ReadCloser, err error)
+ Lstat(path string) (stats os.FileInfo, err error)
+ ReadDir(path string) (contents []os.FileInfo, err error)
+
+ InodeNumber(info os.FileInfo) (number uint64, err error)
+ DeviceNumber(info os.FileInfo) (number uint64, err error)
+ PermTime(info os.FileInfo) (time time.Time, err error)
+
+ // changing contents of the filesystem
+ Rename(oldPath string, newPath string) (err error)
+ WriteFile(path string, data []byte, perm os.FileMode) (err error)
+ Remove(path string) (err error)
+ RemoveAll(path string) (err error)
+
+ // metadata about the filesystem
+ ViewId() (id string) // Some unique id of the user accessing the filesystem
+}
+
+// osFs implements FileSystem using the local disk.
+type osFs struct{}
+
+func (osFs) Open(name string) (io.ReadCloser, error) { return os.Open(name) }
+
+func (osFs) Lstat(path string) (stats os.FileInfo, err error) {
+ return os.Lstat(path)
+}
+
+func (osFs) ReadDir(path string) (contents []os.FileInfo, err error) {
+ return ioutil.ReadDir(path)
+}
+func (osFs) Rename(oldPath string, newPath string) error {
+ return os.Rename(oldPath, newPath)
+}
+
+func (osFs) WriteFile(path string, data []byte, perm os.FileMode) error {
+ return ioutil.WriteFile(path, data, perm)
+}
+
+func (osFs) Remove(path string) error {
+ return os.Remove(path)
+}
+
+func (osFs) RemoveAll(path string) error {
+ return os.RemoveAll(path)
+}
+
+func (osFs) ViewId() (id string) {
+ user, err := user.Current()
+ if err != nil {
+ return ""
+ }
+ username := user.Username
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ return ""
+ }
+
+ return username + "@" + hostname
+}
+
+type Clock struct {
+ time time.Time
+}
+
+func NewClock(startTime time.Time) *Clock {
+ return &Clock{time: startTime}
+
+}
+
+func (c *Clock) Tick() {
+ c.time = c.time.Add(time.Microsecond)
+}
+
+func (c *Clock) Time() time.Time {
+ return c.time
+}
+
+// given "/a/b/c/d", pathSplit returns ("/a/b/c", "d")
+func pathSplit(path string) (dir string, leaf string) {
+ dir, leaf = filepath.Split(path)
+ if dir != "/" && len(dir) > 0 {
+ dir = dir[:len(dir)-1]
+ }
+ return dir, leaf
+}
+
+// MockFs supports singlethreaded writes and multithreaded reads
+type MockFs struct {
+ // configuration
+ viewId string //
+ deviceNumber uint64
+
+ // implementation
+ root mockDir
+ Clock *Clock
+ workDir string
+ nextInodeNumber uint64
+
+ // history of requests, for tests to check
+ StatCalls []string
+ ReadDirCalls []string
+ aggregatesLock sync.Mutex
+}
+
+type mockInode struct {
+ modTime time.Time
+ permTime time.Time
+ sys interface{}
+ inodeNumber uint64
+ readErr error
+}
+
+func (m mockInode) ModTime() time.Time {
+ return m.modTime
+}
+
+func (m mockInode) Sys() interface{} {
+ return m.sys
+}
+
+type mockFile struct {
+ bytes []byte
+
+ mockInode
+}
+
+type mockLink struct {
+ target string
+
+ mockInode
+}
+
+type mockDir struct {
+ mockInode
+
+ subdirs map[string]*mockDir
+ files map[string]*mockFile
+ symlinks map[string]*mockLink
+}
+
+func (m *MockFs) resolve(path string, followLastLink bool) (result string, err error) {
+ if !filepath.IsAbs(path) {
+ path = filepath.Join(m.workDir, path)
+ }
+ path = filepath.Clean(path)
+
+ return m.followLinks(path, followLastLink, 10)
+}
+
+// note that followLinks can return a file path that doesn't exist
+func (m *MockFs) followLinks(path string, followLastLink bool, count int) (canonicalPath string, err error) {
+ if path == "/" {
+ return path, nil
+ }
+
+ parentPath, leaf := pathSplit(path)
+ if parentPath == path {
+ err = fmt.Errorf("Internal error: %v yields itself as a parent", path)
+ panic(err.Error())
+ return "", fmt.Errorf("Internal error: %v yields itself as a parent", path)
+ }
+
+ parentPath, err = m.followLinks(parentPath, true, count)
+ if err != nil {
+ return "", err
+ }
+
+ parentNode, err := m.getDir(parentPath, false)
+ if err != nil {
+ return "", err
+ }
+ if parentNode.readErr != nil {
+ return "", &os.PathError{
+ Op: "read",
+ Path: path,
+ Err: parentNode.readErr,
+ }
+ }
+
+ link, isLink := parentNode.symlinks[leaf]
+ if isLink && followLastLink {
+ if count <= 0 {
+ // probably a loop
+ return "", &os.PathError{
+ Op: "read",
+ Path: path,
+ Err: fmt.Errorf("too many levels of symbolic links"),
+ }
+ }
+
+ if link.readErr != nil {
+ return "", &os.PathError{
+ Op: "read",
+ Path: path,
+ Err: link.readErr,
+ }
+ }
+
+ target := m.followLink(link, parentPath)
+ return m.followLinks(target, followLastLink, count-1)
+ }
+ return path, nil
+}
+
+func (m *MockFs) followLink(link *mockLink, parentPath string) (result string) {
+ return filepath.Clean(filepath.Join(parentPath, link.target))
+}
+
+func (m *MockFs) getFile(parentDir *mockDir, fileName string) (file *mockFile, err error) {
+ file, isFile := parentDir.files[fileName]
+ if !isFile {
+ _, isDir := parentDir.subdirs[fileName]
+ _, isLink := parentDir.symlinks[fileName]
+ if isDir || isLink {
+ return nil, &os.PathError{
+ Op: "open",
+ Path: fileName,
+ Err: os.ErrInvalid,
+ }
+ }
+
+ return nil, &os.PathError{
+ Op: "open",
+ Path: fileName,
+ Err: os.ErrNotExist,
+ }
+ }
+ if file.readErr != nil {
+ return nil, &os.PathError{
+ Op: "open",
+ Path: fileName,
+ Err: file.readErr,
+ }
+ }
+ return file, nil
+}
+
+func (m *MockFs) getInode(parentDir *mockDir, name string) (inode *mockInode, err error) {
+ file, isFile := parentDir.files[name]
+ if isFile {
+ return &file.mockInode, nil
+ }
+ link, isLink := parentDir.symlinks[name]
+ if isLink {
+ return &link.mockInode, nil
+ }
+ dir, isDir := parentDir.subdirs[name]
+ if isDir {
+ return &dir.mockInode, nil
+ }
+ return nil, &os.PathError{
+ Op: "stat",
+ Path: name,
+ Err: os.ErrNotExist,
+ }
+
+}
+
+func (m *MockFs) Open(path string) (io.ReadCloser, error) {
+ path, err := m.resolve(path, true)
+ if err != nil {
+ return nil, err
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ parentPath, base := pathSplit(path)
+ parentDir, err := m.getDir(parentPath, false)
+ if err != nil {
+ return nil, err
+ }
+ file, err := m.getFile(parentDir, base)
+ if err != nil {
+ return nil, err
+ }
+ return struct {
+ io.Closer
+ *bytes.Reader
+ }{
+ ioutil.NopCloser(nil),
+ bytes.NewReader(file.bytes),
+ }, nil
+
+}
+
+// a mockFileInfo is for exporting file stats in a way that satisfies the FileInfo interface
+type mockFileInfo struct {
+ path string
+ size int64
+ modTime time.Time // time at which the inode's contents were modified
+ permTime time.Time // time at which the inode's permissions were modified
+ isDir bool
+ inodeNumber uint64
+ deviceNumber uint64
+}
+
+func (m *mockFileInfo) Name() string {
+ return m.path
+}
+
+func (m *mockFileInfo) Size() int64 {
+ return m.size
+}
+
+func (m *mockFileInfo) Mode() os.FileMode {
+ return 0
+}
+
+func (m *mockFileInfo) ModTime() time.Time {
+ return m.modTime
+}
+
+func (m *mockFileInfo) IsDir() bool {
+ return m.isDir
+}
+
+func (m *mockFileInfo) Sys() interface{} {
+ return nil
+}
+
+func (m *MockFs) dirToFileInfo(d *mockDir, path string) (info *mockFileInfo) {
+ return &mockFileInfo{
+ path: path,
+ size: 1,
+ modTime: d.modTime,
+ permTime: d.permTime,
+ isDir: true,
+ inodeNumber: d.inodeNumber,
+ deviceNumber: m.deviceNumber,
+ }
+
+}
+
+func (m *MockFs) fileToFileInfo(f *mockFile, path string) (info *mockFileInfo) {
+ return &mockFileInfo{
+ path: path,
+ size: 1,
+ modTime: f.modTime,
+ permTime: f.permTime,
+ isDir: false,
+ inodeNumber: f.inodeNumber,
+ deviceNumber: m.deviceNumber,
+ }
+}
+
+func (m *MockFs) linkToFileInfo(l *mockLink, path string) (info *mockFileInfo) {
+ return &mockFileInfo{
+ path: path,
+ size: 1,
+ modTime: l.modTime,
+ permTime: l.permTime,
+ isDir: false,
+ inodeNumber: l.inodeNumber,
+ deviceNumber: m.deviceNumber,
+ }
+}
+
+func (m *MockFs) Lstat(path string) (stats os.FileInfo, err error) {
+ // update aggregates
+ m.aggregatesLock.Lock()
+ m.StatCalls = append(m.StatCalls, path)
+ m.aggregatesLock.Unlock()
+
+ // resolve symlinks
+ path, err = m.resolve(path, false)
+ if err != nil {
+ return nil, err
+ }
+
+ // special case for root dir
+ if path == "/" {
+ return m.dirToFileInfo(&m.root, "/"), nil
+ }
+
+ // determine type and handle appropriately
+ parentPath, baseName := pathSplit(path)
+ dir, err := m.getDir(parentPath, false)
+ if err != nil {
+ return nil, err
+ }
+ subdir, subdirExists := dir.subdirs[baseName]
+ if subdirExists {
+ return m.dirToFileInfo(subdir, path), nil
+ }
+ file, fileExists := dir.files[baseName]
+ if fileExists {
+ return m.fileToFileInfo(file, path), nil
+ }
+ link, linkExists := dir.symlinks[baseName]
+ if linkExists {
+ return m.linkToFileInfo(link, path), nil
+ }
+ // not found
+ return nil, &os.PathError{
+ Op: "stat",
+ Path: path,
+ Err: os.ErrNotExist,
+ }
+}
+
+func (m *MockFs) InodeNumber(info os.FileInfo) (number uint64, err error) {
+ mockInfo, ok := info.(*mockFileInfo)
+ if ok {
+ return mockInfo.inodeNumber, nil
+ }
+ return 0, fmt.Errorf("%v is not a mockFileInfo", info)
+}
+func (m *MockFs) DeviceNumber(info os.FileInfo) (number uint64, err error) {
+ mockInfo, ok := info.(*mockFileInfo)
+ if ok {
+ return mockInfo.deviceNumber, nil
+ }
+ return 0, fmt.Errorf("%v is not a mockFileInfo", info)
+}
+func (m *MockFs) PermTime(info os.FileInfo) (when time.Time, err error) {
+ mockInfo, ok := info.(*mockFileInfo)
+ if ok {
+ return mockInfo.permTime, nil
+ }
+ return time.Date(0, 0, 0, 0, 0, 0, 0, nil),
+ fmt.Errorf("%v is not a mockFileInfo", info)
+}
+
+func (m *MockFs) ReadDir(path string) (contents []os.FileInfo, err error) {
+ // update aggregates
+ m.aggregatesLock.Lock()
+ m.ReadDirCalls = append(m.ReadDirCalls, path)
+ m.aggregatesLock.Unlock()
+
+ // locate directory
+ path, err = m.resolve(path, true)
+ if err != nil {
+ return nil, err
+ }
+ results := []os.FileInfo{}
+ dir, err := m.getDir(path, false)
+ if err != nil {
+ return nil, err
+ }
+ if dir.readErr != nil {
+ return nil, &os.PathError{
+ Op: "read",
+ Path: path,
+ Err: dir.readErr,
+ }
+ }
+ // describe its contents
+ for name, subdir := range dir.subdirs {
+ dirInfo := m.dirToFileInfo(subdir, name)
+ results = append(results, dirInfo)
+ }
+ for name, file := range dir.files {
+ info := m.fileToFileInfo(file, name)
+ results = append(results, info)
+ }
+ for name, link := range dir.symlinks {
+ info := m.linkToFileInfo(link, name)
+ results = append(results, info)
+ }
+ return results, nil
+}
+
+func (m *MockFs) Rename(sourcePath string, destPath string) error {
+ // validate source parent exists
+ sourcePath, err := m.resolve(sourcePath, false)
+ if err != nil {
+ return err
+ }
+ sourceParentPath := filepath.Dir(sourcePath)
+ sourceParentDir, err := m.getDir(sourceParentPath, false)
+ if err != nil {
+ return err
+ }
+ if sourceParentDir == nil {
+ return &os.PathError{
+ Op: "move",
+ Path: sourcePath,
+ Err: os.ErrNotExist,
+ }
+ }
+ if sourceParentDir.readErr != nil {
+ return &os.PathError{
+ Op: "move",
+ Path: sourcePath,
+ Err: sourceParentDir.readErr,
+ }
+ }
+
+ // validate dest parent exists
+ destPath, err = m.resolve(destPath, false)
+ destParentPath := filepath.Dir(destPath)
+ destParentDir, err := m.getDir(destParentPath, false)
+ if err != nil {
+ return err
+ }
+ if destParentDir == nil {
+ return &os.PathError{
+ Op: "move",
+ Path: destParentPath,
+ Err: os.ErrNotExist,
+ }
+ }
+ if destParentDir.readErr != nil {
+ return &os.PathError{
+ Op: "move",
+ Path: destParentPath,
+ Err: destParentDir.readErr,
+ }
+ }
+ // check the source and dest themselves
+ sourceBase := filepath.Base(sourcePath)
+ destBase := filepath.Base(destPath)
+
+ file, sourceIsFile := sourceParentDir.files[sourceBase]
+ dir, sourceIsDir := sourceParentDir.subdirs[sourceBase]
+ link, sourceIsLink := sourceParentDir.symlinks[sourceBase]
+
+ // validate that the source exists
+ if !sourceIsFile && !sourceIsDir && !sourceIsLink {
+ return &os.PathError{
+ Op: "move",
+ Path: sourcePath,
+ Err: os.ErrNotExist,
+ }
+
+ }
+
+ // validate the destination doesn't already exist as an incompatible type
+ _, destWasFile := destParentDir.files[destBase]
+ _, destWasDir := destParentDir.subdirs[destBase]
+ _, destWasLink := destParentDir.symlinks[destBase]
+
+ if destWasDir {
+ return &os.PathError{
+ Op: "move",
+ Path: destPath,
+ Err: errors.New("destination exists as a directory"),
+ }
+ }
+
+ if sourceIsDir && (destWasFile || destWasLink) {
+ return &os.PathError{
+ Op: "move",
+ Path: destPath,
+ Err: errors.New("destination exists as a file"),
+ }
+ }
+
+ if destWasFile {
+ delete(destParentDir.files, destBase)
+ }
+ if destWasDir {
+ delete(destParentDir.subdirs, destBase)
+ }
+ if destWasLink {
+ delete(destParentDir.symlinks, destBase)
+ }
+
+ if sourceIsFile {
+ destParentDir.files[destBase] = file
+ delete(sourceParentDir.files, sourceBase)
+ }
+ if sourceIsDir {
+ destParentDir.subdirs[destBase] = dir
+ delete(sourceParentDir.subdirs, sourceBase)
+ }
+ if sourceIsLink {
+ destParentDir.symlinks[destBase] = link
+ delete(destParentDir.symlinks, sourceBase)
+ }
+
+ destParentDir.modTime = m.Clock.Time()
+ sourceParentDir.modTime = m.Clock.Time()
+ return nil
+}
+
+func (m *MockFs) newInodeNumber() uint64 {
+ result := m.nextInodeNumber
+ m.nextInodeNumber++
+ return result
+}
+
+func (m *MockFs) WriteFile(filePath string, data []byte, perm os.FileMode) error {
+ filePath, err := m.resolve(filePath, true)
+ if err != nil {
+ return err
+ }
+ parentPath := filepath.Dir(filePath)
+ parentDir, err := m.getDir(parentPath, false)
+ if err != nil || parentDir == nil {
+ return &os.PathError{
+ Op: "write",
+ Path: parentPath,
+ Err: os.ErrNotExist,
+ }
+ }
+ if parentDir.readErr != nil {
+ return &os.PathError{
+ Op: "write",
+ Path: parentPath,
+ Err: parentDir.readErr,
+ }
+ }
+
+ baseName := filepath.Base(filePath)
+ _, exists := parentDir.files[baseName]
+ if !exists {
+ parentDir.modTime = m.Clock.Time()
+ parentDir.files[baseName] = m.newFile()
+ } else {
+ readErr := parentDir.files[baseName].readErr
+ if readErr != nil {
+ return &os.PathError{
+ Op: "write",
+ Path: filePath,
+ Err: readErr,
+ }
+ }
+ }
+ file := parentDir.files[baseName]
+ file.bytes = data
+ file.modTime = m.Clock.Time()
+ return nil
+}
+
+func (m *MockFs) newFile() *mockFile {
+ newFile := &mockFile{}
+ newFile.inodeNumber = m.newInodeNumber()
+ newFile.modTime = m.Clock.Time()
+ newFile.permTime = newFile.modTime
+ return newFile
+}
+
+func (m *MockFs) newDir() *mockDir {
+ newDir := &mockDir{
+ subdirs: make(map[string]*mockDir, 0),
+ files: make(map[string]*mockFile, 0),
+ symlinks: make(map[string]*mockLink, 0),
+ }
+ newDir.inodeNumber = m.newInodeNumber()
+ newDir.modTime = m.Clock.Time()
+ newDir.permTime = newDir.modTime
+ return newDir
+}
+
+func (m *MockFs) newLink(target string) *mockLink {
+ newLink := &mockLink{
+ target: target,
+ }
+ newLink.inodeNumber = m.newInodeNumber()
+ newLink.modTime = m.Clock.Time()
+ newLink.permTime = newLink.modTime
+
+ return newLink
+}
+func (m *MockFs) MkDirs(path string) error {
+ _, err := m.getDir(path, true)
+ return err
+}
+
+// getDir doesn't support symlinks
+func (m *MockFs) getDir(path string, createIfMissing bool) (dir *mockDir, err error) {
+ cleanedPath := filepath.Clean(path)
+ if cleanedPath == "/" {
+ return &m.root, nil
+ }
+
+ parentPath, leaf := pathSplit(cleanedPath)
+ if len(parentPath) >= len(path) {
+ return &m.root, nil
+ }
+ parent, err := m.getDir(parentPath, createIfMissing)
+ if err != nil {
+ return nil, err
+ }
+ if parent.readErr != nil {
+ return nil, &os.PathError{
+ Op: "stat",
+ Path: path,
+ Err: parent.readErr,
+ }
+ }
+ childDir, dirExists := parent.subdirs[leaf]
+ if !dirExists {
+ if createIfMissing {
+ // confirm that a file with the same name doesn't already exist
+ _, fileExists := parent.files[leaf]
+ if fileExists {
+ return nil, &os.PathError{
+ Op: "mkdir",
+ Path: path,
+ Err: os.ErrExist,
+ }
+ }
+ // create this directory
+ childDir = m.newDir()
+ parent.subdirs[leaf] = childDir
+ parent.modTime = m.Clock.Time()
+ } else {
+ return nil, &os.PathError{
+ Op: "stat",
+ Path: path,
+ Err: os.ErrNotExist,
+ }
+ }
+ }
+ return childDir, nil
+
+}
+
+func (m *MockFs) Remove(path string) (err error) {
+ path, err = m.resolve(path, false)
+ parentPath, leaf := pathSplit(path)
+ if len(leaf) == 0 {
+ return fmt.Errorf("Cannot remove %v\n", path)
+ }
+ parentDir, err := m.getDir(parentPath, false)
+ if err != nil {
+ return err
+ }
+ if parentDir == nil {
+ return &os.PathError{
+ Op: "remove",
+ Path: path,
+ Err: os.ErrNotExist,
+ }
+ }
+ if parentDir.readErr != nil {
+ return &os.PathError{
+ Op: "remove",
+ Path: path,
+ Err: parentDir.readErr,
+ }
+ }
+ _, isDir := parentDir.subdirs[leaf]
+ if isDir {
+ return &os.PathError{
+ Op: "remove",
+ Path: path,
+ Err: os.ErrInvalid,
+ }
+ }
+ _, isLink := parentDir.symlinks[leaf]
+ if isLink {
+ delete(parentDir.symlinks, leaf)
+ } else {
+ _, isFile := parentDir.files[leaf]
+ if !isFile {
+ return &os.PathError{
+ Op: "remove",
+ Path: path,
+ Err: os.ErrNotExist,
+ }
+ }
+ delete(parentDir.files, leaf)
+ }
+ parentDir.modTime = m.Clock.Time()
+ return nil
+}
+
+func (m *MockFs) Symlink(oldPath string, newPath string) (err error) {
+ newPath, err = m.resolve(newPath, false)
+ if err != nil {
+ return err
+ }
+
+ newParentPath, leaf := pathSplit(newPath)
+ newParentDir, err := m.getDir(newParentPath, false)
+ if newParentDir.readErr != nil {
+ return &os.PathError{
+ Op: "link",
+ Path: newPath,
+ Err: newParentDir.readErr,
+ }
+ }
+ if err != nil {
+ return err
+ }
+ newParentDir.symlinks[leaf] = m.newLink(oldPath)
+ return nil
+}
+
+func (m *MockFs) RemoveAll(path string) (err error) {
+ path, err = m.resolve(path, false)
+ if err != nil {
+ return err
+ }
+ parentPath, leaf := pathSplit(path)
+ if len(leaf) == 0 {
+ return fmt.Errorf("Cannot remove %v\n", path)
+ }
+ parentDir, err := m.getDir(parentPath, false)
+ if err != nil {
+ return err
+ }
+ if parentDir == nil {
+ return &os.PathError{
+ Op: "removeAll",
+ Path: path,
+ Err: os.ErrNotExist,
+ }
+ }
+ if parentDir.readErr != nil {
+ return &os.PathError{
+ Op: "removeAll",
+ Path: path,
+ Err: parentDir.readErr,
+ }
+
+ }
+ _, isFile := parentDir.files[leaf]
+ _, isLink := parentDir.symlinks[leaf]
+ if isFile || isLink {
+ return m.Remove(path)
+ }
+ _, isDir := parentDir.subdirs[leaf]
+ if !isDir {
+ if !isDir {
+ return &os.PathError{
+ Op: "removeAll",
+ Path: path,
+ Err: os.ErrNotExist,
+ }
+ }
+ }
+
+ delete(parentDir.subdirs, leaf)
+ parentDir.modTime = m.Clock.Time()
+ return nil
+}
+
+func (m *MockFs) SetReadable(path string, readable bool) error {
+ var readErr error
+ if !readable {
+ readErr = os.ErrPermission
+ }
+ return m.SetReadErr(path, readErr)
+}
+
+func (m *MockFs) SetReadErr(path string, readErr error) error {
+ path, err := m.resolve(path, false)
+ if err != nil {
+ return err
+ }
+ parentPath, leaf := filepath.Split(path)
+ parentDir, err := m.getDir(parentPath, false)
+ if err != nil {
+ return err
+ }
+ if parentDir.readErr != nil {
+ return &os.PathError{
+ Op: "chmod",
+ Path: parentPath,
+ Err: parentDir.readErr,
+ }
+ }
+
+ inode, err := m.getInode(parentDir, leaf)
+ if err != nil {
+ return err
+ }
+ inode.readErr = readErr
+ inode.permTime = m.Clock.Time()
+ return nil
+}
+
+func (m *MockFs) ClearMetrics() {
+ m.ReadDirCalls = []string{}
+ m.StatCalls = []string{}
+}
+
+func (m *MockFs) ViewId() (id string) {
+ return m.viewId
+}
+
+func (m *MockFs) SetViewId(id string) {
+ m.viewId = id
+}
+func (m *MockFs) SetDeviceNumber(deviceNumber uint64) {
+ m.deviceNumber = deviceNumber
+}
diff --git a/fs/fs_darwin.go b/fs/fs_darwin.go
new file mode 100644
index 0000000..c048835
--- /dev/null
+++ b/fs/fs_darwin.go
@@ -0,0 +1,49 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+ "time"
+)
+
+func (osFs) InodeNumber(info os.FileInfo) (number uint64, err error) {
+ sys := info.Sys()
+ darwinStats, ok := sys.(*syscall.Stat_t)
+ if ok {
+ return darwinStats.Ino, nil
+ }
+ return 0, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) DeviceNumber(info os.FileInfo) (number uint64, err error) {
+ sys := info.Sys()
+ darwinStats, ok := sys.(*syscall.Stat_t)
+ if ok {
+ return uint64(darwinStats.Dev), nil
+ }
+ return 0, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) PermTime(info os.FileInfo) (when time.Time, err error) {
+ sys := info.Sys()
+ darwinStats, ok := sys.(*syscall.Stat_t)
+ if ok {
+ return time.Unix(darwinStats.Ctimespec.Sec, darwinStats.Ctimespec.Nsec), nil
+ }
+ return time.Time{}, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
diff --git a/fs/fs_linux.go b/fs/fs_linux.go
new file mode 100644
index 0000000..0a22a2c
--- /dev/null
+++ b/fs/fs_linux.go
@@ -0,0 +1,49 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+ "time"
+)
+
+func (osFs) InodeNumber(info os.FileInfo) (number uint64, err error) {
+ sys := info.Sys()
+ linuxStats, ok := sys.(*syscall.Stat_t)
+ if ok {
+ return linuxStats.Ino, nil
+ }
+ return 0, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) DeviceNumber(info os.FileInfo) (number uint64, err error) {
+ sys := info.Sys()
+ linuxStats, ok := sys.(*syscall.Stat_t)
+ if ok {
+ return linuxStats.Dev, nil
+ }
+ return 0, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) PermTime(info os.FileInfo) (when time.Time, err error) {
+ sys := info.Sys()
+ linuxStats, ok := sys.(*syscall.Stat_t)
+ if ok {
+ return time.Unix(linuxStats.Ctim.Sec, linuxStats.Ctim.Nsec), nil
+ }
+ return time.Time{}, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
diff --git a/genrule/filegroup.go b/genrule/filegroup.go
index 4029134..8f28638 100644
--- a/genrule/filegroup.go
+++ b/genrule/filegroup.go
@@ -16,6 +16,9 @@
import (
"android/soong/android"
+ "io"
+ "strings"
+ "text/template"
)
func init() {
@@ -32,7 +35,11 @@
// of the path to use. For example, when a filegroup is used as data in a cc_test rule,
// the base path is stripped off the path and the remaining path is used as the
// installation directory.
- Path string
+ Path *string
+
+ // Create a make variable with the specified name that contains the list of files in the
+ // filegroup, relative to the root of the source tree.
+ Export_to_make_var *string
}
type fileGroup struct {
@@ -58,9 +65,30 @@
}
func (fg *fileGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- fg.srcs = ctx.ExpandSourcesSubDir(fg.properties.Srcs, fg.properties.Exclude_srcs, fg.properties.Path)
+ fg.srcs = ctx.ExpandSourcesSubDir(fg.properties.Srcs, fg.properties.Exclude_srcs, String(fg.properties.Path))
}
func (fg *fileGroup) Srcs() android.Paths {
return fg.srcs
}
+
+var androidMkTemplate = template.Must(template.New("filegroup").Parse(`
+ifdef {{.makeVar}}
+ $(error variable {{.makeVar}} set by soong module is already set in make)
+endif
+{{.makeVar}} := {{.value}}
+.KATI_READONLY := {{.makeVar}}
+`))
+
+func (fg *fileGroup) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ if makeVar := String(fg.properties.Export_to_make_var); makeVar != "" {
+ androidMkTemplate.Execute(w, map[string]string{
+ "makeVar": makeVar,
+ "value": strings.Join(fg.srcs.Strings(), " "),
+ })
+ }
+ },
+ }
+}
diff --git a/genrule/genrule.go b/genrule/genrule.go
index c5de1fd..4f3ba93 100644
--- a/genrule/genrule.go
+++ b/genrule/genrule.go
@@ -16,10 +16,11 @@
import (
"fmt"
- "path"
"strings"
"github.com/google/blueprint"
+ "github.com/google/blueprint/bootstrap"
+ "github.com/google/blueprint/proptools"
"android/soong/android"
"android/soong/shared"
@@ -48,26 +49,32 @@
HostToolPath() android.OptionalPath
}
+type hostToolDependencyTag struct {
+ blueprint.BaseDependencyTag
+}
+
+var hostToolDepTag hostToolDependencyTag
+
type generatorProperties struct {
// The command to run on one or more input files. Cmd supports substitution of a few variables
// (the actual substitution is implemented in GenerateAndroidBuildActions below)
//
// Available variables for substitution:
//
- // $(location): the path to the first entry in tools or tool_files
- // $(location <label>): the path to the tool or tool_file with name <label>
- // $(in): one or more input files
- // $(out): a single output file
- // $(depfile): a file to which dependencies will be written, if the depfile property is set to true
- // $(genDir): the sandbox directory for this tool; contains $(out)
- // $$: a literal $
+ // $(location): the path to the first entry in tools or tool_files
+ // $(location <label>): the path to the tool or tool_file with name <label>
+ // $(in): one or more input files
+ // $(out): a single output file
+ // $(depfile): a file to which dependencies will be written, if the depfile property is set to true
+ // $(genDir): the sandbox directory for this tool; contains $(out)
+ // $$: a literal $
//
// All files used must be declared as inputs (to ensure proper up-to-date checks).
// Use "$(in)" directly in Cmd to ensure that all inputs used are declared.
- Cmd string
+ Cmd *string
// Enable reading a file containing dependencies in gcc format after the command completes
- Depfile bool
+ Depfile *bool
// name of the modules (if any) that produces the host executable. Leave empty for
// prebuilts or scripts that do not need a module to build them.
@@ -83,12 +90,16 @@
Srcs []string
}
-type generator struct {
+type Module struct {
android.ModuleBase
+ // For other packages to make their own genrules with extra
+ // properties
+ Extra interface{}
+
properties generatorProperties
- tasks taskFunc
+ taskGenerator taskFunc
deps android.Paths
rule blueprint.Rule
@@ -98,42 +109,39 @@
outputFiles android.Paths
}
-type taskFunc func(ctx android.ModuleContext, srcFiles android.Paths) []generateTask
+type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask
type generateTask struct {
in android.Paths
out android.WritablePaths
+ cmd string
}
-func (g *generator) GeneratedSourceFiles() android.Paths {
+func (g *Module) GeneratedSourceFiles() android.Paths {
return g.outputFiles
}
-func (g *generator) Srcs() android.Paths {
+func (g *Module) Srcs() android.Paths {
return g.outputFiles
}
-func (g *generator) GeneratedHeaderDirs() android.Paths {
+func (g *Module) GeneratedHeaderDirs() android.Paths {
return g.exportedIncludeDirs
}
-func (g *generator) DepsMutator(ctx android.BottomUpMutatorContext) {
+func (g *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
android.ExtractSourcesDeps(ctx, g.properties.Srcs)
- if g, ok := ctx.Module().(*generator); ok {
+ android.ExtractSourcesDeps(ctx, g.properties.Tool_files)
+ if g, ok := ctx.Module().(*Module); ok {
if len(g.properties.Tools) > 0 {
ctx.AddFarVariationDependencies([]blueprint.Variation{
- {"arch", ctx.AConfig().BuildOsVariant},
- }, nil, g.properties.Tools...)
+ {"arch", ctx.Config().BuildOsVariant},
+ }, hostToolDepTag, g.properties.Tools...)
}
}
}
-func (g *generator) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
- ctx.ModuleErrorf("at least one `tools` or `tool_files` is required")
- return
- }
-
+func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
if len(g.properties.Export_include_dirs) > 0 {
for _, dir := range g.properties.Export_include_dirs {
g.exportedIncludeDirs = append(g.exportedIncludeDirs,
@@ -146,61 +154,95 @@
tools := map[string]android.Path{}
if len(g.properties.Tools) > 0 {
- ctx.VisitDirectDeps(func(module blueprint.Module) {
- if t, ok := module.(HostToolProvider); ok {
- p := t.HostToolPath()
- if p.Valid() {
- g.deps = append(g.deps, p.Path())
- tool := ctx.OtherModuleName(module)
- if _, exists := tools[tool]; !exists {
- tools[tool] = p.Path()
+ ctx.VisitDirectDepsBlueprint(func(module blueprint.Module) {
+ switch ctx.OtherModuleDependencyTag(module) {
+ case android.SourceDepTag:
+ // Nothing to do
+ case hostToolDepTag:
+ tool := ctx.OtherModuleName(module)
+ var path android.OptionalPath
+
+ if t, ok := module.(HostToolProvider); ok {
+ if !t.(android.Module).Enabled() {
+ if ctx.Config().AllowMissingDependencies() {
+ ctx.AddMissingDependencies([]string{tool})
+ } else {
+ ctx.ModuleErrorf("depends on disabled module %q", tool)
+ }
+ break
+ }
+ path = t.HostToolPath()
+ } else if t, ok := module.(bootstrap.GoBinaryTool); ok {
+ if s, err := filepath.Rel(android.PathForOutput(ctx).String(), t.InstallPath()); err == nil {
+ path = android.OptionalPathForPath(android.PathForOutput(ctx, s))
} else {
- ctx.ModuleErrorf("multiple tools for %q, %q and %q", tool, tools[tool], p.Path().String())
+ ctx.ModuleErrorf("cannot find path for %q: %v", tool, err)
+ break
}
} else {
- ctx.ModuleErrorf("host tool %q missing output file", ctx.OtherModuleName(module))
+ ctx.ModuleErrorf("%q is not a host tool provider", tool)
+ break
}
+
+ if path.Valid() {
+ g.deps = append(g.deps, path.Path())
+ if _, exists := tools[tool]; !exists {
+ tools[tool] = path.Path()
+ } else {
+ ctx.ModuleErrorf("multiple tools for %q, %q and %q", tool, tools[tool], path.Path().String())
+ }
+ } else {
+ ctx.ModuleErrorf("host tool %q missing output file", tool)
+ }
+ default:
+ ctx.ModuleErrorf("unknown dependency on %q", ctx.OtherModuleName(module))
}
})
}
- for _, tool := range g.properties.Tool_files {
- toolPath := android.PathForModuleSrc(ctx, tool)
- g.deps = append(g.deps, toolPath)
- if _, exists := tools[tool]; !exists {
- tools[tool] = toolPath
+ if ctx.Failed() {
+ return
+ }
+
+ toolFiles := ctx.ExpandSources(g.properties.Tool_files, nil)
+ for _, tool := range toolFiles {
+ g.deps = append(g.deps, tool)
+ if _, exists := tools[tool.Rel()]; !exists {
+ tools[tool.Rel()] = tool
} else {
- ctx.ModuleErrorf("multiple tools for %q, %q and %q", tool, tools[tool], toolPath.String())
+ ctx.ModuleErrorf("multiple tools for %q, %q and %q", tool, tools[tool.Rel()], tool.Rel())
}
}
- rawCommand, err := android.Expand(g.properties.Cmd, func(name string) (string, error) {
+ referencedDepfile := false
+
+ srcFiles := ctx.ExpandSources(g.properties.Srcs, nil)
+ task := g.taskGenerator(ctx, String(g.properties.Cmd), srcFiles)
+
+ rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
switch name {
case "location":
+ if len(g.properties.Tools) == 0 && len(toolFiles) == 0 {
+ return "", fmt.Errorf("at least one `tools` or `tool_files` is required if $(location) is used")
+ }
+
if len(g.properties.Tools) > 0 {
return tools[g.properties.Tools[0]].String(), nil
} else {
- return tools[g.properties.Tool_files[0]].String(), nil
+ return tools[toolFiles[0].Rel()].String(), nil
}
case "in":
return "${in}", nil
case "out":
return "__SBOX_OUT_FILES__", nil
case "depfile":
- if !g.properties.Depfile {
+ referencedDepfile = true
+ if !Bool(g.properties.Depfile) {
return "", fmt.Errorf("$(depfile) used without depfile property")
}
- return "${depfile}", nil
+ return "__SBOX_DEPFILE__", nil
case "genDir":
- genPath := android.PathForModuleGen(ctx, "").String()
- var relativePath string
- var err error
- outputPath := android.PathForOutput(ctx).String()
- relativePath, err = filepath.Rel(outputPath, genPath)
- if err != nil {
- panic(err)
- }
- return path.Join("__SBOX_OUT_DIR__", relativePath), nil
+ return "__SBOX_OUT_DIR__", nil
default:
if strings.HasPrefix(name, "location ") {
label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
@@ -214,6 +256,10 @@
}
})
+ if Bool(g.properties.Depfile) && !referencedDepfile {
+ ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
+ }
+
if err != nil {
ctx.PropertyErrorf("cmd", "%s", err.Error())
return
@@ -225,104 +271,167 @@
// recall that Sprintf replaces percent sign expressions, whereas dollar signs expressions remain as written,
// to be replaced later by ninja_strings.go
- sandboxCommand := fmt.Sprintf("$sboxCmd --sandbox-path %s --output-root %s -c %q $out", sandboxPath, buildDir, rawCommand)
+ depfilePlaceholder := ""
+ if Bool(g.properties.Depfile) {
+ depfilePlaceholder = "$depfileArgs"
+ }
+
+ genDir := android.PathForModuleGen(ctx)
+ // Escape the command for the shell
+ rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'"
+ sandboxCommand := fmt.Sprintf("$sboxCmd --sandbox-path %s --output-root %s -c %s %s $allouts",
+ sandboxPath, genDir, rawCommand, depfilePlaceholder)
ruleParams := blueprint.RuleParams{
Command: sandboxCommand,
CommandDeps: []string{"$sboxCmd"},
}
- var args []string
- if g.properties.Depfile {
+ args := []string{"allouts"}
+ if Bool(g.properties.Depfile) {
ruleParams.Deps = blueprint.DepsGCC
- args = append(args, "depfile")
+ args = append(args, "depfileArgs")
}
g.rule = ctx.Rule(pctx, "generator", ruleParams, args...)
- srcFiles := ctx.ExpandSources(g.properties.Srcs, nil)
- for _, task := range g.tasks(ctx, srcFiles) {
- g.generateSourceFile(ctx, task)
- }
+ g.generateSourceFile(ctx, task)
+
}
-func (g *generator) generateSourceFile(ctx android.ModuleContext, task generateTask) {
+func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask) {
desc := "generate"
+ if len(task.out) == 0 {
+ ctx.ModuleErrorf("must have at least one output file")
+ return
+ }
if len(task.out) == 1 {
desc += " " + task.out[0].Base()
}
- params := android.ModuleBuildParams{
- Rule: g.rule,
- Description: "generate",
- Outputs: task.out,
- Inputs: task.in,
- Implicits: g.deps,
+ var depFile android.ModuleGenPath
+ if Bool(g.properties.Depfile) {
+ depFile = android.PathForModuleGen(ctx, task.out[0].Rel()+".d")
}
- if g.properties.Depfile {
- depfile := android.GenPathWithExt(ctx, "", task.out[0], task.out[0].Ext()+".d")
- params.Depfile = depfile
+
+ params := android.BuildParams{
+ Rule: g.rule,
+ Description: "generate",
+ Output: task.out[0],
+ ImplicitOutputs: task.out[1:],
+ Inputs: task.in,
+ Implicits: g.deps,
+ Args: map[string]string{
+ "allouts": strings.Join(task.out.Strings(), " "),
+ },
}
- ctx.ModuleBuild(pctx, params)
+ if Bool(g.properties.Depfile) {
+ params.Depfile = android.PathForModuleGen(ctx, task.out[0].Rel()+".d")
+ params.Args["depfileArgs"] = "--depfile-out " + depFile.String()
+ }
+
+ ctx.Build(pctx, params)
for _, outputFile := range task.out {
g.outputFiles = append(g.outputFiles, outputFile)
}
}
-func generatorFactory(tasks taskFunc, props ...interface{}) android.Module {
- module := &generator{
- tasks: tasks,
+func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module {
+ module := &Module{
+ taskGenerator: taskGenerator,
}
module.AddProperties(props...)
module.AddProperties(&module.properties)
- android.InitAndroidModule(module)
-
return module
}
-func GenSrcsFactory() android.Module {
+func NewGenSrcs() *Module {
properties := &genSrcsProperties{}
- tasks := func(ctx android.ModuleContext, srcFiles android.Paths) []generateTask {
- tasks := make([]generateTask, 0, len(srcFiles))
+ taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask {
+ commands := []string{}
+ outFiles := android.WritablePaths{}
+ genPath := android.PathForModuleGen(ctx).String()
for _, in := range srcFiles {
- tasks = append(tasks, generateTask{
- in: android.Paths{in},
- out: android.WritablePaths{android.GenPathWithExt(ctx, "", in, properties.Output_extension)},
+ outFile := android.GenPathWithExt(ctx, "", in, String(properties.Output_extension))
+ outFiles = append(outFiles, outFile)
+
+ // replace "out" with "__SBOX_OUT_DIR__/<the value of ${out}>"
+ relOut, err := filepath.Rel(genPath, outFile.String())
+ if err != nil {
+ panic(fmt.Sprintf("Could not make ${out} relative: %v", err))
+ }
+ sandboxOutfile := filepath.Join("__SBOX_OUT_DIR__", relOut)
+ command, err := android.Expand(rawCommand, func(name string) (string, error) {
+ switch name {
+ case "in":
+ return in.String(), nil
+ case "out":
+ return sandboxOutfile, nil
+ default:
+ return "$(" + name + ")", nil
+ }
})
+ if err != nil {
+ ctx.PropertyErrorf("cmd", err.Error())
+ }
+
+ // escape the command in case for example it contains '#', an odd number of '"', etc
+ command = fmt.Sprintf("bash -c %v", proptools.ShellEscape([]string{command})[0])
+ commands = append(commands, command)
}
- return tasks
+ fullCommand := strings.Join(commands, " && ")
+
+ return generateTask{
+ in: srcFiles,
+ out: outFiles,
+ cmd: fullCommand,
+ }
}
- return generatorFactory(tasks, properties)
+ return generatorFactory(taskGenerator, properties)
+}
+
+func GenSrcsFactory() android.Module {
+ m := NewGenSrcs()
+ android.InitAndroidModule(m)
+ return m
}
type genSrcsProperties struct {
// extension that will be substituted for each output file
- Output_extension string
+ Output_extension *string
}
-func GenRuleFactory() android.Module {
+func NewGenRule() *Module {
properties := &genRuleProperties{}
- tasks := func(ctx android.ModuleContext, srcFiles android.Paths) []generateTask {
+ taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask {
outs := make(android.WritablePaths, len(properties.Out))
for i, out := range properties.Out {
outs[i] = android.PathForModuleGen(ctx, out)
}
- return []generateTask{
- {
- in: srcFiles,
- out: outs,
- },
+ return generateTask{
+ in: srcFiles,
+ out: outs,
+ cmd: rawCommand,
}
}
- return generatorFactory(tasks, properties)
+ return generatorFactory(taskGenerator, properties)
+}
+
+func GenRuleFactory() android.Module {
+ m := NewGenRule()
+ android.InitAndroidModule(m)
+ return m
}
type genRuleProperties struct {
// names of the output files that will be generated
Out []string
}
+
+var Bool = proptools.Bool
+var String = proptools.String
diff --git a/cmd/soong_zip/Android.bp b/jar/Android.bp
similarity index 74%
copy from cmd/soong_zip/Android.bp
copy to jar/Android.bp
index 10896ce..6c2e60e 100644
--- a/cmd/soong_zip/Android.bp
+++ b/jar/Android.bp
@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All rights reserved.
+// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,11 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
+bootstrap_go_package {
+ name: "soong-jar",
+ pkgPath: "android/soong/jar",
srcs: [
- "soong_zip.go",
- "rate_limit.go",
+ "jar.go",
+ ],
+ deps: [
+ "android-archive-zip",
],
}
+
diff --git a/jar/jar.go b/jar/jar.go
new file mode 100644
index 0000000..653e5ee
--- /dev/null
+++ b/jar/jar.go
@@ -0,0 +1,126 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jar
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "strings"
+ "time"
+
+ "android/soong/third_party/zip"
+)
+
+const (
+ MetaDir = "META-INF/"
+ ManifestFile = MetaDir + "MANIFEST.MF"
+ ModuleInfoClass = "module-info.class"
+)
+
+var DefaultTime = time.Date(2008, 1, 1, 0, 0, 0, 0, time.UTC)
+
+var MetaDirExtra = [2]byte{0xca, 0xfe}
+
+// EntryNamesLess tells whether <filepathA> should precede <filepathB> in
+// the order of files with a .jar
+func EntryNamesLess(filepathA string, filepathB string) (less bool) {
+ diff := index(filepathA) - index(filepathB)
+ if diff == 0 {
+ return filepathA < filepathB
+ }
+ return diff < 0
+}
+
+// Treats trailing * as a prefix match
+func patternMatch(pattern, name string) bool {
+ if strings.HasSuffix(pattern, "*") {
+ return strings.HasPrefix(name, strings.TrimSuffix(pattern, "*"))
+ } else {
+ return name == pattern
+ }
+}
+
+var jarOrder = []string{
+ MetaDir,
+ ManifestFile,
+ MetaDir + "*",
+ "*",
+}
+
+func index(name string) int {
+ for i, pattern := range jarOrder {
+ if patternMatch(pattern, name) {
+ return i
+ }
+ }
+ panic(fmt.Errorf("file %q did not match any pattern", name))
+}
+
+func MetaDirFileHeader() *zip.FileHeader {
+ dirHeader := &zip.FileHeader{
+ Name: MetaDir,
+ Extra: []byte{MetaDirExtra[1], MetaDirExtra[0], 0, 0},
+ }
+ dirHeader.SetMode(0700 | os.ModeDir)
+ dirHeader.SetModTime(DefaultTime)
+
+ return dirHeader
+}
+
+// Convert manifest source path to zip header and contents. If path is empty uses a default
+// manifest.
+func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) {
+ b, err := manifestContents(src)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ fh := &zip.FileHeader{
+ Name: ManifestFile,
+ Method: zip.Store,
+ UncompressedSize64: uint64(len(b)),
+ }
+ fh.SetMode(0700)
+ fh.SetModTime(DefaultTime)
+
+ return fh, b, nil
+}
+
+// Convert manifest source path to contents. If path is empty uses a default manifest.
+func manifestContents(src string) ([]byte, error) {
+ var givenBytes []byte
+ var err error
+
+ if src != "" {
+ givenBytes, err = ioutil.ReadFile(src)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ manifestMarker := []byte("Manifest-Version:")
+ header := append(manifestMarker, []byte(" 1.0\nCreated-By: soong_zip\n")...)
+
+ var finalBytes []byte
+ if !bytes.Contains(givenBytes, manifestMarker) {
+ finalBytes = append(append(header, givenBytes...), byte('\n'))
+ } else {
+ finalBytes = givenBytes
+ }
+
+ return finalBytes, nil
+}
diff --git a/java/aapt2.go b/java/aapt2.go
new file mode 100644
index 0000000..84e3729
--- /dev/null
+++ b/java/aapt2.go
@@ -0,0 +1,166 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/google/blueprint"
+
+ "android/soong/android"
+)
+
+const AAPT2_SHARD_SIZE = 100
+
+// Convert input resource file path to output file path.
+// values-[config]/<file>.xml -> values-[config]_<file>.arsc.flat;
+// For other resource file, just replace the last "/" with "_" and
+// add .flat extension.
+func pathToAapt2Path(ctx android.ModuleContext, res android.Path) android.WritablePath {
+
+ name := res.Base()
+ subDir := filepath.Dir(res.String())
+ subDir, lastDir := filepath.Split(subDir)
+ if strings.HasPrefix(lastDir, "values") {
+ name = strings.TrimSuffix(name, ".xml") + ".arsc"
+ }
+ name = lastDir + "_" + name + ".flat"
+ return android.PathForModuleOut(ctx, "aapt2", subDir, name)
+}
+
+func pathsToAapt2Paths(ctx android.ModuleContext, resPaths android.Paths) android.WritablePaths {
+ outPaths := make(android.WritablePaths, len(resPaths))
+
+ for i, res := range resPaths {
+ outPaths[i] = pathToAapt2Path(ctx, res)
+ }
+
+ return outPaths
+}
+
+var aapt2CompileRule = pctx.AndroidStaticRule("aapt2Compile",
+ blueprint.RuleParams{
+ Command: `${config.Aapt2Cmd} compile -o $outDir $cFlags --legacy $in`,
+ CommandDeps: []string{"${config.Aapt2Cmd}"},
+ },
+ "outDir", "cFlags")
+
+func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths) android.WritablePaths {
+ shards := shardPaths(paths, AAPT2_SHARD_SIZE)
+
+ ret := make(android.WritablePaths, 0, len(paths))
+
+ for i, shard := range shards {
+ outPaths := pathsToAapt2Paths(ctx, shard)
+ ret = append(ret, outPaths...)
+
+ shardDesc := ""
+ if i != 0 {
+ shardDesc = " " + strconv.Itoa(i+1)
+ }
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: aapt2CompileRule,
+ Description: "aapt2 compile " + dir.String() + shardDesc,
+ Inputs: shard,
+ Outputs: outPaths,
+ Args: map[string]string{
+ "outDir": android.PathForModuleOut(ctx, "aapt2", dir.String()).String(),
+ "cFlags": "--pseudo-localize",
+ },
+ })
+ }
+
+ sort.Slice(ret, func(i, j int) bool {
+ return ret[i].String() < ret[j].String()
+ })
+ return ret
+}
+
+var aapt2LinkRule = pctx.AndroidStaticRule("aapt2Link",
+ blueprint.RuleParams{
+ Command: `${config.Aapt2Cmd} link -o $out $flags --java $genDir --proguard $proguardOptions $inFlags && ` +
+ `${config.SoongZipCmd} -write_if_changed -jar -o $genJar -C $genDir -D $genDir`,
+ CommandDeps: []string{
+ "${config.Aapt2Cmd}",
+ "${config.SoongZipCmd}",
+ },
+ Restat: true,
+ },
+ "flags", "inFlags", "proguardOptions", "genDir", "genJar")
+
+var fileListToFileRule = pctx.AndroidStaticRule("fileListToFile",
+ blueprint.RuleParams{
+ Command: `cp $out.rsp $out`,
+ Rspfile: "$out.rsp",
+ RspfileContent: "$in",
+ })
+
+func aapt2Link(ctx android.ModuleContext,
+ packageRes, genJar, proguardOptions android.WritablePath,
+ flags []string, deps android.Paths,
+ compiledRes, compiledOverlay android.Paths) {
+
+ genDir := android.PathForModuleGen(ctx, "aapt2", "R")
+
+ var inFlags []string
+
+ if len(compiledRes) > 0 {
+ resFileList := android.PathForModuleOut(ctx, "aapt2", "res.list")
+ // Write out file lists to files
+ ctx.Build(pctx, android.BuildParams{
+ Rule: fileListToFileRule,
+ Description: "resource file list",
+ Inputs: compiledRes,
+ Output: resFileList,
+ })
+
+ deps = append(deps, compiledRes...)
+ deps = append(deps, resFileList)
+ inFlags = append(inFlags, "@"+resFileList.String())
+ }
+
+ if len(compiledOverlay) > 0 {
+ overlayFileList := android.PathForModuleOut(ctx, "aapt2", "overlay.list")
+ ctx.Build(pctx, android.BuildParams{
+ Rule: fileListToFileRule,
+ Description: "overlay resource file list",
+ Inputs: compiledOverlay,
+ Output: overlayFileList,
+ })
+
+ deps = append(deps, compiledOverlay...)
+ deps = append(deps, overlayFileList)
+ inFlags = append(inFlags, "-R", "@"+overlayFileList.String())
+ }
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: aapt2LinkRule,
+ Description: "aapt2 link",
+ Implicits: deps,
+ Output: packageRes,
+ ImplicitOutputs: android.WritablePaths{proguardOptions, genJar},
+ Args: map[string]string{
+ "flags": strings.Join(flags, " "),
+ "inFlags": strings.Join(inFlags, " "),
+ "proguardOptions": proguardOptions.String(),
+ "genDir": genDir.String(),
+ "genJar": genJar.String(),
+ },
+ })
+}
diff --git a/java/androidmk.go b/java/androidmk.go
index 086ba7a..af91a33 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -16,20 +16,168 @@
import (
"fmt"
+ "io"
+ "strings"
+
+ "github.com/google/blueprint/proptools"
"android/soong/android"
)
-func (*Library) AndroidMk() (ret android.AndroidMkData, err error) {
- ret.Class = "JAVA_LIBRARIES"
- // TODO
- err = fmt.Errorf("Not yet implemented")
- return
+func (library *Library) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Class: "JAVA_LIBRARIES",
+ OutputFile: android.OptionalPathForPath(library.implementationJarFile),
+ Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk",
+ Extra: []android.AndroidMkExtraFunc{
+ func(w io.Writer, outputFile android.Path) {
+ if len(library.logtagsSrcs) > 0 {
+ var logtags []string
+ for _, l := range library.logtagsSrcs {
+ logtags = append(logtags, l.Rel())
+ }
+ fmt.Fprintln(w, "LOCAL_LOGTAGS_FILES :=", strings.Join(logtags, " "))
+ }
+
+ if library.properties.Installable != nil && *library.properties.Installable == false {
+ fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
+ }
+ if library.dexJarFile != nil {
+ fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", library.dexJarFile.String())
+ if library.deviceProperties.Dex_preopt.Enabled != nil {
+ fmt.Fprintln(w, "LOCAL_DEX_PREOPT :=", *library.deviceProperties.Dex_preopt.Enabled)
+ }
+ if library.deviceProperties.Dex_preopt.App_image != nil {
+ fmt.Fprintln(w, "LOCAL_DEX_PREOPT_APP_IMAGE :=", *library.deviceProperties.Dex_preopt.App_image)
+ }
+ if library.deviceProperties.Dex_preopt.Profile_guided != nil {
+ fmt.Fprintln(w, "LOCAL_DEX_PREOPT_GENERATE_PROFILE :=", *library.deviceProperties.Dex_preopt.Profile_guided)
+ }
+ if library.deviceProperties.Dex_preopt.Profile != nil {
+ fmt.Fprintln(w, "LOCAL_DEX_PREOPT_GENERATE_PROFILE := true")
+ fmt.Fprintln(w, "LOCAL_DEX_PREOPT_PROFILE_CLASS_LISTING := $(LOCAL_PATH)/"+*library.deviceProperties.Dex_preopt.Profile)
+ }
+ }
+ fmt.Fprintln(w, "LOCAL_SDK_VERSION :=", String(library.deviceProperties.Sdk_version))
+ fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", library.headerJarFile.String())
+
+ if library.jacocoReportClassesFile != nil {
+ fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", library.jacocoReportClassesFile.String())
+ }
+
+ // Temporary hack: export sources used to compile framework.jar to Make
+ // to be used for droiddoc
+ // TODO(ccross): remove this once droiddoc is in soong
+ if library.Name() == "framework" {
+ fmt.Fprintln(w, "SOONG_FRAMEWORK_SRCS :=", strings.Join(library.compiledJavaSrcs.Strings(), " "))
+ fmt.Fprintln(w, "SOONG_FRAMEWORK_SRCJARS :=", strings.Join(library.compiledSrcJars.Strings(), " "))
+ }
+ },
+ },
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ android.WriteAndroidMkData(w, data)
+
+ if proptools.Bool(library.deviceProperties.Hostdex) && !library.Host() {
+ fmt.Fprintln(w, "include $(CLEAR_VARS)")
+ fmt.Fprintln(w, "LOCAL_MODULE := "+name+"-hostdex")
+ fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true")
+ fmt.Fprintln(w, "LOCAL_MODULE_CLASS := JAVA_LIBRARIES")
+ fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", library.implementationJarFile.String())
+ if library.properties.Installable != nil && *library.properties.Installable == false {
+ fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
+ }
+ if library.dexJarFile != nil {
+ fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", library.dexJarFile.String())
+ }
+ fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", library.implementationJarFile.String())
+ fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(data.Required, " "))
+ fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_java_prebuilt.mk")
+ }
+ },
+ }
}
-func (*Prebuilt) AndroidMk() (ret android.AndroidMkData, err error) {
- ret.Class = "JAVA_LIBRARIES"
- // TODO
- err = fmt.Errorf("Not yet implemented")
- return
+func (prebuilt *Import) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Class: "JAVA_LIBRARIES",
+ OutputFile: android.OptionalPathForPath(prebuilt.combinedClasspathFile),
+ Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk",
+ Extra: []android.AndroidMkExtraFunc{
+ func(w io.Writer, outputFile android.Path) {
+ fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := ", !proptools.Bool(prebuilt.properties.Installable))
+ fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", prebuilt.combinedClasspathFile.String())
+ fmt.Fprintln(w, "LOCAL_SDK_VERSION :=", String(prebuilt.properties.Sdk_version))
+ },
+ },
+ }
+}
+
+func (binary *Binary) AndroidMk() android.AndroidMkData {
+
+ if !binary.isWrapperVariant {
+ return android.AndroidMkData{
+ Class: "JAVA_LIBRARIES",
+ OutputFile: android.OptionalPathForPath(binary.implementationJarFile),
+ Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk",
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ android.WriteAndroidMkData(w, data)
+
+ fmt.Fprintln(w, "jar_installed_module := $(LOCAL_INSTALLED_MODULE)")
+ },
+ }
+ } else {
+ return android.AndroidMkData{
+ Class: "EXECUTABLES",
+ OutputFile: android.OptionalPathForPath(binary.wrapperFile),
+ Extra: []android.AndroidMkExtraFunc{
+ func(w io.Writer, outputFile android.Path) {
+ fmt.Fprintln(w, "LOCAL_STRIP_MODULE := false")
+ },
+ },
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ android.WriteAndroidMkData(w, data)
+
+ // Ensure that the wrapper script timestamp is always updated when the jar is updated
+ fmt.Fprintln(w, "$(LOCAL_INSTALLED_MODULE): $(jar_installed_module)")
+ fmt.Fprintln(w, "jar_installed_module :=")
+ },
+ }
+ }
+}
+
+func (app *AndroidApp) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Class: "APPS",
+ OutputFile: android.OptionalPathForPath(app.outputFile),
+ Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk",
+ Extra: []android.AndroidMkExtraFunc{
+ func(w io.Writer, outputFile android.Path) {
+ if Bool(app.appProperties.Export_package_resources) {
+ if app.dexJarFile != nil {
+ fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", app.dexJarFile.String())
+ }
+ fmt.Fprintln(w, "LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE :=", app.exportPackage.String())
+
+ if app.jacocoReportClassesFile != nil {
+ fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", app.jacocoReportClassesFile.String())
+ }
+
+ if app.Name() == "framework-res" {
+ fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(TARGET_OUT_JAVA_LIBRARIES)")
+ // Make base_rules.mk not put framework-res in a subdirectory called
+ // framework_res.
+ fmt.Fprintln(w, "LOCAL_NO_STANDARD_LIBRARIES := true")
+ }
+
+ if len(app.rroDirs) > 0 {
+ fmt.Fprintln(w, "LOCAL_SOONG_RRO_DIRS :=", strings.Join(app.rroDirs.Strings(), " "))
+ }
+ fmt.Fprintln(w, "LOCAL_EXPORT_PACKAGE_RESOURCES :=",
+ Bool(app.appProperties.Export_package_resources))
+ fmt.Fprintln(w, "LOCAL_FULL_MANIFEST_FILE :=", app.manifestPath.String())
+ }
+ },
+ },
+ }
+
}
diff --git a/java/app.go b/java/app.go
index 8a221ef..ed6a9db 100644
--- a/java/app.go
+++ b/java/app.go
@@ -20,11 +20,16 @@
"path/filepath"
"strings"
- "github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
"android/soong/android"
)
+func init() {
+ android.RegisterPreSingletonType("overlay", OverlaySingletonFactory)
+ android.RegisterModuleType("android_app", AndroidAppFactory)
+}
+
// AAR prebuilts
// AndroidManifest.xml merging
// package splits
@@ -32,14 +37,14 @@
type androidAppProperties struct {
// path to a certificate, or the name of a certificate in the default
// certificate directory, or blank to use the default product certificate
- Certificate string
+ Certificate *string
// paths to extra certificates to sign the apk with
Additional_certificates []string
// If set, create package-export.apk, which other packages can
// use to get PRODUCT-agnostic resource data like IDs and type definitions.
- Export_package_resources bool
+ Export_package_resources *bool
// flags passed to aapt when creating the apk
Aaptflags []string
@@ -52,8 +57,10 @@
Asset_dirs []string
// list of directories relative to the Blueprints file containing
- // Java resources
- Android_resource_dirs []string
+ // Android resources
+ Resource_dirs []string
+
+ Instrumentation_for *string
}
type AndroidApp struct {
@@ -61,16 +68,18 @@
appProperties androidAppProperties
- aaptJavaFileList android.Path
- exportPackage android.Path
+ aaptSrcJar android.Path
+ exportPackage android.Path
+ rroDirs android.Paths
+ manifestPath android.Path
}
func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) {
a.Module.deps(ctx)
- if !a.properties.No_standard_libraries {
- switch a.deviceProperties.Sdk_version { // TODO: Res_sdk_version?
- case "current", "system_current", "":
+ if !Bool(a.properties.No_framework_libs) && !Bool(a.properties.No_standard_libs) {
+ switch String(a.deviceProperties.Sdk_version) { // TODO: Res_sdk_version?
+ case "current", "system_current", "test_current", "":
ctx.AddDependency(ctx.Module(), frameworkResTag, "framework-res")
default:
// We'll already have a dependency on an sdk prebuilt android.jar
@@ -79,38 +88,29 @@
}
func (a *AndroidApp) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- aaptFlags, aaptDeps, hasResources := a.aaptFlags(ctx)
+ linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, manifestPath := a.aapt2Flags(ctx)
- if hasResources {
- // First generate R.java so we can build the .class files
- aaptRJavaFlags := append([]string(nil), aaptFlags...)
+ packageRes := android.PathForModuleOut(ctx, "package-res.apk")
+ srcJar := android.PathForModuleGen(ctx, "R.jar")
+ proguardOptionsFile := android.PathForModuleGen(ctx, "proguard.options")
- publicResourcesFile, proguardOptionsFile, aaptJavaFileList :=
- CreateResourceJavaFiles(ctx, aaptRJavaFlags, aaptDeps)
- a.aaptJavaFileList = aaptJavaFileList
- a.ExtraSrcLists = append(a.ExtraSrcLists, aaptJavaFileList)
-
- if a.appProperties.Export_package_resources {
- aaptPackageFlags := append([]string(nil), aaptFlags...)
- var hasProduct bool
- for _, f := range aaptPackageFlags {
- if strings.HasPrefix(f, "--product") {
- hasProduct = true
- break
- }
- }
-
- if !hasProduct {
- aaptPackageFlags = append(aaptPackageFlags,
- "--product "+ctx.AConfig().ProductAaptCharacteristics())
- }
- a.exportPackage = CreateExportPackage(ctx, aaptPackageFlags, aaptDeps)
- ctx.CheckbuildFile(a.exportPackage)
- }
- ctx.CheckbuildFile(publicResourcesFile)
- ctx.CheckbuildFile(proguardOptionsFile)
- ctx.CheckbuildFile(aaptJavaFileList)
+ var compiledRes, compiledOverlay android.Paths
+ for _, dir := range resDirs {
+ compiledRes = append(compiledRes, aapt2Compile(ctx, dir.dir, dir.files).Paths()...)
}
+ for _, dir := range overlayDirs {
+ compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files).Paths()...)
+ }
+
+ aapt2Link(ctx, packageRes, srcJar, proguardOptionsFile,
+ linkFlags, linkDeps, compiledRes, compiledOverlay)
+
+ a.exportPackage = packageRes
+ a.aaptSrcJar = srcJar
+
+ ctx.CheckbuildFile(proguardOptionsFile)
+ ctx.CheckbuildFile(a.exportPackage)
+ ctx.CheckbuildFile(a.aaptSrcJar)
// apps manifests are handled by aapt, don't let Module see them
a.properties.Manifest = nil
@@ -119,27 +119,19 @@
// a.properties.Proguard.Enabled = true
//}
- a.Module.compile(ctx)
-
- aaptPackageFlags := append([]string(nil), aaptFlags...)
- var hasProduct bool
- for _, f := range aaptPackageFlags {
- if strings.HasPrefix(f, "--product") {
- hasProduct = true
- break
- }
+ if String(a.appProperties.Instrumentation_for) == "" {
+ a.properties.Instrument = true
}
- if !hasProduct {
- aaptPackageFlags = append(aaptPackageFlags,
- "--product "+ctx.AConfig().ProductAaptCharacteristics())
+ if ctx.ModuleName() != "framework-res" {
+ a.Module.compile(ctx, a.aaptSrcJar)
}
- certificate := a.appProperties.Certificate
+ certificate := String(a.appProperties.Certificate)
if certificate == "" {
- certificate = ctx.AConfig().DefaultAppCertificate(ctx).String()
+ certificate = ctx.Config().DefaultAppCertificate(ctx).String()
} else if dir, _ := filepath.Split(certificate); dir == "" {
- certificate = filepath.Join(ctx.AConfig().DefaultAppCertificateDir(ctx).String(), certificate)
+ certificate = filepath.Join(ctx.Config().DefaultAppCertificateDir(ctx).String(), certificate)
} else {
certificate = filepath.Join(android.PathForSource(ctx).String(), certificate)
}
@@ -149,8 +141,20 @@
certificates = append(certificates, filepath.Join(android.PathForSource(ctx).String(), c))
}
- a.outputFile = CreateAppPackage(ctx, aaptPackageFlags, a.outputFile, certificates)
- ctx.InstallFileName(android.PathForModuleInstall(ctx, "app"), ctx.ModuleName()+".apk", a.outputFile)
+ packageFile := android.PathForModuleOut(ctx, "package.apk")
+
+ CreateAppPackage(ctx, packageFile, a.exportPackage, a.outputFile, certificates)
+
+ a.outputFile = packageFile
+ a.rroDirs = rroDirs
+ a.manifestPath = manifestPath
+
+ if ctx.ModuleName() == "framework-res" {
+ // framework-res.apk is installed as system/framework/framework-res.apk
+ ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), ctx.ModuleName()+".apk", a.outputFile)
+ } else {
+ ctx.InstallFile(android.PathForModuleInstall(ctx, "app"), ctx.ModuleName()+".apk", a.outputFile)
+ }
}
var aaptIgnoreFilenames = []string{
@@ -165,57 +169,57 @@
"*~",
}
-func (a *AndroidApp) aaptFlags(ctx android.ModuleContext) ([]string, android.Paths, bool) {
- aaptFlags := a.appProperties.Aaptflags
+type globbedResourceDir struct {
+ dir android.Path
+ files android.Paths
+}
+
+func (a *AndroidApp) aapt2Flags(ctx android.ModuleContext) (flags []string, deps android.Paths,
+ resDirs, overlayDirs []globbedResourceDir, rroDirs android.Paths, manifestPath android.Path) {
+
hasVersionCode := false
hasVersionName := false
- for _, f := range aaptFlags {
+ hasProduct := false
+ for _, f := range a.appProperties.Aaptflags {
if strings.HasPrefix(f, "--version-code") {
hasVersionCode = true
} else if strings.HasPrefix(f, "--version-name") {
hasVersionName = true
+ } else if strings.HasPrefix(f, "--product") {
+ hasProduct = true
}
}
- if true /* is not a test */ {
- aaptFlags = append(aaptFlags, "-z")
- }
+ var linkFlags []string
+ // Flags specified in Android.bp
+ linkFlags = append(linkFlags, a.appProperties.Aaptflags...)
+
+ linkFlags = append(linkFlags, "--no-static-lib-packages")
+
+ // Find implicit or explicit asset and resource dirs
assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.appProperties.Asset_dirs, "assets")
- resourceDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.appProperties.Android_resource_dirs, "res")
+ resourceDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.appProperties.Resource_dirs, "res")
- var overlayResourceDirs android.Paths
- // For every resource directory, check if there is an overlay directory with the same path.
- // If found, it will be prepended to the list of resource directories.
- for _, overlayDir := range ctx.AConfig().ResourceOverlays() {
- for _, resourceDir := range resourceDirs {
- overlay := overlayDir.OverlayPath(ctx, resourceDir)
- if overlay.Valid() {
- overlayResourceDirs = append(overlayResourceDirs, overlay.Path())
- }
- }
+ var linkDeps android.Paths
+
+ // Glob directories into lists of paths
+ for _, dir := range resourceDirs {
+ resDirs = append(resDirs, globbedResourceDir{
+ dir: dir,
+ files: resourceGlob(ctx, dir),
+ })
+ resOverlayDirs, resRRODirs := overlayResourceGlob(ctx, dir)
+ overlayDirs = append(overlayDirs, resOverlayDirs...)
+ rroDirs = append(rroDirs, resRRODirs...)
}
- if len(overlayResourceDirs) > 0 {
- resourceDirs = append(overlayResourceDirs, resourceDirs...)
+ var assetFiles android.Paths
+ for _, dir := range assetDirs {
+ assetFiles = append(assetFiles, resourceGlob(ctx, dir)...)
}
- // aapt needs to rerun if any files are added or modified in the assets or resource directories,
- // use glob to create a filelist.
- var aaptDeps android.Paths
- var hasResources bool
- for _, d := range resourceDirs {
- newDeps := ctx.Glob(filepath.Join(d.String(), "**/*"), aaptIgnoreFilenames)
- aaptDeps = append(aaptDeps, newDeps...)
- if len(newDeps) > 0 {
- hasResources = true
- }
- }
- for _, d := range assetDirs {
- newDeps := ctx.Glob(filepath.Join(d.String(), "**/*"), aaptIgnoreFilenames)
- aaptDeps = append(aaptDeps, newDeps...)
- }
-
+ // App manifest file
var manifestFile string
if a.properties.Manifest == nil {
manifestFile = "AndroidManifest.xml"
@@ -223,59 +227,79 @@
manifestFile = *a.properties.Manifest
}
- manifestPath := android.PathForModuleSrc(ctx, manifestFile)
- aaptDeps = append(aaptDeps, manifestPath)
+ manifestPath = android.PathForModuleSrc(ctx, manifestFile)
+ linkFlags = append(linkFlags, "--manifest "+manifestPath.String())
+ linkDeps = append(linkDeps, manifestPath)
- aaptFlags = append(aaptFlags, "-M "+manifestPath.String())
- aaptFlags = append(aaptFlags, android.JoinWithPrefix(assetDirs.Strings(), "-A "))
- aaptFlags = append(aaptFlags, android.JoinWithPrefix(resourceDirs.Strings(), "-S "))
+ linkFlags = append(linkFlags, android.JoinWithPrefix(assetDirs.Strings(), "-A "))
+ linkDeps = append(linkDeps, assetFiles...)
- ctx.VisitDirectDeps(func(module blueprint.Module) {
- var depFile android.OptionalPath
- if sdkDep, ok := module.(sdkDependency); ok {
- depFile = android.OptionalPathForPath(sdkDep.ClasspathFile())
- } else if javaDep, ok := module.(Dependency); ok {
+ // Include dirs
+ ctx.VisitDirectDeps(func(module android.Module) {
+ var depFiles android.Paths
+ if javaDep, ok := module.(Dependency); ok {
+ // TODO: shared android libraries
if ctx.OtherModuleName(module) == "framework-res" {
- depFile = android.OptionalPathForPath(javaDep.(*AndroidApp).exportPackage)
+ depFiles = android.Paths{javaDep.(*AndroidApp).exportPackage}
}
}
- if depFile.Valid() {
- aaptFlags = append(aaptFlags, "-I "+depFile.String())
- aaptDeps = append(aaptDeps, depFile.Path())
+
+ for _, dep := range depFiles {
+ linkFlags = append(linkFlags, "-I "+dep.String())
}
+ linkDeps = append(linkDeps, depFiles...)
})
- sdkVersion := a.deviceProperties.Sdk_version
- if sdkVersion == "" {
- sdkVersion = ctx.AConfig().PlatformSdkVersion()
+ // SDK version flags
+ sdkVersion := String(a.deviceProperties.Sdk_version)
+ switch sdkVersion {
+ case "", "current", "system_current", "test_current":
+ sdkVersion = proptools.NinjaEscape([]string{ctx.Config().AppsDefaultVersionName()})[0]
}
- aaptFlags = append(aaptFlags, "--min-sdk-version "+sdkVersion)
- aaptFlags = append(aaptFlags, "--target-sdk-version "+sdkVersion)
+ linkFlags = append(linkFlags, "--min-sdk-version "+sdkVersion)
+ linkFlags = append(linkFlags, "--target-sdk-version "+sdkVersion)
+ // Product characteristics
+ if !hasProduct && len(ctx.Config().ProductAAPTCharacteristics()) > 0 {
+ linkFlags = append(linkFlags, "--product", ctx.Config().ProductAAPTCharacteristics())
+ }
+
+ // Product AAPT config
+ for _, aaptConfig := range ctx.Config().ProductAAPTConfig() {
+ linkFlags = append(linkFlags, "-c", aaptConfig)
+ }
+
+ // Product AAPT preferred config
+ if len(ctx.Config().ProductAAPTPreferredConfig()) > 0 {
+ linkFlags = append(linkFlags, "--preferred-density", ctx.Config().ProductAAPTPreferredConfig())
+ }
+
+ // Version code
if !hasVersionCode {
- aaptFlags = append(aaptFlags, "--version-code "+ctx.AConfig().PlatformSdkVersion())
+ linkFlags = append(linkFlags, "--version-code", ctx.Config().PlatformSdkVersion())
}
if !hasVersionName {
- aaptFlags = append(aaptFlags,
- "--version-name "+ctx.AConfig().PlatformVersion()+"-"+ctx.AConfig().BuildNumber())
+ versionName := proptools.NinjaEscape([]string{ctx.Config().AppsDefaultVersionName()})[0]
+ linkFlags = append(linkFlags, "--version-name ", versionName)
+ }
+
+ if String(a.appProperties.Instrumentation_for) != "" {
+ linkFlags = append(linkFlags,
+ "--rename-instrumentation-target-package",
+ String(a.appProperties.Instrumentation_for))
}
// TODO: LOCAL_PACKAGE_OVERRIDES
// $(addprefix --rename-manifest-package , $(PRIVATE_MANIFEST_PACKAGE_NAME)) \
- // TODO: LOCAL_INSTRUMENTATION_FOR
- // $(addprefix --rename-instrumentation-target-package , $(PRIVATE_MANIFEST_INSTRUMENTATION_FOR))
-
- return aaptFlags, aaptDeps, hasResources
+ return linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, manifestPath
}
func AndroidAppFactory() android.Module {
module := &AndroidApp{}
- module.deviceProperties.Dex = true
-
module.AddProperties(
&module.Module.properties,
&module.Module.deviceProperties,
@@ -284,3 +308,114 @@
android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon)
return module
}
+
+func resourceGlob(ctx android.ModuleContext, dir android.Path) android.Paths {
+ var ret android.Paths
+ files := ctx.Glob(filepath.Join(dir.String(), "**/*"), aaptIgnoreFilenames)
+ for _, f := range files {
+ if isDir, err := ctx.Fs().IsDir(f.String()); err != nil {
+ ctx.ModuleErrorf("error in IsDir(%s): %s", f.String(), err.Error())
+ return nil
+ } else if !isDir {
+ ret = append(ret, f)
+ }
+ }
+ return ret
+}
+
+type overlayGlobResult struct {
+ dir string
+ paths android.DirectorySortedPaths
+
+ // Set to true of the product has selected that values in this overlay should not be moved to
+ // Runtime Resource Overlay (RRO) packages.
+ excludeFromRRO bool
+}
+
+const overlayDataKey = "overlayDataKey"
+
+func overlayResourceGlob(ctx android.ModuleContext, dir android.Path) (res []globbedResourceDir,
+ rroDirs android.Paths) {
+
+ overlayData := ctx.Config().Get(overlayDataKey).([]overlayGlobResult)
+
+ // Runtime resource overlays (RRO) may be turned on by the product config for some modules
+ rroEnabled := false
+ enforceRROTargets := ctx.Config().ProductVariables.EnforceRROTargets
+ if enforceRROTargets != nil {
+ if len(*enforceRROTargets) == 1 && (*enforceRROTargets)[0] == "*" {
+ rroEnabled = true
+ } else if inList(ctx.ModuleName(), *enforceRROTargets) {
+ rroEnabled = true
+ }
+ }
+
+ for _, data := range overlayData {
+ files := data.paths.PathsInDirectory(filepath.Join(data.dir, dir.String()))
+ if len(files) > 0 {
+ overlayModuleDir := android.PathForSource(ctx, data.dir, dir.String())
+ // If enforce RRO is enabled for this module and this overlay is not in the
+ // exclusion list, ignore the overlay. The list of ignored overlays will be
+ // passed to Make to be turned into an RRO package.
+ if rroEnabled && !data.excludeFromRRO {
+ rroDirs = append(rroDirs, overlayModuleDir)
+ } else {
+ res = append(res, globbedResourceDir{
+ dir: overlayModuleDir,
+ files: files,
+ })
+ }
+ }
+ }
+
+ return res, rroDirs
+}
+
+func OverlaySingletonFactory() android.Singleton {
+ return overlaySingleton{}
+}
+
+type overlaySingleton struct{}
+
+func (overlaySingleton) GenerateBuildActions(ctx android.SingletonContext) {
+
+ // Specific overlays may be excluded from Runtime Resource Overlays by the product config
+ var rroExcludedOverlays []string
+ if ctx.Config().ProductVariables.EnforceRROExcludedOverlays != nil {
+ rroExcludedOverlays = *ctx.Config().ProductVariables.EnforceRROExcludedOverlays
+ }
+
+ var overlayData []overlayGlobResult
+ for _, overlay := range ctx.Config().ResourceOverlays() {
+ var result overlayGlobResult
+ result.dir = overlay
+
+ // Mark overlays that will not have Runtime Resource Overlays enforced on them
+ for _, exclude := range rroExcludedOverlays {
+ if strings.HasPrefix(overlay, exclude) {
+ result.excludeFromRRO = true
+ }
+ }
+
+ files, err := ctx.GlobWithDeps(filepath.Join(overlay, "**/*"), aaptIgnoreFilenames)
+ if err != nil {
+ ctx.Errorf("failed to glob resource dir %q: %s", overlay, err.Error())
+ continue
+ }
+ var paths android.Paths
+ for _, f := range files {
+ if isDir, err := ctx.Fs().IsDir(f); err != nil {
+ ctx.Errorf("error in IsDir(%s): %s", f, err.Error())
+ return
+ } else if !isDir {
+ paths = append(paths, android.PathForSource(ctx, f))
+ }
+ }
+ result.paths = android.PathsToDirectorySortedPaths(paths)
+ overlayData = append(overlayData, result)
+ }
+
+ ctx.Config().Once(overlayDataKey, func() interface{} {
+ return overlayData
+ })
+}
diff --git a/java/app_builder.go b/java/app_builder.go
index 55fded5..676ed58 100644
--- a/java/app_builder.go
+++ b/java/app_builder.go
@@ -27,35 +27,11 @@
)
var (
- aaptCreateResourceJavaFile = pctx.AndroidStaticRule("aaptCreateResourceJavaFile",
- blueprint.RuleParams{
- Command: `rm -rf "$javaDir" && mkdir -p "$javaDir" && ` +
- `$aaptCmd package -m $aaptFlags -P $publicResourcesFile -G $proguardOptionsFile ` +
- `-J $javaDir || ( rm -rf "$javaDir/*"; exit 41 ) && ` +
- `find $javaDir -name "*.java" > $javaFileList`,
- CommandDeps: []string{"$aaptCmd"},
- },
- "aaptFlags", "publicResourcesFile", "proguardOptionsFile", "javaDir", "javaFileList")
-
- aaptCreateAssetsPackage = pctx.AndroidStaticRule("aaptCreateAssetsPackage",
- blueprint.RuleParams{
- Command: `rm -f $out && $aaptCmd package $aaptFlags -F $out`,
- CommandDeps: []string{"$aaptCmd"},
- },
- "aaptFlags", "publicResourcesFile", "proguardOptionsFile", "javaDir", "javaFileList")
-
- aaptAddResources = pctx.AndroidStaticRule("aaptAddResources",
- blueprint.RuleParams{
- // TODO: add-jni-shared-libs-to-package
- Command: `cp -f $in $out.tmp && $aaptCmd package -u $aaptFlags -F $out.tmp && mv $out.tmp $out`,
- CommandDeps: []string{"$aaptCmd"},
- },
- "aaptFlags")
-
signapk = pctx.AndroidStaticRule("signapk",
blueprint.RuleParams{
- Command: `java -jar $signapkCmd $certificates $in $out`,
- CommandDeps: []string{"$signapkCmd"},
+ Command: `${config.JavaCmd} -Djava.library.path=$$(dirname $signapkJniLibrary) ` +
+ `-jar $signapkCmd $certificates $in $out`,
+ CommandDeps: []string{"$signapkCmd", "$signapkJniLibrary"},
},
"certificates")
@@ -73,79 +49,50 @@
pctx.SourcePathVariable("androidManifestMergerCmd", "prebuilts/devtools/tools/lib/manifest-merger.jar")
pctx.HostBinToolVariable("aaptCmd", "aapt")
pctx.HostJavaToolVariable("signapkCmd", "signapk.jar")
+ // TODO(ccross): this should come from the signapk dependencies, but we don't have any way
+ // to express host JNI dependencies yet.
+ pctx.HostJNIToolVariable("signapkJniLibrary", "libconscrypt_openjdk_jni")
}
-func CreateResourceJavaFiles(ctx android.ModuleContext, flags []string,
- deps android.Paths) (android.Path, android.Path, android.Path) {
- javaDir := android.PathForModuleGen(ctx, "R")
- javaFileList := android.PathForModuleOut(ctx, "R.filelist")
- publicResourcesFile := android.PathForModuleOut(ctx, "public_resources.xml")
- proguardOptionsFile := android.PathForModuleOut(ctx, "proguard.options")
-
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: aaptCreateResourceJavaFile,
- Description: "aapt create R.java",
- Outputs: android.WritablePaths{publicResourcesFile, proguardOptionsFile, javaFileList},
- Implicits: deps,
- Args: map[string]string{
- "aaptFlags": strings.Join(flags, " "),
- "publicResourcesFile": publicResourcesFile.String(),
- "proguardOptionsFile": proguardOptionsFile.String(),
- "javaDir": javaDir.String(),
- "javaFileList": javaFileList.String(),
- },
+var combineApk = pctx.AndroidStaticRule("combineApk",
+ blueprint.RuleParams{
+ Command: `${config.MergeZipsCmd} $out $in`,
+ CommandDeps: []string{"${config.MergeZipsCmd}"},
})
- return publicResourcesFile, proguardOptionsFile, javaFileList
-}
+func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath,
+ resJarFile, dexJarFile android.Path, certificates []string) {
-func CreateExportPackage(ctx android.ModuleContext, flags []string, deps android.Paths) android.ModuleOutPath {
- outputFile := android.PathForModuleOut(ctx, "package-export.apk")
+ // TODO(ccross): JNI libs
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: aaptCreateAssetsPackage,
- Description: "aapt export package",
- Output: outputFile,
- Implicits: deps,
- Args: map[string]string{
- "aaptFlags": strings.Join(flags, " "),
- },
+ unsignedApk := android.PathForModuleOut(ctx, "unsigned.apk")
+
+ inputs := android.Paths{resJarFile}
+ if dexJarFile != nil {
+ inputs = append(inputs, dexJarFile)
+ }
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: combineApk,
+ Inputs: inputs,
+ Output: unsignedApk,
})
- return outputFile
-}
-
-func CreateAppPackage(ctx android.ModuleContext, flags []string, jarFile android.Path,
- certificates []string) android.Path {
-
- resourceApk := android.PathForModuleOut(ctx, "resources.apk")
-
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: aaptAddResources,
- Description: "aapt package",
- Output: resourceApk,
- Input: jarFile,
- Args: map[string]string{
- "aaptFlags": strings.Join(flags, " "),
- },
- })
-
- outputFile := android.PathForModuleOut(ctx, "package.apk")
-
var certificateArgs []string
for _, c := range certificates {
certificateArgs = append(certificateArgs, c+".x509.pem", c+".pk8")
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ // TODO(ccross): sometimes uncompress dex
+ // TODO(ccross): sometimes strip dex
+
+ ctx.Build(pctx, android.BuildParams{
Rule: signapk,
Description: "signapk",
Output: outputFile,
- Input: resourceApk,
+ Input: unsignedApk,
Args: map[string]string{
"certificates": strings.Join(certificateArgs, " "),
},
})
-
- return outputFile
}
diff --git a/java/app_test.go b/java/app_test.go
new file mode 100644
index 0000000..73ac3f7
--- /dev/null
+++ b/java/app_test.go
@@ -0,0 +1,239 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+ "android/soong/android"
+ "reflect"
+ "sort"
+ "testing"
+)
+
+var (
+ resourceFiles = []string{
+ "res/layout/layout.xml",
+ "res/values/strings.xml",
+ "res/values-en-rUS/strings.xml",
+ }
+
+ compiledResourceFiles = []string{
+ "aapt2/res/layout_layout.xml.flat",
+ "aapt2/res/values_strings.arsc.flat",
+ "aapt2/res/values-en-rUS_strings.arsc.flat",
+ }
+)
+
+func testAppContext(config android.Config, bp string, fs map[string][]byte) *android.TestContext {
+ appFS := map[string][]byte{}
+ for k, v := range fs {
+ appFS[k] = v
+ }
+
+ for _, file := range resourceFiles {
+ appFS[file] = nil
+ }
+
+ return testContext(config, bp, appFS)
+}
+
+func testApp(t *testing.T, bp string) *android.TestContext {
+ config := testConfig(nil)
+
+ ctx := testAppContext(config, bp, nil)
+
+ run(t, ctx, config)
+
+ return ctx
+}
+
+func TestApp(t *testing.T) {
+ ctx := testApp(t, `
+ android_app {
+ name: "foo",
+ srcs: ["a.java"],
+ }
+ `)
+
+ foo := ctx.ModuleForTests("foo", "android_common")
+
+ expectedLinkImplicits := []string{"AndroidManifest.xml"}
+
+ frameworkRes := ctx.ModuleForTests("framework-res", "android_common")
+ expectedLinkImplicits = append(expectedLinkImplicits,
+ frameworkRes.Output("package-res.apk").Output.String())
+
+ // Test the mapping from input files to compiled output file names
+ compile := foo.Output(compiledResourceFiles[0])
+ if !reflect.DeepEqual(resourceFiles, compile.Inputs.Strings()) {
+ t.Errorf("expected aapt2 compile inputs expected:\n %#v\n got:\n %#v",
+ resourceFiles, compile.Inputs.Strings())
+ }
+
+ compiledResourceOutputs := compile.Outputs.Strings()
+ sort.Strings(compiledResourceOutputs)
+
+ expectedLinkImplicits = append(expectedLinkImplicits, compiledResourceOutputs...)
+
+ list := foo.Output("aapt2/res.list")
+ expectedLinkImplicits = append(expectedLinkImplicits, list.Output.String())
+
+ // Check that the link rule uses
+ res := ctx.ModuleForTests("foo", "android_common").Output("package-res.apk")
+ if !reflect.DeepEqual(expectedLinkImplicits, res.Implicits.Strings()) {
+ t.Errorf("expected aapt2 link implicits expected:\n %#v\n got:\n %#v",
+ expectedLinkImplicits, res.Implicits.Strings())
+ }
+}
+
+var testEnforceRROTests = []struct {
+ name string
+ enforceRROTargets []string
+ enforceRROExcludedOverlays []string
+ fooOverlayFiles []string
+ fooRRODirs []string
+ barOverlayFiles []string
+ barRRODirs []string
+}{
+ {
+ name: "no RRO",
+ enforceRROTargets: nil,
+ enforceRROExcludedOverlays: nil,
+ fooOverlayFiles: []string{
+ "device/vendor/blah/overlay/foo/res/values/strings.xml",
+ "device/vendor/blah/static_overlay/foo/res/values/strings.xml",
+ },
+ fooRRODirs: nil,
+ barOverlayFiles: []string{
+ "device/vendor/blah/overlay/bar/res/values/strings.xml",
+ "device/vendor/blah/static_overlay/bar/res/values/strings.xml",
+ },
+ barRRODirs: nil,
+ },
+ {
+ name: "enforce RRO on foo",
+ enforceRROTargets: []string{"foo"},
+ enforceRROExcludedOverlays: []string{"device/vendor/blah/static_overlay"},
+ fooOverlayFiles: []string{
+ "device/vendor/blah/static_overlay/foo/res/values/strings.xml",
+ },
+ fooRRODirs: []string{
+ "device/vendor/blah/overlay/foo/res",
+ },
+ barOverlayFiles: []string{
+ "device/vendor/blah/overlay/bar/res/values/strings.xml",
+ "device/vendor/blah/static_overlay/bar/res/values/strings.xml",
+ },
+ barRRODirs: nil,
+ },
+ {
+ name: "enforce RRO on all",
+ enforceRROTargets: []string{"*"},
+ enforceRROExcludedOverlays: []string{"device/vendor/blah/static_overlay"},
+ fooOverlayFiles: []string{
+ "device/vendor/blah/static_overlay/foo/res/values/strings.xml",
+ },
+ fooRRODirs: []string{
+ "device/vendor/blah/overlay/foo/res",
+ },
+ barOverlayFiles: []string{
+ "device/vendor/blah/static_overlay/bar/res/values/strings.xml",
+ },
+ barRRODirs: []string{
+ "device/vendor/blah/overlay/bar/res",
+ },
+ },
+}
+
+func TestEnforceRRO(t *testing.T) {
+ resourceOverlays := []string{
+ "device/vendor/blah/overlay",
+ "device/vendor/blah/overlay2",
+ "device/vendor/blah/static_overlay",
+ }
+
+ fs := map[string][]byte{
+ "foo/res/res/values/strings.xml": nil,
+ "bar/res/res/values/strings.xml": nil,
+ "device/vendor/blah/overlay/foo/res/values/strings.xml": nil,
+ "device/vendor/blah/overlay/bar/res/values/strings.xml": nil,
+ "device/vendor/blah/static_overlay/foo/res/values/strings.xml": nil,
+ "device/vendor/blah/static_overlay/bar/res/values/strings.xml": nil,
+ "device/vendor/blah/overlay2/res/values/strings.xml": nil,
+ }
+
+ bp := `
+ android_app {
+ name: "foo",
+ resource_dirs: ["foo/res"],
+ }
+
+ android_app {
+ name: "bar",
+ resource_dirs: ["bar/res"],
+ }
+ `
+
+ for _, testCase := range testEnforceRROTests {
+ t.Run(testCase.name, func(t *testing.T) {
+ config := testConfig(nil)
+ config.ProductVariables.ResourceOverlays = &resourceOverlays
+ if testCase.enforceRROTargets != nil {
+ config.ProductVariables.EnforceRROTargets = &testCase.enforceRROTargets
+ }
+ if testCase.enforceRROExcludedOverlays != nil {
+ config.ProductVariables.EnforceRROExcludedOverlays = &testCase.enforceRROExcludedOverlays
+ }
+
+ ctx := testAppContext(config, bp, fs)
+ run(t, ctx, config)
+
+ getOverlays := func(moduleName string) ([]string, []string) {
+ module := ctx.ModuleForTests(moduleName, "android_common")
+ overlayCompiledPaths := module.Output("aapt2/overlay.list").Inputs.Strings()
+
+ var overlayFiles []string
+ for _, o := range overlayCompiledPaths {
+ overlayFiles = append(overlayFiles, module.Output(o).Inputs.Strings()...)
+ }
+
+ rroDirs := module.Module().(*AndroidApp).rroDirs.Strings()
+
+ return overlayFiles, rroDirs
+ }
+
+ fooOverlayFiles, fooRRODirs := getOverlays("foo")
+ barOverlayFiles, barRRODirs := getOverlays("bar")
+
+ if !reflect.DeepEqual(fooOverlayFiles, testCase.fooOverlayFiles) {
+ t.Errorf("expected foo overlay files:\n %#v\n got:\n %#v",
+ testCase.fooOverlayFiles, fooOverlayFiles)
+ }
+ if !reflect.DeepEqual(fooRRODirs, testCase.fooRRODirs) {
+ t.Errorf("expected foo rroDirs: %#v\n got:\n %#v",
+ testCase.fooRRODirs, fooRRODirs)
+ }
+
+ if !reflect.DeepEqual(barOverlayFiles, testCase.barOverlayFiles) {
+ t.Errorf("expected bar overlay files:\n %#v\n got:\n %#v",
+ testCase.barOverlayFiles, barOverlayFiles)
+ }
+ if !reflect.DeepEqual(barRRODirs, testCase.barRRODirs) {
+ t.Errorf("expected bar rroDirs: %#v\n got:\n %#v",
+ testCase.barRRODirs, barRRODirs)
+ }
+
+ })
+ }
+}
diff --git a/java/builder.go b/java/builder.go
index ed9d82c..10dfe06 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -20,12 +20,13 @@
import (
"path/filepath"
+ "strconv"
"strings"
- "android/soong/android"
-
"github.com/google/blueprint"
- _ "github.com/google/blueprint/bootstrap"
+
+ "android/soong/android"
+ "android/soong/java/config"
)
var (
@@ -39,127 +40,317 @@
// read from directly using @<listfile>)
javac = pctx.AndroidGomaStaticRule("javac",
blueprint.RuleParams{
+ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +
+ `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` +
+ `${config.SoongJavacWrapper} ${config.JavacWrapper}${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` +
+ `$javacFlags $bootClasspath $classpath ` +
+ `-source $javaVersion -target $javaVersion ` +
+ `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list && ` +
+ `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
+ CommandDeps: []string{
+ "${config.JavacCmd}",
+ "${config.SoongZipCmd}",
+ "${config.ExtractSrcJarsCmd}",
+ },
+ CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},
+ Rspfile: "$out.rsp",
+ RspfileContent: "$in",
+ },
+ "javacFlags", "bootClasspath", "classpath", "srcJars", "srcJarDir",
+ "outDir", "annoDir", "javaVersion")
+
+ kotlinc = pctx.AndroidGomaStaticRule("kotlinc",
+ blueprint.RuleParams{
+ // TODO(ccross): kotlinc doesn't support @ file for arguments, which will limit the
+ // maximum number of input files, especially on darwin.
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
- `${JavacWrapper}$javacCmd ` +
- `-encoding UTF-8 $javacFlags $bootClasspath $classpath ` +
- `-extdirs "" -d $outDir @$out.rsp || ( rm -rf "$outDir"; exit 41 ) && ` +
- `find $outDir -name "*.class" > $out`,
+ `${config.KotlincCmd} $classpath $kotlincFlags ` +
+ `-jvm-target $kotlinJvmTarget -d $outDir $in && ` +
+ `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
+ CommandDeps: []string{
+ "${config.KotlincCmd}",
+ "${config.KotlinCompilerJar}",
+ "${config.SoongZipCmd}",
+ },
+ },
+ "kotlincFlags", "classpath", "outDir", "kotlinJvmTarget")
+
+ errorprone = pctx.AndroidStaticRule("errorprone",
+ blueprint.RuleParams{
+ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +
+ `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` +
+ `${config.SoongJavacWrapper} ${config.ErrorProneCmd} ` +
+ `$javacFlags $bootClasspath $classpath ` +
+ `-source $javaVersion -target $javaVersion ` +
+ `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list && ` +
+ `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
+ CommandDeps: []string{
+ "${config.JavaCmd}",
+ "${config.ErrorProneJavacJar}",
+ "${config.ErrorProneJar}",
+ "${config.SoongZipCmd}",
+ "${config.ExtractSrcJarsCmd}",
+ },
+ CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},
+ Rspfile: "$out.rsp",
+ RspfileContent: "$in",
+ },
+ "javacFlags", "bootClasspath", "classpath", "srcJars", "srcJarDir",
+ "outDir", "annoDir", "javaVersion")
+
+ turbine = pctx.AndroidStaticRule("turbine",
+ blueprint.RuleParams{
+ Command: `rm -rf "$outDir" "$srcJarDir" && mkdir -p "$outDir" "$srcJarDir" && ` +
+ `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` +
+ `${config.JavaCmd} -jar ${config.TurbineJar} --output $out.tmp ` +
+ `--temp_dir "$outDir" --sources @$out.rsp @$srcJarDir/list ` +
+ `--javacopts ${config.CommonJdkFlags} ` +
+ `$javacFlags -source $javaVersion -target $javaVersion $bootClasspath $classpath && ` +
+ `${config.Ziptime} $out.tmp && ` +
+ `(if cmp -s $out.tmp $out ; then rm $out.tmp ; else mv $out.tmp $out ; fi )`,
+ CommandDeps: []string{
+ "${config.TurbineJar}",
+ "${config.JavaCmd}",
+ "${config.Ziptime}",
+ "${config.ExtractSrcJarsCmd}",
+ },
Rspfile: "$out.rsp",
RspfileContent: "$in",
+ Restat: true,
},
- "javacCmd", "javacFlags", "bootClasspath", "classpath", "outDir")
+ "javacFlags", "bootClasspath", "classpath", "srcJars", "srcJarDir",
+ "outDir", "javaVersion")
jar = pctx.AndroidStaticRule("jar",
blueprint.RuleParams{
- Command: `$jarCmd -o $out $jarArgs`,
- CommandDeps: []string{"$jarCmd"},
+ Command: `${config.SoongZipCmd} -jar -o $out $jarArgs`,
+ CommandDeps: []string{"${config.SoongZipCmd}"},
},
- "jarCmd", "jarArgs")
+ "jarArgs")
+
+ combineJar = pctx.AndroidStaticRule("combineJar",
+ blueprint.RuleParams{
+ Command: `${config.MergeZipsCmd} -j $jarArgs $out $in`,
+ CommandDeps: []string{"${config.MergeZipsCmd}"},
+ },
+ "jarArgs")
+
+ desugar = pctx.AndroidStaticRule("desugar",
+ blueprint.RuleParams{
+ Command: `rm -rf $dumpDir && mkdir -p $dumpDir && ` +
+ `${config.JavaCmd} ` +
+ `-Djdk.internal.lambda.dumpProxyClasses=$$(cd $dumpDir && pwd) ` +
+ `$javaFlags ` +
+ `-jar ${config.DesugarJar} $classpathFlags $desugarFlags ` +
+ `-i $in -o $out`,
+ CommandDeps: []string{"${config.DesugarJar}", "${config.JavaCmd}"},
+ },
+ "javaFlags", "classpathFlags", "desugarFlags", "dumpDir")
dx = pctx.AndroidStaticRule("dx",
blueprint.RuleParams{
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
- `$dxCmd --dex --output=$outDir $dxFlags $in || ( rm -rf "$outDir"; exit 41 ) && ` +
- `find "$outDir" -name "classes*.dex" > $out`,
- CommandDeps: []string{"$dxCmd"},
+ `${config.DxCmd} --dex --output=$outDir $dxFlags $in && ` +
+ `${config.SoongZipCmd} -o $outDir/classes.dex.jar -C $outDir -D $outDir && ` +
+ `${config.MergeZipsCmd} -D -stripFile "*.class" $out $outDir/classes.dex.jar $in`,
+ CommandDeps: []string{
+ "${config.DxCmd}",
+ "${config.SoongZipCmd}",
+ "${config.MergeZipsCmd}",
+ },
+ },
+ "outDir", "dxFlags")
+
+ d8 = pctx.AndroidStaticRule("d8",
+ blueprint.RuleParams{
+ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
+ `${config.D8Cmd} --output $outDir $dxFlags $in && ` +
+ `${config.SoongZipCmd} -o $outDir/classes.dex.jar -C $outDir -D $outDir && ` +
+ `${config.MergeZipsCmd} -D -stripFile "*.class" $out $outDir/classes.dex.jar $in`,
+ CommandDeps: []string{
+ "${config.DxCmd}",
+ "${config.SoongZipCmd}",
+ "${config.MergeZipsCmd}",
+ },
},
"outDir", "dxFlags")
jarjar = pctx.AndroidStaticRule("jarjar",
blueprint.RuleParams{
- Command: "java -jar $jarjarCmd process $rulesFile $in $out",
- CommandDeps: []string{"$jarjarCmd", "$rulesFile"},
+ Command: "${config.JavaCmd} -jar ${config.JarjarCmd} process $rulesFile $in $out",
+ CommandDeps: []string{"${config.JavaCmd}", "${config.JarjarCmd}", "$rulesFile"},
},
"rulesFile")
-
- extractPrebuilt = pctx.AndroidStaticRule("extractPrebuilt",
- blueprint.RuleParams{
- Command: `rm -rf $outDir && unzip -qo $in -d $outDir && ` +
- `find $outDir -name "*.class" > $classFile && ` +
- `find $outDir -type f -a \! -name "*.class" -a \! -name "MANIFEST.MF" > $resourceFile || ` +
- `(rm -rf $outDir; exit 42)`,
- },
- "outDir", "classFile", "resourceFile")
)
func init() {
- pctx.Import("github.com/google/blueprint/bootstrap")
- pctx.StaticVariable("commonJdkFlags", "-source 1.7 -target 1.7 -Xmaxerrs 9999999")
- pctx.StaticVariable("javacCmd", "javac -J-Xmx1024M $commonJdkFlags")
- pctx.StaticVariable("jarCmd", filepath.Join("${bootstrap.ToolDir}", "soong_zip"))
- pctx.HostBinToolVariable("dxCmd", "dx")
- pctx.HostJavaToolVariable("jarjarCmd", "jarjar.jar")
-
- pctx.VariableFunc("JavacWrapper", func(config interface{}) (string, error) {
- if override := config.(android.Config).Getenv("JAVAC_WRAPPER"); override != "" {
- return override + " ", nil
- }
- return "", nil
- })
+ pctx.Import("android/soong/java/config")
}
type javaBuilderFlags struct {
javacFlags string
dxFlags string
- bootClasspath string
- classpath string
+ bootClasspath classpath
+ classpath classpath
+ systemModules classpath
+ desugarFlags string
aidlFlags string
+ javaVersion string
+
+ kotlincFlags string
+ kotlincClasspath classpath
+
+ protoFlags []string
+ protoOutFlag string
}
-type jarSpec struct {
- fileList, dir android.Path
+func TransformKotlinToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
+ srcFiles, srcJars android.Paths,
+ flags javaBuilderFlags) {
+
+ classDir := android.PathForModuleOut(ctx, "kotlinc", "classes")
+
+ inputs := append(android.Paths(nil), srcFiles...)
+ inputs = append(inputs, srcJars...)
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: kotlinc,
+ Description: "kotlinc",
+ Output: outputFile,
+ Inputs: inputs,
+ Args: map[string]string{
+ "classpath": flags.kotlincClasspath.FormJavaClassPath("-classpath"),
+ "kotlincFlags": flags.kotlincFlags,
+ "outDir": classDir.String(),
+ // http://b/69160377 kotlinc only supports -jvm-target 1.6 and 1.8
+ "kotlinJvmTarget": "1.8",
+ },
+ })
}
-func (j jarSpec) soongJarArgs() string {
- return "-C " + j.dir.String() + " -l " + j.fileList.String()
+func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, shardIdx int,
+ srcFiles, srcJars android.Paths, flags javaBuilderFlags, deps android.Paths) {
+
+ // Compile java sources into .class files
+ desc := "javac"
+ if shardIdx >= 0 {
+ desc += strconv.Itoa(shardIdx)
+ }
+
+ transformJavaToClasses(ctx, outputFile, shardIdx, srcFiles, srcJars, flags, deps, "javac", desc, javac)
}
-func TransformJavaToClasses(ctx android.ModuleContext, srcFiles android.Paths, srcFileLists android.Paths,
- flags javaBuilderFlags, deps android.Paths) jarSpec {
+func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
+ srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
- classDir := android.PathForModuleOut(ctx, "classes")
- classFileList := android.PathForModuleOut(ctx, "classes.list")
+ if config.ErrorProneJar == "" {
+ ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?")
+ }
- javacFlags := flags.javacFlags + android.JoinWithPrefix(srcFileLists.Strings(), "@")
+ transformJavaToClasses(ctx, outputFile, -1, srcFiles, srcJars, flags, nil,
+ "errorprone", "errorprone", errorprone)
+}
- deps = append(deps, srcFileLists...)
+func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath,
+ srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: javac,
- Description: "javac",
- Output: classFileList,
+ var deps android.Paths
+ deps = append(deps, srcJars...)
+ deps = append(deps, flags.bootClasspath...)
+ deps = append(deps, flags.classpath...)
+
+ var bootClasspath string
+ if len(flags.bootClasspath) == 0 && ctx.Device() {
+ // explicitly specify -bootclasspath "" if the bootclasspath is empty to
+ // ensure java does not fall back to the default bootclasspath.
+ bootClasspath = `--bootclasspath ""`
+ } else {
+ bootClasspath = flags.bootClasspath.FormJavaClassPath("--bootclasspath")
+ }
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: turbine,
+ Description: "turbine",
+ Output: outputFile,
Inputs: srcFiles,
Implicits: deps,
Args: map[string]string{
- "javacFlags": javacFlags,
- "bootClasspath": flags.bootClasspath,
- "classpath": flags.classpath,
- "outDir": classDir.String(),
+ "javacFlags": flags.javacFlags,
+ "bootClasspath": bootClasspath,
+ "srcJars": strings.Join(srcJars.Strings(), " "),
+ "srcJarDir": android.PathForModuleOut(ctx, "turbine", "srcjars").String(),
+ "classpath": flags.classpath.FormJavaClassPath("--classpath"),
+ "outDir": android.PathForModuleOut(ctx, "turbine", "classes").String(),
+ "javaVersion": flags.javaVersion,
},
})
-
- return jarSpec{classFileList, classDir}
}
-func TransformClassesToJar(ctx android.ModuleContext, classes []jarSpec,
- manifest android.OptionalPath) android.Path {
+// transformJavaToClasses takes source files and converts them to a jar containing .class files.
+// srcFiles is a list of paths to sources, srcJars is a list of paths to jar files that contain
+// sources. flags contains various command line flags to be passed to the compiler.
+//
+// This method may be used for different compilers, including javac and Error Prone. The rule
+// argument specifies which command line to use and desc sets the description of the rule that will
+// be printed at build time. The stem argument provides the file name of the output jar, and
+// suffix will be appended to various intermediate files and directories to avoid collisions when
+// this function is called twice in the same module directory.
+func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
+ shardIdx int, srcFiles, srcJars android.Paths,
+ flags javaBuilderFlags, deps android.Paths,
+ intermediatesDir, desc string, rule blueprint.Rule) {
- outputFile := android.PathForModuleOut(ctx, "classes-full-debug.jar")
+ deps = append(deps, srcJars...)
- deps := android.Paths{}
- jarArgs := []string{}
-
- for _, j := range classes {
- deps = append(deps, j.fileList)
- jarArgs = append(jarArgs, j.soongJarArgs())
+ var bootClasspath string
+ if flags.javaVersion == "1.9" {
+ deps = append(deps, flags.systemModules...)
+ bootClasspath = flags.systemModules.FormJavaSystemModulesPath("--system=", ctx.Device())
+ } else {
+ deps = append(deps, flags.bootClasspath...)
+ if len(flags.bootClasspath) == 0 && ctx.Device() {
+ // explicitly specify -bootclasspath "" if the bootclasspath is empty to
+ // ensure java does not fall back to the default bootclasspath.
+ bootClasspath = `-bootclasspath ""`
+ } else {
+ bootClasspath = flags.bootClasspath.FormJavaClassPath("-bootclasspath")
+ }
}
- if manifest.Valid() {
- deps = append(deps, manifest.Path())
- jarArgs = append(jarArgs, "-m "+manifest.String())
- }
+ deps = append(deps, flags.classpath...)
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ srcJarDir := "srcjars"
+ outDir := "classes"
+ annoDir := "anno"
+ if shardIdx >= 0 {
+ shardDir := "shard" + strconv.Itoa(shardIdx)
+ srcJarDir = filepath.Join(shardDir, srcJarDir)
+ outDir = filepath.Join(shardDir, outDir)
+ annoDir = filepath.Join(shardDir, annoDir)
+ }
+ ctx.Build(pctx, android.BuildParams{
+ Rule: rule,
+ Description: desc,
+ Output: outputFile,
+ Inputs: srcFiles,
+ Implicits: deps,
+ Args: map[string]string{
+ "javacFlags": flags.javacFlags,
+ "bootClasspath": bootClasspath,
+ "classpath": flags.classpath.FormJavaClassPath("-classpath"),
+ "srcJars": strings.Join(srcJars.Strings(), " "),
+ "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, srcJarDir).String(),
+ "outDir": android.PathForModuleOut(ctx, intermediatesDir, outDir).String(),
+ "annoDir": android.PathForModuleOut(ctx, intermediatesDir, annoDir).String(),
+ "javaVersion": flags.javaVersion,
+ },
+ })
+}
+
+func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath,
+ jarArgs []string, deps android.Paths) {
+
+ ctx.Build(pctx, android.BuildParams{
Rule: jar,
Description: "jar",
Output: outputFile,
@@ -168,19 +359,90 @@
"jarArgs": strings.Join(jarArgs, " "),
},
})
-
- return outputFile
}
-func TransformClassesJarToDex(ctx android.ModuleContext, classesJar android.Path,
- flags javaBuilderFlags) jarSpec {
+func TransformJarsToJar(ctx android.ModuleContext, outputFile android.WritablePath, desc string,
+ jars android.Paths, manifest android.OptionalPath, stripDirs bool, dirsToStrip []string) {
+
+ var deps android.Paths
+
+ var jarArgs []string
+ if manifest.Valid() {
+ jarArgs = append(jarArgs, "-m ", manifest.String())
+ deps = append(deps, manifest.Path())
+ }
+
+ if dirsToStrip != nil {
+ for _, dir := range dirsToStrip {
+ jarArgs = append(jarArgs, "-stripDir ", dir)
+ }
+ }
+
+ if stripDirs {
+ jarArgs = append(jarArgs, "-D")
+ }
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: combineJar,
+ Description: desc,
+ Output: outputFile,
+ Inputs: jars,
+ Implicits: deps,
+ Args: map[string]string{
+ "jarArgs": strings.Join(jarArgs, " "),
+ },
+ })
+}
+
+func TransformDesugar(ctx android.ModuleContext, outputFile android.WritablePath,
+ classesJar android.Path, flags javaBuilderFlags) {
+
+ dumpDir := android.PathForModuleOut(ctx, "desugar", "classes")
+
+ javaFlags := ""
+ if ctx.Config().UseOpenJDK9() {
+ javaFlags = "--add-opens java.base/java.lang.invoke=ALL-UNNAMED"
+ }
+
+ var desugarFlags []string
+ desugarFlags = append(desugarFlags, flags.bootClasspath.FormDesugarClasspath("--bootclasspath_entry")...)
+ desugarFlags = append(desugarFlags, flags.classpath.FormDesugarClasspath("--classpath_entry")...)
+
+ var deps android.Paths
+ deps = append(deps, flags.bootClasspath...)
+ deps = append(deps, flags.classpath...)
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: desugar,
+ Description: "desugar",
+ Output: outputFile,
+ Input: classesJar,
+ Implicits: deps,
+ Args: map[string]string{
+ "dumpDir": dumpDir.String(),
+ "javaFlags": javaFlags,
+ "classpathFlags": strings.Join(desugarFlags, " "),
+ "desugarFlags": flags.desugarFlags,
+ },
+ })
+}
+
+// Converts a classes.jar file to classes*.dex, then combines the dex files with any resources
+// in the classes.jar file into a dex jar.
+func TransformClassesJarToDexJar(ctx android.ModuleContext, outputFile android.WritablePath,
+ classesJar android.Path, flags javaBuilderFlags) {
outDir := android.PathForModuleOut(ctx, "dex")
- outputFile := android.PathForModuleOut(ctx, "dex.filelist")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: dx,
- Description: "dx",
+ rule := dx
+ desc := "dx"
+ if ctx.AConfig().IsEnvTrue("USE_D8_DESUGAR") {
+ rule = d8
+ desc = "d8"
+ }
+ ctx.Build(pctx, android.BuildParams{
+ Rule: rule,
+ Description: desc,
Output: outputFile,
Input: classesJar,
Args: map[string]string{
@@ -188,41 +450,11 @@
"outDir": outDir.String(),
},
})
-
- return jarSpec{outputFile, outDir}
}
-func TransformDexToJavaLib(ctx android.ModuleContext, resources []jarSpec,
- dexJarSpec jarSpec) android.Path {
-
- outputFile := android.PathForModuleOut(ctx, "javalib.jar")
- var deps android.Paths
- var jarArgs []string
-
- for _, j := range resources {
- deps = append(deps, j.fileList)
- jarArgs = append(jarArgs, j.soongJarArgs())
- }
-
- deps = append(deps, dexJarSpec.fileList)
- jarArgs = append(jarArgs, dexJarSpec.soongJarArgs())
-
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: jar,
- Description: "jar",
- Output: outputFile,
- Implicits: deps,
- Args: map[string]string{
- "jarArgs": strings.Join(jarArgs, " "),
- },
- })
-
- return outputFile
-}
-
-func TransformJarJar(ctx android.ModuleContext, classesJar android.Path, rulesFile android.Path) android.Path {
- outputFile := android.PathForModuleOut(ctx, "classes-jarjar.jar")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath,
+ classesJar android.Path, rulesFile android.Path) {
+ ctx.Build(pctx, android.BuildParams{
Rule: jarjar,
Description: "jarjar",
Output: outputFile,
@@ -232,28 +464,64 @@
"rulesFile": rulesFile.String(),
},
})
-
- return outputFile
}
-func TransformPrebuiltJarToClasses(ctx android.ModuleContext,
- prebuilt android.Path) (classJarSpec, resourceJarSpec jarSpec) {
+type classpath []android.Path
- classDir := android.PathForModuleOut(ctx, "extracted/classes")
- classFileList := android.PathForModuleOut(ctx, "extracted/classes.list")
- resourceFileList := android.PathForModuleOut(ctx, "extracted/resources.list")
+func (x *classpath) FormJavaClassPath(optName string) string {
+ if len(*x) > 0 {
+ return optName + " " + strings.Join(x.Strings(), ":")
+ } else {
+ return ""
+ }
+}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: extractPrebuilt,
- Description: "extract classes",
- Outputs: android.WritablePaths{classFileList, resourceFileList},
- Input: prebuilt,
- Args: map[string]string{
- "outDir": classDir.String(),
- "classFile": classFileList.String(),
- "resourceFile": resourceFileList.String(),
- },
- })
+// Returns a --system argument in the form javac expects with -source 1.9. If forceEmpty is true,
+// returns --system=none if the list is empty to ensure javac does not fall back to the default
+// system modules.
+func (x *classpath) FormJavaSystemModulesPath(optName string, forceEmpty bool) string {
+ if len(*x) > 1 {
+ panic("more than one system module")
+ } else if len(*x) == 1 {
+ return optName + strings.TrimSuffix((*x)[0].String(), "lib/modules")
+ } else if forceEmpty {
+ return optName + "none"
+ } else {
+ return ""
+ }
+}
- return jarSpec{classFileList, classDir}, jarSpec{resourceFileList, classDir}
+func (x *classpath) FormDesugarClasspath(optName string) []string {
+ if x == nil || *x == nil {
+ return nil
+ }
+ flags := make([]string, len(*x))
+ for i, v := range *x {
+ flags[i] = optName + " " + v.String()
+ }
+
+ return flags
+}
+
+// Append an android.Paths to the end of the classpath list
+func (x *classpath) AddPaths(paths android.Paths) {
+ for _, path := range paths {
+ *x = append(*x, path)
+ }
+}
+
+// Convert a classpath to an android.Paths
+func (x *classpath) Paths() android.Paths {
+ return append(android.Paths(nil), (*x)...)
+}
+
+func (x *classpath) Strings() []string {
+ if x == nil {
+ return nil
+ }
+ ret := make([]string, len(*x))
+ for i, path := range *x {
+ ret[i] = path.String()
+ }
+ return ret
}
diff --git a/java/config/config.go b/java/config/config.go
index 848d09d..c43f9a3 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -14,10 +14,145 @@
package config
-import "android/soong/android"
+import (
+ "path/filepath"
+ "runtime"
+ "strings"
-var (
- DefaultLibraries = []string{"core-oj", "core-libart", "ext", "framework", "okhttp"}
+ _ "github.com/google/blueprint/bootstrap"
+
+ "android/soong/android"
)
-var pctx = android.NewPackageContext("android/soong/java/config")
+var (
+ pctx = android.NewPackageContext("android/soong/java/config")
+
+ DefaultBootclasspathLibraries = []string{"core-oj", "core-libart"}
+ DefaultSystemModules = "core-system-modules"
+ DefaultLibraries = []string{"ext", "framework", "okhttp"}
+
+ DefaultJacocoExcludeFilter = []string{"org.junit.*", "org.jacoco.*", "org.mockito.*"}
+
+ InstrumentFrameworkModules = []string{
+ "framework",
+ "telephony-common",
+ "services",
+ "android.car",
+ "android.car7",
+ }
+)
+
+func init() {
+ pctx.Import("github.com/google/blueprint/bootstrap")
+
+ pctx.StaticVariable("JavacHeapSize", "2048M")
+ pctx.StaticVariable("JavacHeapFlags", "-J-Xmx${JavacHeapSize}")
+
+ pctx.StaticVariable("CommonJdkFlags", strings.Join([]string{
+ `-Xmaxerrs 9999999`,
+ `-encoding UTF-8`,
+ `-sourcepath ""`,
+ `-g`,
+ // Turbine leaves out bridges which can cause javac to unnecessarily insert them into
+ // subclasses (b/65645120). Setting this flag causes our custom javac to assume that
+ // the missing bridges will exist at runtime and not recreate them in subclasses.
+ // If a different javac is used the flag will be ignored and extra bridges will be inserted.
+ // The flag is implemented by https://android-review.googlesource.com/c/486427
+ `-XDskipDuplicateBridges=true`,
+
+ // b/65004097: prevent using java.lang.invoke.StringConcatFactory when using -target 1.9
+ `-XDstringConcat=inline`,
+ }, " "))
+
+ pctx.VariableConfigMethod("hostPrebuiltTag", android.Config.PrebuiltOS)
+
+ pctx.VariableFunc("JavaHome", func(config android.Config) (string, error) {
+ // This is set up and guaranteed by soong_ui
+ return config.Getenv("ANDROID_JAVA_HOME"), nil
+ })
+
+ pctx.SourcePathVariable("JavaToolchain", "${JavaHome}/bin")
+ pctx.SourcePathVariableWithEnvOverride("JavacCmd",
+ "${JavaToolchain}/javac", "ALTERNATE_JAVAC")
+ pctx.SourcePathVariable("JavaCmd", "${JavaToolchain}/java")
+ pctx.SourcePathVariable("JarCmd", "${JavaToolchain}/jar")
+ pctx.SourcePathVariable("JavadocCmd", "${JavaToolchain}/javadoc")
+ pctx.SourcePathVariable("JlinkCmd", "${JavaToolchain}/jlink")
+ pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod")
+ pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar")
+ pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime")
+
+ pctx.SourcePathVariable("ExtractSrcJarsCmd", "build/soong/scripts/extract-srcjars.sh")
+ pctx.SourcePathVariable("JarArgsCmd", "build/soong/scripts/jar-args.sh")
+ pctx.HostBinToolVariable("SoongZipCmd", "soong_zip")
+ pctx.HostBinToolVariable("MergeZipsCmd", "merge_zips")
+ pctx.HostBinToolVariable("Zip2ZipCmd", "zip2zip")
+ pctx.VariableFunc("DxCmd", func(config android.Config) (string, error) {
+ if config.IsEnvFalse("USE_D8") {
+ if config.UnbundledBuild() || config.IsPdkBuild() {
+ return "prebuilts/build-tools/common/bin/dx", nil
+ } else {
+ path, err := pctx.HostBinToolPath(config, "dx")
+ if err != nil {
+ return "", err
+ }
+ return path.String(), nil
+ }
+ } else {
+ path, err := pctx.HostBinToolPath(config, "d8-compat-dx")
+ if err != nil {
+ return "", err
+ }
+ return path.String(), nil
+ }
+ })
+ pctx.VariableFunc("D8Cmd", func(config android.Config) (string, error) {
+ path, err := pctx.HostBinToolPath(config, "d8")
+ if err != nil {
+ return "", err
+ }
+ return path.String(), nil
+ })
+ pctx.VariableFunc("TurbineJar", func(config android.Config) (string, error) {
+ turbine := "turbine.jar"
+ if config.UnbundledBuild() {
+ return "prebuilts/build-tools/common/framework/" + turbine, nil
+ } else {
+ path, err := pctx.HostJavaToolPath(config, turbine)
+ if err != nil {
+ return "", err
+ }
+ return path.String(), nil
+ }
+ })
+
+ pctx.HostJavaToolVariable("JarjarCmd", "jarjar.jar")
+ pctx.HostJavaToolVariable("DesugarJar", "desugar.jar")
+
+ pctx.HostBinToolVariable("SoongJavacWrapper", "soong_javac_wrapper")
+
+ pctx.VariableFunc("JavacWrapper", func(config android.Config) (string, error) {
+ if override := config.Getenv("JAVAC_WRAPPER"); override != "" {
+ return override + " ", nil
+ }
+ return "", nil
+ })
+
+ pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar")
+
+ hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) {
+ pctx.VariableFunc(name, func(config android.Config) (string, error) {
+ if config.UnbundledBuild() || config.IsPdkBuild() {
+ return filepath.Join(prebuiltDir, runtime.GOOS, "bin", tool), nil
+ } else {
+ if path, err := pctx.HostBinToolPath(config, tool); err != nil {
+ return "", err
+ } else {
+ return path.String(), nil
+ }
+ }
+ })
+ }
+
+ hostBinToolVariableWithPrebuilt("Aapt2Cmd", "prebuilts/sdk/tools", "aapt2")
+}
diff --git a/java/config/error_prone.go b/java/config/error_prone.go
new file mode 100644
index 0000000..862217f
--- /dev/null
+++ b/java/config/error_prone.go
@@ -0,0 +1,47 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import "android/soong/android"
+
+var (
+ // These will be filled out by external/error_prone/soong/error_prone.go if it is available
+ ErrorProneJavacJar string
+ ErrorProneJar string
+ ErrorProneClasspath string
+ ErrorProneChecksError string
+ ErrorProneFlags string
+)
+
+// Wrapper that grabs value of val late so it can be initialized by a later module's init function
+func errorProneVar(name string, val *string) {
+ pctx.VariableFunc(name, func(config android.Config) (string, error) {
+ return *val, nil
+ })
+}
+
+func init() {
+ errorProneVar("ErrorProneJar", &ErrorProneJar)
+ errorProneVar("ErrorProneJavacJar", &ErrorProneJavacJar)
+ errorProneVar("ErrorProneClasspath", &ErrorProneClasspath)
+ errorProneVar("ErrorProneChecksError", &ErrorProneChecksError)
+ errorProneVar("ErrorProneFlags", &ErrorProneFlags)
+
+ pctx.StaticVariable("ErrorProneCmd",
+ "${JavaCmd} -Xmx${JavacHeapSize} -Xbootclasspath/p:${ErrorProneJavacJar} "+
+ "-cp ${ErrorProneJar}:${ErrorProneClasspath} "+
+ "${ErrorProneFlags} ${CommonJdkFlags} ${ErrorProneChecksError}")
+
+}
diff --git a/java/config/kotlin.go b/java/config/kotlin.go
new file mode 100644
index 0000000..35f9e9d
--- /dev/null
+++ b/java/config/kotlin.go
@@ -0,0 +1,25 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+var (
+ KotlinStdlibJar = "external/kotlinc/lib/kotlin-stdlib.jar"
+)
+
+func init() {
+ pctx.SourcePathVariable("KotlincCmd", "external/kotlinc/bin/kotlinc")
+ pctx.SourcePathVariable("KotlinCompilerJar", "external/kotlinc/lib/kotlin-compiler.jar")
+ pctx.SourcePathVariable("KotlinStdlibJar", KotlinStdlibJar)
+}
diff --git a/java/config/makevars.go b/java/config/makevars.go
index 6702454..5c8589e 100644
--- a/java/config/makevars.go
+++ b/java/config/makevars.go
@@ -26,4 +26,52 @@
func makeVarsProvider(ctx android.MakeVarsContext) {
ctx.Strict("TARGET_DEFAULT_JAVA_LIBRARIES", strings.Join(DefaultLibraries, " "))
+ ctx.Strict("TARGET_DEFAULT_BOOTCLASSPATH_LIBRARIES", strings.Join(DefaultBootclasspathLibraries, " "))
+ ctx.Strict("DEFAULT_SYSTEM_MODULES", DefaultSystemModules)
+
+ if ctx.Config().TargetOpenJDK9() {
+ ctx.Strict("DEFAULT_JAVA_LANGUAGE_VERSION", "1.9")
+ } else {
+ ctx.Strict("DEFAULT_JAVA_LANGUAGE_VERSION", "1.8")
+ }
+
+ ctx.Strict("ANDROID_JAVA_HOME", "${JavaHome}")
+ ctx.Strict("ANDROID_JAVA8_HOME", "prebuilts/jdk/jdk8/${hostPrebuiltTag}")
+ ctx.Strict("ANDROID_JAVA9_HOME", "prebuilts/jdk/jdk9/${hostPrebuiltTag}")
+ ctx.Strict("ANDROID_JAVA_TOOLCHAIN", "${JavaToolchain}")
+ ctx.Strict("JAVA", "${JavaCmd}")
+ ctx.Strict("JAVAC", "${JavacCmd}")
+ ctx.Strict("JAR", "${JarCmd}")
+ ctx.Strict("JAR_ARGS", "${JarArgsCmd}")
+ ctx.Strict("JAVADOC", "${JavadocCmd}")
+ ctx.Strict("COMMON_JDK_FLAGS", "${CommonJdkFlags}")
+
+ if ctx.Config().IsEnvTrue("USE_D8_DESUGAR") {
+ ctx.Strict("DX", "${D8Cmd}")
+ ctx.Strict("DX_COMMAND", "${D8Cmd} -JXms16M -JXmx2048M")
+ } else {
+ ctx.Strict("DX", "${DxCmd}")
+ ctx.Strict("DX_COMMAND", "${DxCmd} -JXms16M -JXmx2048M")
+ }
+
+ ctx.Strict("TURBINE", "${TurbineJar}")
+
+ if ctx.Config().IsEnvTrue("RUN_ERROR_PRONE") {
+ ctx.Strict("TARGET_JAVAC", "${ErrorProneCmd}")
+ ctx.Strict("HOST_JAVAC", "${ErrorProneCmd}")
+ } else {
+ ctx.Strict("TARGET_JAVAC", "${JavacCmd} ${CommonJdkFlags}")
+ ctx.Strict("HOST_JAVAC", "${JavacCmd} ${CommonJdkFlags}")
+ }
+
+ if ctx.Config().UseOpenJDK9() {
+ ctx.Strict("JLINK", "${JlinkCmd}")
+ ctx.Strict("JMOD", "${JmodCmd}")
+ }
+
+ ctx.Strict("SOONG_JAVAC_WRAPPER", "${SoongJavacWrapper}")
+ ctx.Strict("EXTRACT_SRCJARS", "${ExtractSrcJarsCmd}")
+
+ ctx.Strict("JACOCO_CLI_JAR", "${JacocoCLIJar}")
+ ctx.Strict("DEFAULT_JACOCO_EXCLUDE_FILTER", strings.Join(DefaultJacocoExcludeFilter, ","))
}
diff --git a/java/gen.go b/java/gen.go
index e473859..7a0dcac 100644
--- a/java/gen.go
+++ b/java/gen.go
@@ -28,8 +28,6 @@
pctx.HostBinToolVariable("aidlCmd", "aidl")
pctx.SourcePathVariable("logtagsCmd", "build/tools/java-event-log-tags.py")
pctx.SourcePathVariable("mergeLogtagsCmd", "build/tools/merge-event-log-tags.py")
-
- pctx.IntermediatesPathVariable("allLogtagsFile", "all-event-log-tags.txt")
}
var (
@@ -42,7 +40,7 @@
logtags = pctx.AndroidStaticRule("logtags",
blueprint.RuleParams{
- Command: "$logtagsCmd -o $out $in $allLogtagsFile",
+ Command: "$logtagsCmd -o $out $in",
CommandDeps: []string{"$logtagsCmd"},
})
@@ -57,7 +55,7 @@
javaFile := android.GenPathWithExt(ctx, "aidl", aidlFile, "java")
depFile := javaFile.String() + ".d"
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: aidl,
Description: "aidl " + aidlFile.Rel(),
Output: javaFile,
@@ -74,7 +72,7 @@
func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Path {
javaFile := android.GenPathWithExt(ctx, "logtags", logtagsFile, "java")
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: logtags,
Description: "logtags " + logtagsFile.Rel(),
Output: javaFile,
@@ -87,22 +85,37 @@
func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
flags javaBuilderFlags) android.Paths {
- for i, srcFile := range srcFiles {
+ var protoFiles android.Paths
+ outSrcFiles := make(android.Paths, 0, len(srcFiles))
+
+ for _, srcFile := range srcFiles {
switch srcFile.Ext() {
case ".aidl":
javaFile := genAidl(ctx, srcFile, flags.aidlFlags)
- srcFiles[i] = javaFile
+ outSrcFiles = append(outSrcFiles, javaFile)
case ".logtags":
j.logtagsSrcs = append(j.logtagsSrcs, srcFile)
javaFile := genLogtags(ctx, srcFile)
- srcFiles[i] = javaFile
+ outSrcFiles = append(outSrcFiles, javaFile)
+ case ".proto":
+ protoFiles = append(protoFiles, srcFile)
+ default:
+ outSrcFiles = append(outSrcFiles, srcFile)
}
}
- return srcFiles
+ if len(protoFiles) > 0 {
+ protoSrcJar := android.PathForModuleGen(ctx, "proto.srcjar")
+ genProto(ctx, protoSrcJar, protoFiles,
+ flags.protoFlags, flags.protoOutFlag, "")
+
+ outSrcFiles = append(outSrcFiles, protoSrcJar)
+ }
+
+ return outSrcFiles
}
-func LogtagsSingleton() blueprint.Singleton {
+func LogtagsSingleton() android.Singleton {
return &logtagsSingleton{}
}
@@ -112,18 +125,18 @@
type logtagsSingleton struct{}
-func (l *logtagsSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
+func (l *logtagsSingleton) GenerateBuildActions(ctx android.SingletonContext) {
var allLogtags android.Paths
- ctx.VisitAllModules(func(module blueprint.Module) {
+ ctx.VisitAllModules(func(module android.Module) {
if logtags, ok := module.(logtagsProducer); ok {
allLogtags = append(allLogtags, logtags.logtags()...)
}
})
- ctx.Build(pctx, blueprint.BuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: mergeLogtags,
Description: "merge logtags",
- Outputs: []string{"$allLogtagsFile"},
- Inputs: allLogtags.Strings(),
+ Output: android.PathForIntermediates(ctx, "all-event-log-tags.txt"),
+ Inputs: allLogtags,
})
}
diff --git a/java/genrule.go b/java/genrule.go
new file mode 100644
index 0000000..80b7030
--- /dev/null
+++ b/java/genrule.go
@@ -0,0 +1,35 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+ "android/soong/android"
+ "android/soong/genrule"
+)
+
+func init() {
+ android.RegisterModuleType("java_genrule", genRuleFactory)
+}
+
+// java_genrule is a genrule that can depend on other java_* objects.
+// The cmd may be run multiple times, once for each of the different host/device
+// variations.
+func genRuleFactory() android.Module {
+ module := genrule.NewGenRule()
+
+ android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon)
+
+ return module
+}
diff --git a/java/jacoco.go b/java/jacoco.go
new file mode 100644
index 0000000..b26b046
--- /dev/null
+++ b/java/jacoco.go
@@ -0,0 +1,108 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+// Rules for instrumenting classes using jacoco
+
+import (
+ "strings"
+
+ "github.com/google/blueprint"
+
+ "android/soong/android"
+)
+
+var (
+ jacoco = pctx.AndroidStaticRule("jacoco", blueprint.RuleParams{
+ Command: `${config.Zip2ZipCmd} -i $in -o $strippedJar $stripSpec && ` +
+ `${config.JavaCmd} -jar ${config.JacocoCLIJar} instrument -quiet -dest $instrumentedJar $strippedJar && ` +
+ `${config.Ziptime} $instrumentedJar && ` +
+ `${config.MergeZipsCmd} --ignore-duplicates -j $out $instrumentedJar $in`,
+ CommandDeps: []string{
+ "${config.Zip2ZipCmd}",
+ "${config.JavaCmd}",
+ "${config.JacocoCLIJar}",
+ "${config.Ziptime}",
+ "${config.MergeZipsCmd}",
+ },
+ },
+ "strippedJar", "stripSpec", "instrumentedJar")
+)
+
+func jacocoInstrumentJar(ctx android.ModuleContext, outputJar, strippedJar android.WritablePath,
+ inputJar android.Path, stripSpec string) {
+ instrumentedJar := android.PathForModuleOut(ctx, "jacoco/instrumented.jar")
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: jacoco,
+ Description: "jacoco",
+ Output: outputJar,
+ ImplicitOutput: strippedJar,
+ Input: inputJar,
+ Args: map[string]string{
+ "strippedJar": strippedJar.String(),
+ "stripSpec": stripSpec,
+ "instrumentedJar": instrumentedJar.String(),
+ },
+ })
+}
+
+func (j *Module) jacocoStripSpecs(ctx android.ModuleContext) string {
+ includes := jacocoFiltersToSpecs(ctx,
+ j.properties.Jacoco.Include_filter, "jacoco.include_filter")
+ excludes := jacocoFiltersToSpecs(ctx,
+ j.properties.Jacoco.Exclude_filter, "jacoco.exclude_filter")
+
+ specs := ""
+ if len(excludes) > 0 {
+ specs += android.JoinWithPrefix(excludes, "-x") + " "
+ }
+
+ if len(includes) > 0 {
+ specs += strings.Join(includes, " ")
+ } else {
+ specs += "**/*.class"
+ }
+
+ return specs
+}
+
+func jacocoFiltersToSpecs(ctx android.ModuleContext, filters []string, property string) []string {
+ specs := make([]string, len(filters))
+ for i, f := range filters {
+ specs[i] = jacocoFilterToSpec(ctx, f, property)
+ }
+ return specs
+}
+
+func jacocoFilterToSpec(ctx android.ModuleContext, filter string, property string) string {
+ wildcard := strings.HasSuffix(filter, "*")
+ filter = strings.TrimSuffix(filter, "*")
+ recursiveWildcard := wildcard && (strings.HasSuffix(filter, ".") || filter == "")
+
+ if strings.ContainsRune(filter, '*') {
+ ctx.PropertyErrorf(property, "'*' is only supported as the last character in a filter")
+ }
+
+ spec := strings.Replace(filter, ".", "/", -1)
+
+ if recursiveWildcard {
+ spec += "**/*.class"
+ } else if wildcard {
+ spec += "*.class"
+ }
+
+ return spec
+}
diff --git a/java/java.go b/java/java.go
index 1ef1c26..d9075b1 100644
--- a/java/java.go
+++ b/java/java.go
@@ -20,39 +20,36 @@
import (
"fmt"
+ "path/filepath"
+ "strconv"
"strings"
"github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
"android/soong/android"
- "android/soong/genrule"
"android/soong/java/config"
)
func init() {
android.RegisterModuleType("java_defaults", defaultsFactory)
- android.RegisterModuleType("java_library", LibraryFactory)
- android.RegisterModuleType("java_library_static", LibraryFactory)
+ android.RegisterModuleType("java_library", LibraryFactory(true))
+ android.RegisterModuleType("java_library_static", LibraryFactory(false))
android.RegisterModuleType("java_library_host", LibraryHostFactory)
android.RegisterModuleType("java_binary", BinaryFactory)
android.RegisterModuleType("java_binary_host", BinaryHostFactory)
- android.RegisterModuleType("java_prebuilt_library", PrebuiltFactory)
- android.RegisterModuleType("android_prebuilt_sdk", SdkPrebuiltFactory)
- android.RegisterModuleType("android_app", AndroidAppFactory)
+ android.RegisterModuleType("java_import", ImportFactory)
+ android.RegisterModuleType("java_import_host", ImportFactoryHost)
android.RegisterSingletonType("logtags", LogtagsSingleton)
}
// TODO:
// Autogenerated files:
-// Proto
// Renderscript
// Post-jar passes:
// Proguard
-// Jacoco
-// Jarjar
-// Dex
// Rmtypedefs
// DroidDoc
// Findbugs
@@ -67,14 +64,24 @@
Exclude_srcs []string `android:"arch_variant"`
// list of directories containing Java resources
- Resource_dirs []string `android:"arch_variant"`
+ Java_resource_dirs []string `android:"arch_variant"`
- // list of directories that should be excluded from resource_dirs
- Exclude_resource_dirs []string `android:"arch_variant"`
+ // list of directories that should be excluded from java_resource_dirs
+ Exclude_java_resource_dirs []string `android:"arch_variant"`
- // don't build against the default libraries (legacy-test, core-junit,
+ // list of files to use as Java resources
+ Java_resources []string `android:"arch_variant"`
+
+ // list of files that should be excluded from java_resources
+ Exclude_java_resources []string `android:"arch_variant"`
+
+ // don't build against the default libraries (bootclasspath, legacy-test, core-junit,
// ext, and framework for device targets)
- No_standard_libraries bool
+ No_standard_libs *bool
+
+ // don't build against the framework libraries (legacy-test, core-junit,
+ // ext, and framework for device targets)
+ No_framework_libs *bool
// list of module-specific flags that will be used for javac compiles
Javacflags []string `android:"arch_variant"`
@@ -89,7 +96,53 @@
Manifest *string
// if not blank, run jarjar using the specified rules file
- Jarjar_rules *string
+ Jarjar_rules *string `android:"arch_variant"`
+
+ // If not blank, set the java version passed to javac as -source and -target
+ Java_version *string
+
+ // If set to false, don't allow this module to be installed. Defaults to true.
+ Installable *bool
+
+ // If set to true, include sources used to compile the module in to the final jar
+ Include_srcs *bool
+
+ // List of modules to use as annotation processors
+ Annotation_processors []string
+
+ // List of classes to pass to javac to use as annotation processors
+ Annotation_processor_classes []string
+
+ // The number of Java source entries each Javac instance can process
+ Javac_shard_size *int64
+
+ Openjdk9 struct {
+ // List of source files that should only be used when passing -source 1.9
+ Srcs []string
+
+ // List of javac flags that should only be used when passing -source 1.9
+ Javacflags []string
+ }
+
+ Jacoco struct {
+ // List of classes to include for instrumentation with jacoco to collect coverage
+ // information at runtime when building with coverage enabled. If unset defaults to all
+ // classes.
+ // Supports '*' as the last character of an entry in the list as a wildcard match.
+ // If preceded by '.' it matches all classes in the package and subpackages, otherwise
+ // it matches classes in the package that have the class name as a prefix.
+ Include_filter []string
+
+ // List of classes to exclude from instrumentation with jacoco to collect coverage
+ // information at runtime when building with coverage enabled. Overrides classes selected
+ // by the include_filter property.
+ // Supports '*' as the last character of an entry in the list as a wildcard match.
+ // If preceded by '.' it matches all classes in the package and subpackages, otherwise
+ // it matches classes in the package that have the class name as a prefix.
+ Exclude_filter []string
+ }
+
+ Instrument bool `blueprint:"mutated"`
}
type CompilerDeviceProperties struct {
@@ -97,18 +150,44 @@
Dxflags []string `android:"arch_variant"`
// if not blank, set to the version of the sdk to compile against
- Sdk_version string
+ Sdk_version *string
- // Set for device java libraries, and for host versions of device java libraries
- // built for testing
- Dex bool `blueprint:"mutated"`
+ Aidl struct {
+ // Top level directories to pass to aidl tool
+ Include_dirs []string
- // directories to pass to aidl tool
- Aidl_includes []string
+ // Directories rooted at the Android.bp file to pass to aidl tool
+ Local_include_dirs []string
- // directories that should be added as include directories
- // for any aidl sources of modules that depend on this module
- Export_aidl_include_dirs []string
+ // directories that should be added as include directories for any aidl sources of modules
+ // that depend on this module, as well as to aidl for this module.
+ Export_include_dirs []string
+ }
+
+ // If true, export a copy of the module as a -hostdex module for host testing.
+ Hostdex *bool
+
+ Dex_preopt struct {
+ // If false, prevent dexpreopting and stripping the dex file from the final jar. Defaults to
+ // true.
+ Enabled *bool
+
+ // If true, generate an app image (.art file) for this module.
+ App_image *bool
+
+ // If true, use a checked-in profile to guide optimization. Defaults to false unless
+ // a matching profile is set or a profile is found in PRODUCT_DEX_PREOPT_PROFILE_DIR
+ // that matches the name of this module, in which case it is defaulted to true.
+ Profile_guided *bool
+
+ // If set, provides the path to profile relative to the Android.bp file. If not set,
+ // defaults to searching for a file that matches the name of this module in the default
+ // profile location set by PRODUCT_DEX_PREOPT_PROFILE_DIR, or empty if not found.
+ Profile *string
+ }
+
+ // When targeting 1.9, override the modules to use with --system
+ System_modules *string
}
// Module contains the properties and members used by all java module types
@@ -117,36 +196,45 @@
android.DefaultableModuleBase
properties CompilerProperties
+ protoProperties android.ProtoProperties
deviceProperties CompilerDeviceProperties
- // output file suitable for inserting into the classpath of another compile
- classpathFile android.Path
+ // header jar file suitable for inserting into the bootclasspath/classpath of another compile
+ headerJarFile android.Path
+
+ // full implementation jar file suitable for static dependency of another module compile
+ implementationJarFile android.Path
+
+ // output file containing classes.dex
+ dexJarFile android.Path
+
+ // output file containing uninstrumented classes that will be instrumented by jacoco
+ jacocoReportClassesFile android.Path
// output file suitable for installing or running
outputFile android.Path
- // jarSpecs suitable for inserting classes from a static library into another jar
- classJarSpecs []jarSpec
-
- // jarSpecs suitable for inserting resources from a static library into another jar
- resourceJarSpecs []jarSpec
-
exportAidlIncludeDirs android.Paths
logtagsSrcs android.Paths
- // filelists of extra source files that should be included in the javac command line,
- // for example R.java generated by aapt for android apps
- ExtraSrcLists android.Paths
-
// installed file for binary dependency
installFile android.Path
+
+ // list of .java files and srcjars that was passed to javac
+ compiledJavaSrcs android.Paths
+ compiledSrcJars android.Paths
}
+func (j *Module) Srcs() android.Paths {
+ return android.Paths{j.implementationJarFile}
+}
+
+var _ android.SourceFileProducer = (*Module)(nil)
+
type Dependency interface {
- ClasspathFile() android.Path
- ClassJarSpecs() []jarSpec
- ResourceJarSpecs() []jarSpec
+ HeaderJars() android.Paths
+ ImplementationJars() android.Paths
AidlIncludeDirs() android.Paths
}
@@ -164,43 +252,191 @@
staticLibTag = dependencyTag{name: "staticlib"}
libTag = dependencyTag{name: "javalib"}
bootClasspathTag = dependencyTag{name: "bootclasspath"}
+ systemModulesTag = dependencyTag{name: "system modules"}
frameworkResTag = dependencyTag{name: "framework-res"}
- sdkDependencyTag = dependencyTag{name: "sdk"}
+ kotlinStdlibTag = dependencyTag{name: "kotlin-stdlib"}
)
-func (j *Module) deps(ctx android.BottomUpMutatorContext) {
- if !j.properties.No_standard_libraries {
- if ctx.Device() {
- switch j.deviceProperties.Sdk_version {
- case "":
- ctx.AddDependency(ctx.Module(), bootClasspathTag, "core-libart")
- case "current":
- // TODO: !TARGET_BUILD_APPS
- // TODO: export preprocessed framework.aidl from android_stubs_current
- ctx.AddDependency(ctx.Module(), bootClasspathTag, "android_stubs_current")
- case "system_current":
- ctx.AddDependency(ctx.Module(), bootClasspathTag, "android_system_stubs_current")
- default:
- ctx.AddDependency(ctx.Module(), sdkDependencyTag, "sdk_v"+j.deviceProperties.Sdk_version)
- }
+type sdkDep struct {
+ useModule, useFiles, useDefaultLibs, invalidVersion bool
+
+ module string
+ systemModules string
+
+ jar android.Path
+ aidl android.Path
+}
+
+func sdkStringToNumber(ctx android.BaseContext, v string) int {
+ switch v {
+ case "", "current", "system_current", "test_current":
+ return 10000
+ default:
+ if i, err := strconv.Atoi(android.GetNumericSdkVersion(v)); err != nil {
+ ctx.PropertyErrorf("sdk_version", "invalid sdk version")
+ return -1
} else {
- if j.deviceProperties.Dex {
- ctx.AddDependency(ctx.Module(), bootClasspathTag, "core-libart")
+ return i
+ }
+ }
+}
+
+func decodeSdkDep(ctx android.BaseContext, v string) sdkDep {
+ i := sdkStringToNumber(ctx, v)
+ if i == -1 {
+ // Invalid sdk version, error handled by sdkStringToNumber.
+ return sdkDep{}
+ }
+
+ toFile := func(v string) sdkDep {
+ dir := filepath.Join("prebuilts/sdk", v)
+ jar := filepath.Join(dir, "android.jar")
+ aidl := filepath.Join(dir, "framework.aidl")
+ jarPath := android.ExistentPathForSource(ctx, "sdkdir", jar)
+ aidlPath := android.ExistentPathForSource(ctx, "sdkdir", aidl)
+
+ if (!jarPath.Valid() || !aidlPath.Valid()) && ctx.Config().AllowMissingDependencies() {
+ if strings.Contains(v, "system_") {
+ return sdkDep{
+ invalidVersion: true,
+ module: "vsdk_v" + strings.Replace(v, "system_", "", 1),
+ }
+ }
+ return sdkDep{
+ invalidVersion: true,
+ module: "sdk_v" + v,
}
}
- if ctx.Device() && j.deviceProperties.Sdk_version == "" {
- ctx.AddDependency(ctx.Module(), libTag, config.DefaultLibraries...)
+ if !jarPath.Valid() {
+ ctx.PropertyErrorf("sdk_version", "invalid sdk version %q, %q does not exist", v, jar)
+ return sdkDep{}
+ }
+
+ if !aidlPath.Valid() {
+ ctx.PropertyErrorf("sdk_version", "invalid sdk version %q, %q does not exist", v, aidl)
+ return sdkDep{}
+ }
+
+ return sdkDep{
+ useFiles: true,
+ jar: jarPath.Path(),
+ aidl: aidlPath.Path(),
}
}
+
+ //toModule := func(m string) sdkDep {
+ // return sdkDep{
+ // useModule: true,
+ // module: m,
+ // systemModules: m + "_system_modules",
+ // }
+ //}
+
+ if ctx.Config().UnbundledBuild() && v != "" {
+ return toFile(v)
+ }
+
+ switch v {
+ case "":
+ return sdkDep{
+ useDefaultLibs: true,
+ }
+ // TODO(ccross): re-enable these once we generate stubs, until then
+ // use the stubs in prebuilts/sdk/*current
+ //case "current":
+ // return toModule("android_stubs_current")
+ //case "system_current":
+ // return toModule("android_system_stubs_current")
+ //case "test_current":
+ // return toModule("android_test_stubs_current")
+ default:
+ return toFile(v)
+ }
+}
+
+func (j *Module) deps(ctx android.BottomUpMutatorContext) {
+ if ctx.Device() {
+ if !proptools.Bool(j.properties.No_standard_libs) {
+ sdkDep := decodeSdkDep(ctx, String(j.deviceProperties.Sdk_version))
+ if sdkDep.useDefaultLibs {
+ ctx.AddDependency(ctx.Module(), bootClasspathTag, config.DefaultBootclasspathLibraries...)
+ if ctx.Config().TargetOpenJDK9() {
+ ctx.AddDependency(ctx.Module(), systemModulesTag, config.DefaultSystemModules)
+ }
+ if !proptools.Bool(j.properties.No_framework_libs) {
+ ctx.AddDependency(ctx.Module(), libTag, config.DefaultLibraries...)
+ }
+ } else if sdkDep.useModule {
+ if ctx.Config().TargetOpenJDK9() {
+ ctx.AddDependency(ctx.Module(), systemModulesTag, sdkDep.systemModules)
+ }
+ ctx.AddDependency(ctx.Module(), bootClasspathTag, sdkDep.module)
+ }
+ } else if j.deviceProperties.System_modules == nil {
+ ctx.PropertyErrorf("no_standard_libs",
+ "system_modules is required to be set when no_standard_libs is true, did you mean no_framework_libs?")
+ } else if *j.deviceProperties.System_modules != "none" && ctx.Config().TargetOpenJDK9() {
+ ctx.AddDependency(ctx.Module(), systemModulesTag, *j.deviceProperties.System_modules)
+ }
+ if ctx.ModuleName() == "framework" {
+ ctx.AddDependency(ctx.Module(), frameworkResTag, "framework-res")
+ }
+ }
+
ctx.AddDependency(ctx.Module(), libTag, j.properties.Libs...)
ctx.AddDependency(ctx.Module(), staticLibTag, j.properties.Static_libs...)
+ ctx.AddDependency(ctx.Module(), libTag, j.properties.Annotation_processors...)
+
+ android.ExtractSourcesDeps(ctx, j.properties.Srcs)
+ android.ExtractSourcesDeps(ctx, j.properties.Java_resources)
+ android.ExtractSourceDeps(ctx, j.properties.Manifest)
+
+ if j.hasSrcExt(".proto") {
+ protoDeps(ctx, &j.protoProperties)
+ }
+
+ if j.hasSrcExt(".kt") {
+ // TODO(ccross): move this to a mutator pass that can tell if generated sources contain
+ // Kotlin files
+ ctx.AddDependency(ctx.Module(), kotlinStdlibTag, "kotlin-stdlib")
+ }
+}
+
+func hasSrcExt(srcs []string, ext string) bool {
+ for _, src := range srcs {
+ if filepath.Ext(src) == ext {
+ return true
+ }
+ }
+
+ return false
+}
+
+func shardPaths(paths android.Paths, shardSize int) []android.Paths {
+ ret := make([]android.Paths, 0, (len(paths)+shardSize-1)/shardSize)
+ for len(paths) > shardSize {
+ ret = append(ret, paths[0:shardSize])
+ paths = paths[shardSize:]
+ }
+ if len(paths) > 0 {
+ ret = append(ret, paths)
+ }
+ return ret
+}
+
+func (j *Module) hasSrcExt(ext string) bool {
+ return hasSrcExt(j.properties.Srcs, ext)
}
func (j *Module) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.OptionalPath,
aidlIncludeDirs android.Paths) []string {
- localAidlIncludes := android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl_includes)
+ aidlIncludes := android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl.Local_include_dirs)
+ aidlIncludes = append(aidlIncludes,
+ android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl.Export_include_dirs)...)
+ aidlIncludes = append(aidlIncludes,
+ android.PathsForSource(ctx, j.deviceProperties.Aidl.Include_dirs)...)
var flags []string
if aidlPreprocess.Valid() {
@@ -210,205 +446,558 @@
}
flags = append(flags, android.JoinWithPrefix(j.exportAidlIncludeDirs.Strings(), "-I"))
- flags = append(flags, android.JoinWithPrefix(localAidlIncludes.Strings(), "-I"))
+ flags = append(flags, android.JoinWithPrefix(aidlIncludes.Strings(), "-I"))
flags = append(flags, "-I"+android.PathForModuleSrc(ctx).String())
- if src := android.ExistentPathForSource(ctx, "", "src"); src.Valid() {
+ if src := android.ExistentPathForSource(ctx, "", ctx.ModuleDir(), "src"); src.Valid() {
flags = append(flags, "-I"+src.String())
}
return flags
}
-func (j *Module) collectDeps(ctx android.ModuleContext) (classpath android.Paths,
- bootClasspath android.OptionalPath, classJarSpecs, resourceJarSpecs []jarSpec, aidlPreprocess android.OptionalPath,
- aidlIncludeDirs android.Paths, srcFileLists android.Paths) {
+type deps struct {
+ classpath android.Paths
+ bootClasspath android.Paths
+ staticJars android.Paths
+ staticHeaderJars android.Paths
+ staticJarResources android.Paths
+ aidlIncludeDirs android.Paths
+ srcJars android.Paths
+ systemModules android.Path
+ aidlPreprocess android.OptionalPath
+ kotlinStdlib android.Paths
+}
- ctx.VisitDirectDeps(func(module blueprint.Module) {
+func checkProducesJars(ctx android.ModuleContext, dep android.SourceFileProducer) {
+ for _, f := range dep.Srcs() {
+ if f.Ext() != ".jar" {
+ ctx.ModuleErrorf("genrule %q must generate files ending with .jar to be used as a libs or static_libs dependency",
+ ctx.OtherModuleName(dep.(blueprint.Module)))
+ }
+ }
+}
+
+func (j *Module) collectDeps(ctx android.ModuleContext) deps {
+ var deps deps
+
+ sdkDep := decodeSdkDep(ctx, String(j.deviceProperties.Sdk_version))
+ if sdkDep.invalidVersion {
+ ctx.AddMissingDependencies([]string{sdkDep.module})
+ } else if sdkDep.useFiles {
+ // sdkDep.jar is actually equivalent to turbine header.jar.
+ deps.classpath = append(deps.classpath, sdkDep.jar)
+ deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, sdkDep.aidl)
+ }
+
+ ctx.VisitDirectDeps(func(module android.Module) {
otherName := ctx.OtherModuleName(module)
tag := ctx.OtherModuleDependencyTag(module)
- dep, _ := module.(Dependency)
- if dep == nil {
+ switch dep := module.(type) {
+ case Dependency:
+ switch tag {
+ case bootClasspathTag:
+ deps.bootClasspath = append(deps.bootClasspath, dep.HeaderJars()...)
+ case libTag:
+ deps.classpath = append(deps.classpath, dep.HeaderJars()...)
+ case staticLibTag:
+ deps.classpath = append(deps.classpath, dep.HeaderJars()...)
+ deps.staticJars = append(deps.staticJars, dep.ImplementationJars()...)
+ deps.staticHeaderJars = append(deps.staticHeaderJars, dep.HeaderJars()...)
+ case frameworkResTag:
+ if ctx.ModuleName() == "framework" {
+ // framework.jar has a one-off dependency on the R.java and Manifest.java files
+ // generated by framework-res.apk
+ deps.srcJars = append(deps.srcJars, dep.(*AndroidApp).aaptSrcJar)
+ }
+ case kotlinStdlibTag:
+ deps.kotlinStdlib = dep.HeaderJars()
+ default:
+ panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName()))
+ }
+
+ deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...)
+ case android.SourceFileProducer:
+ switch tag {
+ case libTag:
+ checkProducesJars(ctx, dep)
+ deps.classpath = append(deps.classpath, dep.Srcs()...)
+ case staticLibTag:
+ checkProducesJars(ctx, dep)
+ deps.classpath = append(deps.classpath, dep.Srcs()...)
+ deps.staticJars = append(deps.staticJars, dep.Srcs()...)
+ deps.staticHeaderJars = append(deps.staticHeaderJars, dep.Srcs()...)
+ case android.DefaultsDepTag, android.SourceDepTag:
+ // Nothing to do
+ default:
+ ctx.ModuleErrorf("dependency on genrule %q may only be in srcs, libs, or static_libs", otherName)
+ }
+ default:
switch tag {
case android.DefaultsDepTag, android.SourceDepTag:
+ // Nothing to do
+ case systemModulesTag:
+ if deps.systemModules != nil {
+ panic("Found two system module dependencies")
+ }
+ sm := module.(*SystemModules)
+ if sm.outputFile == nil {
+ panic("Missing directory for system module dependency")
+ }
+ deps.systemModules = sm.outputFile
default:
ctx.ModuleErrorf("depends on non-java module %q", otherName)
}
- return
}
-
- switch tag {
- case bootClasspathTag:
- bootClasspath = android.OptionalPathForPath(dep.ClasspathFile())
- case libTag:
- classpath = append(classpath, dep.ClasspathFile())
- case staticLibTag:
- classpath = append(classpath, dep.ClasspathFile())
- classJarSpecs = append(classJarSpecs, dep.ClassJarSpecs()...)
- resourceJarSpecs = append(resourceJarSpecs, dep.ResourceJarSpecs()...)
- case frameworkResTag:
- if ctx.ModuleName() == "framework" {
- // framework.jar has a one-off dependency on the R.java and Manifest.java files
- // generated by framework-res.apk
- srcFileLists = append(srcFileLists, module.(*AndroidApp).aaptJavaFileList)
- }
- case sdkDependencyTag:
- sdkDep := module.(sdkDependency)
- if sdkDep.AidlPreprocessed().Valid() {
- if aidlPreprocess.Valid() {
- ctx.ModuleErrorf("multiple dependencies with preprocessed aidls:\n %q\n %q",
- aidlPreprocess, sdkDep.AidlPreprocessed())
- } else {
- aidlPreprocess = sdkDep.AidlPreprocessed()
- }
- }
- default:
- panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName()))
- }
-
- aidlIncludeDirs = append(aidlIncludeDirs, dep.AidlIncludeDirs()...)
})
- return classpath, bootClasspath, classJarSpecs, resourceJarSpecs, aidlPreprocess,
- aidlIncludeDirs, srcFileLists
+ return deps
}
-func (j *Module) compile(ctx android.ModuleContext) {
-
- j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.deviceProperties.Export_aidl_include_dirs)
-
- classpath, bootClasspath, classJarSpecs, resourceJarSpecs, aidlPreprocess,
- aidlIncludeDirs, srcFileLists := j.collectDeps(ctx)
+func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaBuilderFlags {
var flags javaBuilderFlags
+ // javac flags.
javacFlags := j.properties.Javacflags
+ if ctx.Config().TargetOpenJDK9() {
+ javacFlags = append(javacFlags, j.properties.Openjdk9.Javacflags...)
+ }
+ if ctx.Config().MinimizeJavaDebugInfo() {
+ // Override the -g flag passed globally to remove local variable debug info to reduce
+ // disk and memory usage.
+ javacFlags = append(javacFlags, "-g:source,lines")
+ }
if len(javacFlags) > 0 {
+ // optimization.
ctx.Variable(pctx, "javacFlags", strings.Join(javacFlags, " "))
flags.javacFlags = "$javacFlags"
}
- aidlFlags := j.aidlFlags(ctx, aidlPreprocess, aidlIncludeDirs)
+ // javaVersion flag.
+ sdk := sdkStringToNumber(ctx, String(j.deviceProperties.Sdk_version))
+ if j.properties.Java_version != nil {
+ flags.javaVersion = *j.properties.Java_version
+ } else if ctx.Device() && sdk <= 23 {
+ flags.javaVersion = "1.7"
+ } else if ctx.Device() && sdk <= 26 || !ctx.Config().TargetOpenJDK9() {
+ flags.javaVersion = "1.8"
+ } else if ctx.Device() && String(j.deviceProperties.Sdk_version) != "" && sdk == 10000 {
+ // TODO(ccross): once we generate stubs we should be able to use 1.9 for sdk_version: "current"
+ flags.javaVersion = "1.8"
+ } else {
+ flags.javaVersion = "1.9"
+ }
+
+ // classpath
+ flags.bootClasspath.AddPaths(deps.bootClasspath)
+ flags.classpath.AddPaths(deps.classpath)
+ // systemModules
+ if deps.systemModules != nil {
+ flags.systemModules = append(flags.systemModules, deps.systemModules)
+ }
+
+ // aidl flags.
+ aidlFlags := j.aidlFlags(ctx, deps.aidlPreprocess, deps.aidlIncludeDirs)
if len(aidlFlags) > 0 {
+ // optimization.
ctx.Variable(pctx, "aidlFlags", strings.Join(aidlFlags, " "))
flags.aidlFlags = "$aidlFlags"
}
- var deps android.Paths
+ return flags
+}
- if bootClasspath.Valid() {
- flags.bootClasspath = "-bootclasspath " + bootClasspath.String()
- deps = append(deps, bootClasspath.Path())
+func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path) {
+
+ j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl.Export_include_dirs)
+
+ deps := j.collectDeps(ctx)
+ flags := j.collectBuilderFlags(ctx, deps)
+
+ if ctx.Config().TargetOpenJDK9() {
+ j.properties.Srcs = append(j.properties.Srcs, j.properties.Openjdk9.Srcs...)
}
-
- if len(classpath) > 0 {
- flags.classpath = "-classpath " + strings.Join(classpath.Strings(), ":")
- deps = append(deps, classpath...)
- }
-
srcFiles := ctx.ExpandSources(j.properties.Srcs, j.properties.Exclude_srcs)
+ if hasSrcExt(srcFiles.Strings(), ".proto") {
+ flags = protoFlags(ctx, &j.protoProperties, flags)
+ }
srcFiles = j.genSources(ctx, srcFiles, flags)
- ctx.VisitDirectDeps(func(module blueprint.Module) {
- if gen, ok := module.(genrule.SourceFileGenerator); ok {
- srcFiles = append(srcFiles, gen.GeneratedSourceFiles()...)
+ srcJars := srcFiles.FilterByExt(".srcjar")
+ srcJars = append(srcJars, deps.srcJars...)
+ srcJars = append(srcJars, extraSrcJars...)
+
+ var jars android.Paths
+
+ jarName := ctx.ModuleName() + ".jar"
+
+ if srcFiles.HasExt(".kt") {
+ // If there are kotlin files, compile them first but pass all the kotlin and java files
+ // kotlinc will use the java files to resolve types referenced by the kotlin files, but
+ // won't emit any classes for them.
+
+ flags.kotlincFlags = "-no-stdlib"
+ if ctx.Device() {
+ flags.kotlincFlags += " -no-jdk"
}
- })
- srcFileLists = append(srcFileLists, j.ExtraSrcLists...)
+ flags.kotlincClasspath = append(flags.kotlincClasspath, deps.kotlinStdlib...)
+ flags.kotlincClasspath = append(flags.kotlincClasspath, deps.classpath...)
- if len(srcFiles) > 0 {
- // Compile java sources into .class files
- classes := TransformJavaToClasses(ctx, srcFiles, srcFileLists, flags, deps)
+ kotlinJar := android.PathForModuleOut(ctx, "kotlin", jarName)
+ TransformKotlinToClasses(ctx, kotlinJar, srcFiles, srcJars, flags)
if ctx.Failed() {
return
}
- classJarSpecs = append([]jarSpec{classes}, classJarSpecs...)
+ // Make javac rule depend on the kotlinc rule
+ flags.classpath = append(flags.classpath, kotlinJar)
+ // Jar kotlin classes into the final jar after javac
+ jars = append(jars, kotlinJar)
+ jars = append(jars, deps.kotlinStdlib...)
}
- resourceJarSpecs = append(ResourceDirsToJarSpecs(ctx, j.properties.Resource_dirs, j.properties.Exclude_resource_dirs),
- resourceJarSpecs...)
+ javaSrcFiles := srcFiles.FilterByExt(".java")
+ var uniqueSrcFiles android.Paths
+ set := make(map[string]bool)
+ for _, v := range javaSrcFiles {
+ if _, found := set[v.String()]; !found {
+ set[v.String()] = true
+ uniqueSrcFiles = append(uniqueSrcFiles, v)
+ }
+ }
- manifest := android.OptionalPathForModuleSrc(ctx, j.properties.Manifest)
+ // Store the list of .java files that was passed to javac
+ j.compiledJavaSrcs = uniqueSrcFiles
+ j.compiledSrcJars = srcJars
+ fullD8 := ctx.AConfig().IsEnvTrue("USE_D8_DESUGAR")
- allJarSpecs := append([]jarSpec(nil), classJarSpecs...)
- allJarSpecs = append(allJarSpecs, resourceJarSpecs...)
+ enable_sharding := false
+ if ctx.Device() && !ctx.Config().IsEnvFalse("TURBINE_ENABLED") {
+ if j.properties.Javac_shard_size != nil && *(j.properties.Javac_shard_size) > 0 {
+ enable_sharding = true
+ if len(j.properties.Annotation_processors) != 0 ||
+ len(j.properties.Annotation_processor_classes) != 0 {
+ ctx.PropertyErrorf("javac_shard_size",
+ "%q cannot be set when annotation processors are enabled.",
+ j.properties.Javac_shard_size)
+ }
+ }
+ // If sdk jar is java module, then directly return classesJar as header.jar
+ if j.Name() != "android_stubs_current" && j.Name() != "android_system_stubs_current" &&
+ j.Name() != "android_test_stubs_current" {
+ j.headerJarFile = j.compileJavaHeader(ctx, uniqueSrcFiles, srcJars, deps, flags, jarName)
+ if ctx.Failed() {
+ return
+ }
+ }
+ }
+ if len(uniqueSrcFiles) > 0 || len(srcJars) > 0 {
+ var extraJarDeps android.Paths
+ if ctx.Config().IsEnvTrue("RUN_ERROR_PRONE") {
+ // If error-prone is enabled, add an additional rule to compile the java files into
+ // a separate set of classes (so that they don't overwrite the normal ones and require
+ // a rebuild when error-prone is turned off).
+ // TODO(ccross): Once we always compile with javac9 we may be able to conditionally
+ // enable error-prone without affecting the output class files.
+ errorprone := android.PathForModuleOut(ctx, "errorprone", jarName)
+ RunErrorProne(ctx, errorprone, uniqueSrcFiles, srcJars, flags)
+ extraJarDeps = append(extraJarDeps, errorprone)
+ }
- // Combine classes + resources into classes-full-debug.jar
- outputFile := TransformClassesToJar(ctx, allJarSpecs, manifest)
- if ctx.Failed() {
- return
+ if enable_sharding {
+ flags.classpath.AddPaths([]android.Path{j.headerJarFile})
+ shardSize := int(*(j.properties.Javac_shard_size))
+ var shardSrcs []android.Paths
+ if len(uniqueSrcFiles) > 0 {
+ shardSrcs = shardPaths(uniqueSrcFiles, shardSize)
+ for idx, shardSrc := range shardSrcs {
+ classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx))
+ TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps)
+ jars = append(jars, classes)
+ }
+ }
+ if len(srcJars) > 0 {
+ classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(len(shardSrcs)))
+ TransformJavaToClasses(ctx, classes, len(shardSrcs), nil, srcJars, flags, extraJarDeps)
+ jars = append(jars, classes)
+ }
+ } else {
+ classes := android.PathForModuleOut(ctx, "javac", jarName)
+ TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps)
+ jars = append(jars, classes)
+ }
+ if ctx.Failed() {
+ return
+ }
+ }
+
+ dirArgs, dirDeps := ResourceDirsToJarArgs(ctx, j.properties.Java_resource_dirs, j.properties.Exclude_java_resource_dirs)
+ fileArgs, fileDeps := ResourceFilesToJarArgs(ctx, j.properties.Java_resources, j.properties.Exclude_java_resources)
+
+ var resArgs []string
+ var resDeps android.Paths
+
+ resArgs = append(resArgs, dirArgs...)
+ resDeps = append(resDeps, dirDeps...)
+
+ resArgs = append(resArgs, fileArgs...)
+ resDeps = append(resDeps, fileDeps...)
+
+ if proptools.Bool(j.properties.Include_srcs) {
+ srcArgs, srcDeps := SourceFilesToJarArgs(ctx, j.properties.Srcs, j.properties.Exclude_srcs)
+ resArgs = append(resArgs, srcArgs...)
+ resDeps = append(resDeps, srcDeps...)
+ }
+
+ if len(resArgs) > 0 {
+ resourceJar := android.PathForModuleOut(ctx, "res", jarName)
+ TransformResourcesToJar(ctx, resourceJar, resArgs, resDeps)
+ if ctx.Failed() {
+ return
+ }
+
+ jars = append(jars, resourceJar)
+ }
+
+ // static classpath jars have the resources in them, so the resource jars aren't necessary here
+ jars = append(jars, deps.staticJars...)
+
+ var manifest android.OptionalPath
+ if j.properties.Manifest != nil {
+ manifest = android.OptionalPathForPath(ctx.ExpandSource(*j.properties.Manifest, "manifest"))
+ }
+
+ // Combine the classes built from sources, any manifests, and any static libraries into
+ // classes.jar. If there is only one input jar this step will be skipped.
+ var outputFile android.Path
+
+ if len(jars) == 1 && !manifest.Valid() {
+ // Optimization: skip the combine step if there is nothing to do
+ outputFile = jars[0]
+ } else {
+ combinedJar := android.PathForModuleOut(ctx, "combined", jarName)
+ TransformJarsToJar(ctx, combinedJar, "for javac", jars, manifest, false, nil)
+ outputFile = combinedJar
}
if j.properties.Jarjar_rules != nil {
jarjar_rules := android.PathForModuleSrc(ctx, *j.properties.Jarjar_rules)
- // Transform classes-full-debug.jar into classes-jarjar.jar
- outputFile = TransformJarJar(ctx, outputFile, jarjar_rules)
+ // Transform classes.jar into classes-jarjar.jar
+ jarjarFile := android.PathForModuleOut(ctx, "jarjar", jarName)
+ TransformJarJar(ctx, jarjarFile, outputFile, jarjar_rules)
+ outputFile = jarjarFile
if ctx.Failed() {
return
}
-
- classes, _ := TransformPrebuiltJarToClasses(ctx, outputFile)
- classJarSpecs = []jarSpec{classes}
+ }
+ j.implementationJarFile = outputFile
+ if j.headerJarFile == nil {
+ j.headerJarFile = j.implementationJarFile
}
- j.resourceJarSpecs = resourceJarSpecs
- j.classJarSpecs = classJarSpecs
- j.classpathFile = outputFile
+ if !fullD8 && ctx.Device() && j.installable() {
+ outputFile = j.desugar(ctx, flags, outputFile, jarName)
+ }
- if j.deviceProperties.Dex && len(srcFiles) > 0 {
- dxFlags := j.deviceProperties.Dxflags
- if false /* emma enabled */ {
- // If you instrument class files that have local variable debug information in
- // them emma does not correctly maintain the local variable table.
- // This will cause an error when you try to convert the class files for Android.
- // The workaround here is to build different dex file here based on emma switch
- // then later copy into classes.dex. When emma is on, dx is run with --no-locals
- // option to remove local variable information
- dxFlags = append(dxFlags, "--no-locals")
+ if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") {
+ if inList(ctx.ModuleName(), config.InstrumentFrameworkModules) {
+ j.properties.Instrument = true
}
+ }
- if ctx.AConfig().Getenv("NO_OPTIMIZE_DX") != "" {
- dxFlags = append(dxFlags, "--no-optimize")
+ if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT") && j.properties.Instrument {
+ outputFile = j.instrument(ctx, flags, outputFile, jarName)
+ }
+
+ if ctx.Device() && j.installable() {
+ if fullD8 {
+ outputFile = j.compileDexFullD8(ctx, flags, outputFile, jarName)
+ } else {
+ outputFile = j.compileDex(ctx, flags, outputFile, jarName)
}
-
- if ctx.AConfig().Getenv("GENERATE_DEX_DEBUG") != "" {
- dxFlags = append(dxFlags,
- "--debug",
- "--verbose",
- "--dump-to="+android.PathForModuleOut(ctx, "classes.lst").String(),
- "--dump-width=1000")
- }
-
- flags.dxFlags = strings.Join(dxFlags, " ")
-
- // Compile classes.jar into classes.dex
- dexJarSpec := TransformClassesJarToDex(ctx, outputFile, flags)
if ctx.Failed() {
return
}
-
- // Combine classes.dex + resources into javalib.jar
- outputFile = TransformDexToJavaLib(ctx, resourceJarSpecs, dexJarSpec)
}
ctx.CheckbuildFile(outputFile)
j.outputFile = outputFile
}
+func (j *Module) compileJavaHeader(ctx android.ModuleContext, srcFiles, srcJars android.Paths,
+ deps deps, flags javaBuilderFlags, jarName string) android.Path {
+
+ var jars android.Paths
+ if len(srcFiles) > 0 || len(srcJars) > 0 {
+ // Compile java sources into turbine.jar.
+ turbineJar := android.PathForModuleOut(ctx, "turbine", jarName)
+ TransformJavaToHeaderClasses(ctx, turbineJar, srcFiles, srcJars, flags)
+ if ctx.Failed() {
+ return nil
+ }
+ jars = append(jars, turbineJar)
+ }
+
+ // Combine any static header libraries into classes-header.jar. If there is only
+ // one input jar this step will be skipped.
+ var headerJar android.Path
+ jars = append(jars, deps.staticHeaderJars...)
+
+ // we cannot skip the combine step for now if there is only one jar
+ // since we have to strip META-INF/TRANSITIVE dir from turbine.jar
+ combinedJar := android.PathForModuleOut(ctx, "turbine-combined", jarName)
+ TransformJarsToJar(ctx, combinedJar, "for turbine", jars, android.OptionalPath{}, false, []string{"META-INF"})
+ headerJar = combinedJar
+
+ if j.properties.Jarjar_rules != nil {
+ jarjar_rules := android.PathForModuleSrc(ctx, *j.properties.Jarjar_rules)
+ // Transform classes.jar into classes-jarjar.jar
+ jarjarFile := android.PathForModuleOut(ctx, "turbine-jarjar", jarName)
+ TransformJarJar(ctx, jarjarFile, headerJar, jarjar_rules)
+ headerJar = jarjarFile
+ if ctx.Failed() {
+ return nil
+ }
+ }
+
+ return headerJar
+}
+
+func (j *Module) desugar(ctx android.ModuleContext, flags javaBuilderFlags,
+ classesJar android.Path, jarName string) android.Path {
+
+ desugarFlags := []string{
+ "--min_sdk_version " + j.minSdkVersionNumber(ctx),
+ "--desugar_try_with_resources_if_needed=false",
+ "--allow_empty_bootclasspath",
+ }
+
+ if inList("--core-library", j.deviceProperties.Dxflags) {
+ desugarFlags = append(desugarFlags, "--core_library")
+ }
+
+ flags.desugarFlags = strings.Join(desugarFlags, " ")
+
+ desugarJar := android.PathForModuleOut(ctx, "desugar", jarName)
+ TransformDesugar(ctx, desugarJar, classesJar, flags)
+
+ return desugarJar
+}
+
+func (j *Module) instrument(ctx android.ModuleContext, flags javaBuilderFlags,
+ classesJar android.Path, jarName string) android.Path {
+
+ specs := j.jacocoStripSpecs(ctx)
+
+ jacocoReportClassesFile := android.PathForModuleOut(ctx, "jacoco", "jacoco-report-classes.jar")
+ instrumentedJar := android.PathForModuleOut(ctx, "jacoco", jarName)
+
+ jacocoInstrumentJar(ctx, instrumentedJar, jacocoReportClassesFile, classesJar, specs)
+
+ j.jacocoReportClassesFile = jacocoReportClassesFile
+
+ return instrumentedJar
+}
+
+func (j *Module) compileDex(ctx android.ModuleContext, flags javaBuilderFlags,
+ classesJar android.Path, jarName string) android.Path {
+
+ dxFlags := j.deviceProperties.Dxflags
+
+ if ctx.Config().Getenv("NO_OPTIMIZE_DX") != "" {
+ dxFlags = append(dxFlags, "--no-optimize")
+ }
+
+ if ctx.Config().Getenv("GENERATE_DEX_DEBUG") != "" {
+ dxFlags = append(dxFlags,
+ "--debug",
+ "--verbose",
+ "--dump-to="+android.PathForModuleOut(ctx, "classes.lst").String(),
+ "--dump-width=1000")
+ }
+
+ dxFlags = append(dxFlags, "--min-sdk-version="+j.minSdkVersionNumber(ctx))
+
+ flags.dxFlags = strings.Join(dxFlags, " ")
+
+ // Compile classes.jar into classes.dex and then javalib.jar
+ javalibJar := android.PathForModuleOut(ctx, "dex", jarName)
+ TransformClassesJarToDexJar(ctx, javalibJar, classesJar, flags)
+
+ j.dexJarFile = javalibJar
+ return javalibJar
+}
+
+func (j *Module) compileDexFullD8(ctx android.ModuleContext, flags javaBuilderFlags,
+ classesJar android.Path, jarName string) android.Path {
+
+ // Translate all the DX flags to D8 ones until all the build files have been migrated
+ // to D8 flags. See: b/69377755
+ var dxFlags []string
+ for _, x := range j.deviceProperties.Dxflags {
+ if x == "--core-library" {
+ continue
+ }
+ if x == "--dex" {
+ continue
+ }
+ if x == "--multi-dex" {
+ continue
+ }
+ if x == "--no-locals" {
+ dxFlags = append(dxFlags, "--release")
+ continue
+ }
+ dxFlags = append(dxFlags, x)
+ }
+
+ if ctx.AConfig().Getenv("NO_OPTIMIZE_DX") != "" {
+ dxFlags = append(dxFlags, "--debug")
+ }
+
+ if ctx.AConfig().Getenv("GENERATE_DEX_DEBUG") != "" {
+ dxFlags = append(dxFlags,
+ "--debug",
+ "--verbose")
+ }
+
+ dxFlags = append(dxFlags, "--min-api "+j.minSdkVersionNumber(ctx))
+
+ flags.dxFlags = strings.Join(dxFlags, " ")
+
+ // Compile classes.jar into classes.dex and then javalib.jar
+ javalibJar := android.PathForModuleOut(ctx, "dex", jarName)
+ TransformClassesJarToDexJar(ctx, javalibJar, classesJar, flags)
+
+ j.dexJarFile = javalibJar
+ return javalibJar
+}
+
+// Returns a sdk version as a string that is guaranteed to be a parseable as a number. For
+// modules targeting an unreleased SDK (meaning it does not yet have a number) it returns "10000".
+func (j *Module) minSdkVersionNumber(ctx android.ModuleContext) string {
+ switch String(j.deviceProperties.Sdk_version) {
+ case "", "current", "test_current", "system_current":
+ return strconv.Itoa(ctx.Config().DefaultAppTargetSdkInt())
+ default:
+ return android.GetNumericSdkVersion(String(j.deviceProperties.Sdk_version))
+ }
+}
+
+func (j *Module) installable() bool {
+ return j.properties.Installable == nil || *j.properties.Installable
+}
+
var _ Dependency = (*Library)(nil)
-func (j *Module) ClasspathFile() android.Path {
- return j.classpathFile
+func (j *Module) HeaderJars() android.Paths {
+ return android.Paths{j.headerJarFile}
}
-func (j *Module) ClassJarSpecs() []jarSpec {
- return j.classJarSpecs
-}
-
-func (j *Module) ResourceJarSpecs() []jarSpec {
- return j.resourceJarSpecs
+func (j *Module) ImplementationJars() android.Paths {
+ return android.Paths{j.implementationJarFile}
}
func (j *Module) AidlIncludeDirs() android.Paths {
@@ -432,30 +1021,40 @@
func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) {
j.compile(ctx)
- j.installFile = ctx.InstallFileName(android.PathForModuleInstall(ctx, "framework"), ctx.ModuleName()+".jar", j.outputFile)
+ if j.installable() {
+ j.installFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"),
+ ctx.ModuleName()+".jar", j.outputFile)
+ }
}
func (j *Library) DepsMutator(ctx android.BottomUpMutatorContext) {
j.deps(ctx)
}
-func LibraryFactory() android.Module {
- module := &Library{}
+func LibraryFactory(installable bool) func() android.Module {
+ return func() android.Module {
+ module := &Library{}
- module.deviceProperties.Dex = true
+ if !installable {
+ module.properties.Installable = proptools.BoolPtr(false)
+ }
- module.AddProperties(
- &module.Module.properties,
- &module.Module.deviceProperties)
+ module.AddProperties(
+ &module.Module.properties,
+ &module.Module.deviceProperties,
+ &module.Module.protoProperties)
- InitJavaModule(module, android.HostAndDeviceSupported)
- return module
+ InitJavaModule(module, android.HostAndDeviceSupported)
+ return module
+ }
}
func LibraryHostFactory() android.Module {
module := &Library{}
- module.AddProperties(&module.Module.properties)
+ module.AddProperties(
+ &module.Module.properties,
+ &module.Module.protoProperties)
InitJavaModule(module, android.HostSupported)
return module
@@ -467,39 +1066,66 @@
type binaryProperties struct {
// installable script to execute the resulting jar
- Wrapper string
+ Wrapper *string
}
type Binary struct {
Library
binaryProperties binaryProperties
+
+ isWrapperVariant bool
+
+ wrapperFile android.Path
+ binaryFile android.OutputPath
+}
+
+func (j *Binary) HostToolPath() android.OptionalPath {
+ return android.OptionalPathForPath(j.binaryFile)
}
func (j *Binary) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- j.Library.GenerateAndroidBuildActions(ctx)
+ if ctx.Arch().ArchType == android.Common {
+ // Compile the jar
+ j.Library.GenerateAndroidBuildActions(ctx)
+ } else {
+ // Handle the binary wrapper
+ j.isWrapperVariant = true
- // Depend on the installed jar (j.installFile) so that the wrapper doesn't get executed by
- // another build rule before the jar has been installed.
- ctx.InstallFile(android.PathForModuleInstall(ctx, "bin"), android.PathForModuleSrc(ctx, j.binaryProperties.Wrapper),
- j.installFile)
+ if j.binaryProperties.Wrapper != nil {
+ j.wrapperFile = ctx.ExpandSource(*j.binaryProperties.Wrapper, "wrapper")
+ } else {
+ j.wrapperFile = android.PathForSource(ctx, "build/soong/scripts/jar-wrapper.sh")
+ }
+
+ // Depend on the installed jar so that the wrapper doesn't get executed by
+ // another build rule before the jar has been installed.
+ jarFile := ctx.PrimaryModule().(*Binary).installFile
+
+ j.binaryFile = ctx.InstallExecutable(android.PathForModuleInstall(ctx, "bin"),
+ ctx.ModuleName(), j.wrapperFile, jarFile)
+ }
}
func (j *Binary) DepsMutator(ctx android.BottomUpMutatorContext) {
- j.deps(ctx)
+ if ctx.Arch().ArchType == android.Common {
+ j.deps(ctx)
+ } else {
+ android.ExtractSourceDeps(ctx, j.binaryProperties.Wrapper)
+ }
}
func BinaryFactory() android.Module {
module := &Binary{}
- module.deviceProperties.Dex = true
-
module.AddProperties(
&module.Module.properties,
&module.Module.deviceProperties,
+ &module.Module.protoProperties,
&module.binaryProperties)
- InitJavaModule(module, android.HostAndDeviceSupported)
+ android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommonFirst)
+ android.InitDefaultableModule(module)
return module
}
@@ -509,9 +1135,11 @@
module.AddProperties(
&module.Module.properties,
&module.Module.deviceProperties,
+ &module.Module.protoProperties,
&module.binaryProperties)
- InitJavaModule(module, android.HostSupported)
+ android.InitAndroidArchModule(module, android.HostSupported, android.MultilibCommonFirst)
+ android.InitDefaultableModule(module)
return module
}
@@ -519,100 +1147,80 @@
// Java prebuilts
//
-type Prebuilt struct {
+type ImportProperties struct {
+ Jars []string
+
+ Sdk_version *string
+
+ Installable *bool
+}
+
+type Import struct {
android.ModuleBase
prebuilt android.Prebuilt
- classpathFile android.Path
- classJarSpecs, resourceJarSpecs []jarSpec
+ properties ImportProperties
+
+ classpathFiles android.Paths
+ combinedClasspathFile android.Path
}
-func (j *Prebuilt) Prebuilt() *android.Prebuilt {
+func (j *Import) Prebuilt() *android.Prebuilt {
return &j.prebuilt
}
-func (j *Prebuilt) DepsMutator(ctx android.BottomUpMutatorContext) {
+func (j *Import) PrebuiltSrcs() []string {
+ return j.properties.Jars
}
-func (j *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- prebuilt := j.prebuilt.Path(ctx)
-
- classJarSpec, resourceJarSpec := TransformPrebuiltJarToClasses(ctx, prebuilt)
-
- j.classpathFile = prebuilt
- j.classJarSpecs = []jarSpec{classJarSpec}
- j.resourceJarSpecs = []jarSpec{resourceJarSpec}
- ctx.InstallFileName(android.PathForModuleInstall(ctx, "framework"), ctx.ModuleName()+".jar", j.classpathFile)
+func (j *Import) Name() string {
+ return j.prebuilt.Name(j.ModuleBase.Name())
}
-var _ Dependency = (*Prebuilt)(nil)
-
-func (j *Prebuilt) ClasspathFile() android.Path {
- return j.classpathFile
+func (j *Import) DepsMutator(ctx android.BottomUpMutatorContext) {
}
-func (j *Prebuilt) ClassJarSpecs() []jarSpec {
- return j.classJarSpecs
+func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ j.classpathFiles = android.PathsForModuleSrc(ctx, j.properties.Jars)
+
+ outputFile := android.PathForModuleOut(ctx, "classes.jar")
+ TransformJarsToJar(ctx, outputFile, "for prebuilts", j.classpathFiles, android.OptionalPath{}, false, nil)
+ j.combinedClasspathFile = outputFile
}
-func (j *Prebuilt) ResourceJarSpecs() []jarSpec {
- return j.resourceJarSpecs
+var _ Dependency = (*Import)(nil)
+
+func (j *Import) HeaderJars() android.Paths {
+ return j.classpathFiles
}
-func (j *Prebuilt) AidlIncludeDirs() android.Paths {
+func (j *Import) ImplementationJars() android.Paths {
+ return j.classpathFiles
+}
+
+func (j *Import) AidlIncludeDirs() android.Paths {
return nil
}
-func PrebuiltFactory() android.Module {
- module := &Prebuilt{}
+var _ android.PrebuiltInterface = (*Import)(nil)
- module.AddProperties(&module.prebuilt.Properties)
+func ImportFactory() android.Module {
+ module := &Import{}
+ module.AddProperties(&module.properties)
+
+ android.InitPrebuiltModule(module, &module.properties.Jars)
android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon)
return module
}
-//
-// SDK java prebuilts (.jar containing resources plus framework.aidl)
-//
+func ImportFactoryHost() android.Module {
+ module := &Import{}
-type sdkDependency interface {
- Dependency
- AidlPreprocessed() android.OptionalPath
-}
+ module.AddProperties(&module.properties)
-var _ sdkDependency = (*sdkPrebuilt)(nil)
-
-type sdkPrebuiltProperties struct {
- Aidl_preprocessed *string
-}
-
-type sdkPrebuilt struct {
- Prebuilt
-
- sdkProperties sdkPrebuiltProperties
-
- aidlPreprocessed android.OptionalPath
-}
-
-func (j *sdkPrebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- j.Prebuilt.GenerateAndroidBuildActions(ctx)
-
- j.aidlPreprocessed = android.OptionalPathForModuleSrc(ctx, j.sdkProperties.Aidl_preprocessed)
-}
-
-func (j *sdkPrebuilt) AidlPreprocessed() android.OptionalPath {
- return j.aidlPreprocessed
-}
-
-func SdkPrebuiltFactory() android.Module {
- module := &sdkPrebuilt{}
-
- module.AddProperties(
- &module.prebuilt.Properties,
- &module.sdkProperties)
-
- android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon)
+ android.InitPrebuiltModule(module, &module.properties.Jars)
+ android.InitAndroidArchModule(module, android.HostSupported, android.MultilibCommon)
return module
}
@@ -656,3 +1264,6 @@
return module
}
+
+var Bool = proptools.Bool
+var String = proptools.String
diff --git a/java/java_test.go b/java/java_test.go
index 3443610..78fbd41 100644
--- a/java/java_test.go
+++ b/java/java_test.go
@@ -16,10 +16,13 @@
import (
"android/soong/android"
+ "android/soong/genrule"
"fmt"
"io/ioutil"
"os"
"path/filepath"
+ "reflect"
+ "strconv"
"strings"
"testing"
)
@@ -49,45 +52,146 @@
os.Exit(run())
}
-func testJava(t *testing.T, bp string) *android.TestContext {
- config := android.TestConfig(buildDir)
+func testConfig(env map[string]string) android.Config {
+ return android.TestArchConfig(buildDir, env)
- ctx := android.NewTestContext()
+}
+
+func testContext(config android.Config, bp string,
+ fs map[string][]byte) *android.TestContext {
+
+ ctx := android.NewTestArchContext()
ctx.RegisterModuleType("android_app", android.ModuleFactoryAdaptor(AndroidAppFactory))
- ctx.RegisterModuleType("java_library", android.ModuleFactoryAdaptor(LibraryFactory))
- ctx.RegisterModuleType("java_prebuilt_library", android.ModuleFactoryAdaptor(PrebuiltFactory))
+ ctx.RegisterModuleType("java_binary_host", android.ModuleFactoryAdaptor(BinaryHostFactory))
+ ctx.RegisterModuleType("java_library", android.ModuleFactoryAdaptor(LibraryFactory(true)))
+ ctx.RegisterModuleType("java_library_host", android.ModuleFactoryAdaptor(LibraryHostFactory))
+ ctx.RegisterModuleType("java_import", android.ModuleFactoryAdaptor(ImportFactory))
ctx.RegisterModuleType("java_defaults", android.ModuleFactoryAdaptor(defaultsFactory))
+ ctx.RegisterModuleType("java_system_modules", android.ModuleFactoryAdaptor(SystemModulesFactory))
+ ctx.RegisterModuleType("java_genrule", android.ModuleFactoryAdaptor(genRuleFactory))
+ ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(genrule.FileGroupFactory))
+ ctx.RegisterModuleType("genrule", android.ModuleFactoryAdaptor(genrule.GenRuleFactory))
+ ctx.PreArchMutators(android.RegisterPrebuiltsPreArchMutators)
+ ctx.PreArchMutators(android.RegisterPrebuiltsPostDepsMutators)
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
+ ctx.RegisterPreSingletonType("overlay", android.SingletonFactoryAdaptor(OverlaySingletonFactory))
ctx.Register()
- extraModules := []string{"core-libart", "frameworks", "sdk_v14"}
+ extraModules := []string{
+ "core-oj",
+ "core-libart",
+ "framework",
+ "ext",
+ "okhttp",
+ "android_stubs_current",
+ "android_system_stubs_current",
+ "android_test_stubs_current",
+ "kotlin-stdlib",
+ }
for _, extra := range extraModules {
bp += fmt.Sprintf(`
java_library {
name: "%s",
- no_standard_libraries: true,
+ srcs: ["a.java"],
+ no_standard_libs: true,
+ system_modules: "core-system-modules",
}
`, extra)
}
- ctx.MockFileSystem(map[string][]byte{
- "Android.bp": []byte(bp),
- "a.java": nil,
- "b.java": nil,
- "c.java": nil,
- "a.jar": nil,
- "b.jar": nil,
- })
+ bp += `
+ android_app {
+ name: "framework-res",
+ no_framework_libs: true,
+ }
+ `
- _, errs := ctx.ParseBlueprintsFiles("Android.bp")
+ systemModules := []string{
+ "core-system-modules",
+ "android_stubs_current_system_modules",
+ "android_system_stubs_current_system_modules",
+ "android_test_stubs_current_system_modules",
+ }
+
+ for _, extra := range systemModules {
+ bp += fmt.Sprintf(`
+ java_system_modules {
+ name: "%s",
+ }
+ `, extra)
+ }
+
+ mockFS := map[string][]byte{
+ "Android.bp": []byte(bp),
+ "a.java": nil,
+ "b.java": nil,
+ "c.java": nil,
+ "b.kt": nil,
+ "a.jar": nil,
+ "b.jar": nil,
+ "java-res/a": nil,
+ "java-res/b": nil,
+ "java-res2/a": nil,
+
+ "prebuilts/sdk/14/android.jar": nil,
+ "prebuilts/sdk/14/framework.aidl": nil,
+ "prebuilts/sdk/current/android.jar": nil,
+ "prebuilts/sdk/current/framework.aidl": nil,
+ "prebuilts/sdk/system_current/android.jar": nil,
+ "prebuilts/sdk/system_current/framework.aidl": nil,
+ "prebuilts/sdk/system_14/android.jar": nil,
+ "prebuilts/sdk/system_14/framework.aidl": nil,
+ "prebuilts/sdk/test_current/android.jar": nil,
+ "prebuilts/sdk/test_current/framework.aidl": nil,
+
+ // For framework-res, which is an implicit dependency for framework
+ "AndroidManifest.xml": nil,
+ "build/target/product/security/testkey": nil,
+
+ "build/soong/scripts/jar-wrapper.sh": nil,
+ }
+
+ for k, v := range fs {
+ mockFS[k] = v
+ }
+
+ ctx.MockFileSystem(mockFS)
+
+ return ctx
+}
+
+func run(t *testing.T, ctx *android.TestContext, config android.Config) {
+ t.Helper()
+ _, errs := ctx.ParseFileList(".", []string{"Android.bp"})
fail(t, errs)
_, errs = ctx.PrepareBuildActions(config)
fail(t, errs)
+}
+
+func testJava(t *testing.T, bp string) *android.TestContext {
+ t.Helper()
+ config := testConfig(nil)
+ ctx := testContext(config, bp, nil)
+ run(t, ctx, config)
return ctx
}
+func moduleToPath(name string) string {
+ switch {
+ case name == `""`:
+ return name
+ case strings.HasSuffix(name, ".jar"):
+ return name
+ case name == "android_stubs_current" || name == "android_system_stubs_current" ||
+ name == "android_test_stubs_current":
+ return filepath.Join(buildDir, ".intermediates", name, "android_common", "javac", name+".jar")
+ default:
+ return filepath.Join(buildDir, ".intermediates", name, "android_common", "turbine-combined", name+".jar")
+ }
+}
+
func TestSimple(t *testing.T) {
ctx := testJava(t, `
java_library {
@@ -106,96 +210,280 @@
name: "baz",
srcs: ["c.java"],
}
- `)
+ `)
- javac := ctx.ModuleForTests("foo", "").Rule("javac")
- jar := ctx.ModuleForTests("foo", "").Rule("jar")
+ javac := ctx.ModuleForTests("foo", "android_common").Rule("javac")
+ combineJar := ctx.ModuleForTests("foo", "android_common").Description("for javac")
if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" {
t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs)
}
- bar := filepath.Join(buildDir, ".intermediates", "bar", "classes-full-debug.jar")
- if !strings.Contains(javac.Args["classpath"], bar) {
- t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], bar)
+ baz := ctx.ModuleForTests("baz", "android_common").Rule("javac").Output.String()
+ barTurbine := filepath.Join(buildDir, ".intermediates", "bar", "android_common", "turbine-combined", "bar.jar")
+ bazTurbine := filepath.Join(buildDir, ".intermediates", "baz", "android_common", "turbine-combined", "baz.jar")
+
+ if !strings.Contains(javac.Args["classpath"], barTurbine) {
+ t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], barTurbine)
}
- baz := filepath.Join(buildDir, ".intermediates", "baz", "classes.list")
- if !strings.Contains(jar.Args["jarArgs"], baz) {
- t.Errorf("foo jarArgs %v does not contain %q", jar.Args["jarArgs"], baz)
+ if !strings.Contains(javac.Args["classpath"], bazTurbine) {
+ t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], bazTurbine)
+ }
+
+ if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != baz {
+ t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, baz)
}
}
-func TestSdk(t *testing.T) {
+func TestArchSpecific(t *testing.T) {
ctx := testJava(t, `
java_library {
- name: "foo1",
+ name: "foo",
+ srcs: ["a.java"],
+ target: {
+ android: {
+ srcs: ["b.java"],
+ },
+ },
+ }
+ `)
+
+ javac := ctx.ModuleForTests("foo", "android_common").Rule("javac")
+ if len(javac.Inputs) != 2 || javac.Inputs[0].String() != "a.java" || javac.Inputs[1].String() != "b.java" {
+ t.Errorf(`foo inputs %v != ["a.java", "b.java"]`, javac.Inputs)
+ }
+}
+
+func TestBinary(t *testing.T) {
+ ctx := testJava(t, `
+ java_library_host {
+ name: "foo",
srcs: ["a.java"],
}
- java_library {
- name: "foo2",
- srcs: ["a.java"],
- sdk_version: "",
+ java_binary_host {
+ name: "bar",
+ srcs: ["b.java"],
+ static_libs: ["foo"],
}
+ `)
- java_library {
- name: "foo3",
- srcs: ["a.java"],
- sdk_version: "14",
- }
+ buildOS := android.BuildOs.String()
- java_library {
- name: "foo4",
- srcs: ["a.java"],
- sdk_version: "current",
- }
+ bar := ctx.ModuleForTests("bar", buildOS+"_common")
+ barJar := bar.Output("bar.jar").Output.String()
+ barWrapper := ctx.ModuleForTests("bar", buildOS+"_x86_64")
+ barWrapperDeps := barWrapper.Output("bar").Implicits.Strings()
- java_library {
- name: "foo5",
- srcs: ["a.java"],
- sdk_version: "system_current",
- }
-
- java_library {
- name: "foo6",
- srcs: ["a.java"],
- sdk_version: "test_current",
- }
- `)
-
- type depType int
- const (
- staticLib = iota
- classpathLib
- bootclasspathLib
- )
-
- check := func(module, dep string, depType depType) {
- if dep != "" {
- dep = filepath.Join(buildDir, ".intermediates", dep, "classes-full-debug.jar")
- }
-
- javac := ctx.ModuleForTests(module, "").Rule("javac")
-
- if depType == bootclasspathLib {
- got := strings.TrimPrefix(javac.Args["bootClasspath"], "-bootclasspath ")
- if got != dep {
- t.Errorf("module %q bootclasspath %q != %q", module, got, dep)
- }
- } else if depType == classpathLib {
- got := strings.TrimPrefix(javac.Args["classpath"], "-classpath ")
- if got != dep {
- t.Errorf("module %q classpath %q != %q", module, got, dep)
- }
- }
-
- if len(javac.Implicits) != 1 || javac.Implicits[0].String() != dep {
- t.Errorf("module %q implicits != [%q]", dep)
- }
+ // Test that the install binary wrapper depends on the installed jar file
+ if len(barWrapperDeps) != 1 || barWrapperDeps[0] != barJar {
+ t.Errorf("expected binary wrapper implicits [%q], got %v",
+ barJar, barWrapperDeps)
}
- check("foo1", "core-libart", bootclasspathLib)
+}
+
+var classpathTestcases = []struct {
+ name string
+ moduleType string
+ host android.OsClass
+ properties string
+ bootclasspath []string
+ system string
+ classpath []string
+}{
+ {
+ name: "default",
+ bootclasspath: []string{"core-oj", "core-libart"},
+ system: "core-system-modules",
+ classpath: []string{"ext", "framework", "okhttp"},
+ },
+ {
+ name: "blank sdk version",
+ properties: `sdk_version: "",`,
+ bootclasspath: []string{"core-oj", "core-libart"},
+ system: "core-system-modules",
+ classpath: []string{"ext", "framework", "okhttp"},
+ },
+ {
+
+ name: "sdk v14",
+ properties: `sdk_version: "14",`,
+ bootclasspath: []string{`""`},
+ system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath
+ classpath: []string{"prebuilts/sdk/14/android.jar"},
+ },
+ {
+
+ name: "current",
+ properties: `sdk_version: "current",`,
+ bootclasspath: []string{`""`},
+ system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath
+ classpath: []string{"prebuilts/sdk/current/android.jar"},
+ },
+ {
+
+ name: "system_current",
+ properties: `sdk_version: "system_current",`,
+ bootclasspath: []string{`""`},
+ system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath
+ classpath: []string{"prebuilts/sdk/system_current/android.jar"},
+ },
+ {
+
+ name: "system_14",
+ properties: `sdk_version: "system_14",`,
+ bootclasspath: []string{`""`},
+ system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath
+ classpath: []string{"prebuilts/sdk/system_14/android.jar"},
+ },
+ {
+
+ name: "test_current",
+ properties: `sdk_version: "test_current",`,
+ bootclasspath: []string{`""`},
+ system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath
+ classpath: []string{"prebuilts/sdk/test_current/android.jar"},
+ },
+ {
+
+ name: "nostdlib",
+ properties: `no_standard_libs: true, system_modules: "none"`,
+ system: "none",
+ bootclasspath: []string{`""`},
+ classpath: []string{},
+ },
+ {
+
+ name: "nostdlib system_modules",
+ properties: `no_standard_libs: true, system_modules: "core-system-modules"`,
+ system: "core-system-modules",
+ bootclasspath: []string{`""`},
+ classpath: []string{},
+ },
+ {
+
+ name: "host default",
+ moduleType: "java_library_host",
+ properties: ``,
+ host: android.Host,
+ classpath: []string{},
+ },
+ {
+ name: "host nostdlib",
+ moduleType: "java_library_host",
+ host: android.Host,
+ properties: `no_standard_libs: true`,
+ classpath: []string{},
+ },
+ {
+
+ name: "host supported default",
+ host: android.Host,
+ properties: `host_supported: true,`,
+ classpath: []string{},
+ },
+ {
+ name: "host supported nostdlib",
+ host: android.Host,
+ properties: `host_supported: true, no_standard_libs: true, system_modules: "none"`,
+ classpath: []string{},
+ },
+}
+
+func TestClasspath(t *testing.T) {
+ for _, testcase := range classpathTestcases {
+ t.Run(testcase.name, func(t *testing.T) {
+ moduleType := "java_library"
+ if testcase.moduleType != "" {
+ moduleType = testcase.moduleType
+ }
+
+ bp := moduleType + ` {
+ name: "foo",
+ srcs: ["a.java"],
+ ` + testcase.properties + `
+ }`
+
+ variant := "android_common"
+ if testcase.host == android.Host {
+ variant = android.BuildOs.String() + "_common"
+ }
+
+ convertModulesToPaths := func(cp []string) []string {
+ ret := make([]string, len(cp))
+ for i, e := range cp {
+ ret[i] = moduleToPath(e)
+ }
+ return ret
+ }
+
+ bootclasspath := convertModulesToPaths(testcase.bootclasspath)
+ classpath := convertModulesToPaths(testcase.classpath)
+
+ bc := strings.Join(bootclasspath, ":")
+ if bc != "" {
+ bc = "-bootclasspath " + bc
+ }
+
+ c := strings.Join(classpath, ":")
+ if c != "" {
+ c = "-classpath " + c
+ }
+ system := ""
+ if testcase.system == "none" {
+ system = "--system=none"
+ } else if testcase.system != "" {
+ system = "--system=" + filepath.Join(buildDir, ".intermediates", testcase.system, "android_common", "system") + "/"
+ }
+
+ t.Run("1.8", func(t *testing.T) {
+ // Test default javac 1.8
+ ctx := testJava(t, bp)
+
+ javac := ctx.ModuleForTests("foo", variant).Rule("javac")
+
+ got := javac.Args["bootClasspath"]
+ if got != bc {
+ t.Errorf("bootclasspath expected %q != got %q", bc, got)
+ }
+
+ got = javac.Args["classpath"]
+ if got != c {
+ t.Errorf("classpath expected %q != got %q", c, got)
+ }
+
+ var deps []string
+ if len(bootclasspath) > 0 && bootclasspath[0] != `""` {
+ deps = append(deps, bootclasspath...)
+ }
+ deps = append(deps, classpath...)
+
+ if !reflect.DeepEqual(javac.Implicits.Strings(), deps) {
+ t.Errorf("implicits expected %q != got %q", deps, javac.Implicits.Strings())
+ }
+ })
+
+ // Test again with javac 1.9
+ t.Run("1.9", func(t *testing.T) {
+ config := testConfig(map[string]string{"EXPERIMENTAL_USE_OPENJDK9": "true"})
+ ctx := testContext(config, bp, nil)
+ run(t, ctx, config)
+
+ javac := ctx.ModuleForTests("foo", variant).Rule("javac")
+ got := javac.Args["bootClasspath"]
+ expected := system
+ if testcase.system == "bootclasspath" {
+ expected = bc
+ }
+ if got != expected {
+ t.Errorf("bootclasspath expected %q != got %q", expected, got)
+ }
+ })
+ })
+ }
+
}
func TestPrebuilts(t *testing.T) {
@@ -207,28 +495,27 @@
static_libs: ["baz"],
}
- java_prebuilt_library {
+ java_import {
name: "bar",
- srcs: ["a.jar"],
+ jars: ["a.jar"],
}
- java_prebuilt_library {
+ java_import {
name: "baz",
- srcs: ["b.jar"],
+ jars: ["b.jar"],
}
`)
- javac := ctx.ModuleForTests("foo", "").Rule("javac")
- jar := ctx.ModuleForTests("foo", "").Rule("jar")
+ javac := ctx.ModuleForTests("foo", "android_common").Rule("javac")
+ combineJar := ctx.ModuleForTests("foo", "android_common").Description("for javac")
bar := "a.jar"
if !strings.Contains(javac.Args["classpath"], bar) {
t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], bar)
}
- baz := filepath.Join(buildDir, ".intermediates", "baz", "extracted", "classes.list")
- if !strings.Contains(jar.Args["jarArgs"], baz) {
- t.Errorf("foo jarArgs %v does not contain %q", jar.Args["jarArgs"], baz)
+ if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != "b.jar" {
+ t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, "b.jar")
}
}
@@ -257,25 +544,328 @@
}
`)
- javac := ctx.ModuleForTests("foo", "").Rule("javac")
- jar := ctx.ModuleForTests("foo", "").Rule("jar")
+ javac := ctx.ModuleForTests("foo", "android_common").Rule("javac")
+ combineJar := ctx.ModuleForTests("foo", "android_common").Description("for javac")
if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" {
t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs)
}
- bar := filepath.Join(buildDir, ".intermediates", "bar", "classes-full-debug.jar")
- if !strings.Contains(javac.Args["classpath"], bar) {
- t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], bar)
+ barTurbine := filepath.Join(buildDir, ".intermediates", "bar", "android_common", "turbine-combined", "bar.jar")
+ if !strings.Contains(javac.Args["classpath"], barTurbine) {
+ t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], barTurbine)
}
- baz := filepath.Join(buildDir, ".intermediates", "baz", "classes.list")
- if !strings.Contains(jar.Args["jarArgs"], baz) {
- t.Errorf("foo jarArgs %v does not contain %q", jar.Args["jarArgs"], baz)
+ baz := ctx.ModuleForTests("baz", "android_common").Rule("javac").Output.String()
+ if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != baz {
+ t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, baz)
+ }
+}
+
+func TestResources(t *testing.T) {
+ var table = []struct {
+ name string
+ prop string
+ extra string
+ args string
+ }{
+ {
+ // Test that a module with java_resource_dirs includes the files
+ name: "resource dirs",
+ prop: `java_resource_dirs: ["java-res"]`,
+ args: "-C java-res -f java-res/a -f java-res/b",
+ },
+ {
+ // Test that a module with java_resources includes the files
+ name: "resource files",
+ prop: `java_resources: ["java-res/a", "java-res/b"]`,
+ args: "-C . -f java-res/a -f java-res/b",
+ },
+ {
+ // Test that a module with a filegroup in java_resources includes the files with the
+ // path prefix
+ name: "resource filegroup",
+ prop: `java_resources: [":foo-res"]`,
+ extra: `
+ filegroup {
+ name: "foo-res",
+ path: "java-res",
+ srcs: ["java-res/a", "java-res/b"],
+ }`,
+ args: "-C java-res -f java-res/a -f java-res/b",
+ },
+ {
+ // Test that a module with "include_srcs: true" includes its source files in the resources jar
+ name: "include sources",
+ prop: `include_srcs: true`,
+ args: "-C . -f a.java -f b.java -f c.java",
+ },
+ }
+
+ for _, test := range table {
+ t.Run(test.name, func(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: [
+ "a.java",
+ "b.java",
+ "c.java",
+ ],
+ `+test.prop+`,
+ }
+ `+test.extra)
+
+ foo := ctx.ModuleForTests("foo", "android_common").Output("combined/foo.jar")
+ fooRes := ctx.ModuleForTests("foo", "android_common").Output("res/foo.jar")
+
+ if !inList(fooRes.Output.String(), foo.Inputs.Strings()) {
+ t.Errorf("foo combined jars %v does not contain %q",
+ foo.Inputs.Strings(), fooRes.Output.String())
+ }
+
+ if fooRes.Args["jarArgs"] != test.args {
+ t.Errorf("foo resource jar args %q is not %q",
+ fooRes.Args["jarArgs"], test.args)
+ }
+ })
+ }
+}
+
+func TestExcludeResources(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: ["a.java"],
+ java_resource_dirs: ["java-res", "java-res2"],
+ exclude_java_resource_dirs: ["java-res2"],
+ }
+
+ java_library {
+ name: "bar",
+ srcs: ["a.java"],
+ java_resources: ["java-res/*"],
+ exclude_java_resources: ["java-res/b"],
+ }
+ `)
+
+ fooRes := ctx.ModuleForTests("foo", "android_common").Output("res/foo.jar")
+
+ expected := "-C java-res -f java-res/a -f java-res/b"
+ if fooRes.Args["jarArgs"] != expected {
+ t.Errorf("foo resource jar args %q is not %q",
+ fooRes.Args["jarArgs"], expected)
+
+ }
+
+ barRes := ctx.ModuleForTests("bar", "android_common").Output("res/bar.jar")
+
+ expected = "-C . -f java-res/a"
+ if barRes.Args["jarArgs"] != expected {
+ t.Errorf("bar resource jar args %q is not %q",
+ barRes.Args["jarArgs"], expected)
+
+ }
+}
+
+func TestGeneratedSources(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: [
+ "a*.java",
+ ":gen",
+ "b*.java",
+ ],
+ }
+
+ genrule {
+ name: "gen",
+ tool_files: ["java-res/a"],
+ out: ["gen.java"],
+ }
+ `)
+
+ javac := ctx.ModuleForTests("foo", "android_common").Rule("javac")
+ genrule := ctx.ModuleForTests("gen", "").Rule("generator")
+
+ if filepath.Base(genrule.Output.String()) != "gen.java" {
+ t.Fatalf(`gen output file %v is not ".../gen.java"`, genrule.Output.String())
+ }
+
+ if len(javac.Inputs) != 3 ||
+ javac.Inputs[0].String() != "a.java" ||
+ javac.Inputs[1].String() != genrule.Output.String() ||
+ javac.Inputs[2].String() != "b.java" {
+ t.Errorf(`foo inputs %v != ["a.java", ".../gen.java", "b.java"]`, javac.Inputs)
+ }
+}
+
+func TestKotlin(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: ["a.java", "b.kt"],
+ }
+
+ java_library {
+ name: "bar",
+ srcs: ["b.kt"],
+ }
+ `)
+
+ kotlinc := ctx.ModuleForTests("foo", "android_common").Rule("kotlinc")
+ javac := ctx.ModuleForTests("foo", "android_common").Rule("javac")
+ jar := ctx.ModuleForTests("foo", "android_common").Output("combined/foo.jar")
+
+ if len(kotlinc.Inputs) != 2 || kotlinc.Inputs[0].String() != "a.java" ||
+ kotlinc.Inputs[1].String() != "b.kt" {
+ t.Errorf(`foo kotlinc inputs %v != ["a.java", "b.kt"]`, kotlinc.Inputs)
+ }
+
+ if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" {
+ t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs)
+ }
+
+ if !strings.Contains(javac.Args["classpath"], kotlinc.Output.String()) {
+ t.Errorf("foo classpath %v does not contain %q",
+ javac.Args["classpath"], kotlinc.Output.String())
+ }
+
+ if !inList(kotlinc.Output.String(), jar.Inputs.Strings()) {
+ t.Errorf("foo jar inputs %v does not contain %q",
+ jar.Inputs.Strings(), kotlinc.Output.String())
+ }
+
+ kotlinc = ctx.ModuleForTests("bar", "android_common").Rule("kotlinc")
+ jar = ctx.ModuleForTests("bar", "android_common").Output("combined/bar.jar")
+
+ if len(kotlinc.Inputs) != 1 || kotlinc.Inputs[0].String() != "b.kt" {
+ t.Errorf(`bar kotlinc inputs %v != ["b.kt"]`, kotlinc.Inputs)
+ }
+}
+
+func TestTurbine(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: ["a.java"],
+ }
+
+ java_library {
+ name: "bar",
+ srcs: ["b.java"],
+ static_libs: ["foo"],
+ }
+
+ java_library {
+ name: "baz",
+ srcs: ["c.java"],
+ libs: ["bar"],
+ sdk_version: "14",
+ }
+ `)
+
+ fooTurbine := ctx.ModuleForTests("foo", "android_common").Rule("turbine")
+ barTurbine := ctx.ModuleForTests("bar", "android_common").Rule("turbine")
+ barJavac := ctx.ModuleForTests("bar", "android_common").Rule("javac")
+ barTurbineCombined := ctx.ModuleForTests("bar", "android_common").Description("for turbine")
+ bazJavac := ctx.ModuleForTests("baz", "android_common").Rule("javac")
+
+ if len(fooTurbine.Inputs) != 1 || fooTurbine.Inputs[0].String() != "a.java" {
+ t.Errorf(`foo inputs %v != ["a.java"]`, fooTurbine.Inputs)
+ }
+
+ fooHeaderJar := filepath.Join(buildDir, ".intermediates", "foo", "android_common", "turbine-combined", "foo.jar")
+ if !strings.Contains(barTurbine.Args["classpath"], fooHeaderJar) {
+ t.Errorf("bar turbine classpath %v does not contain %q", barTurbine.Args["classpath"], fooHeaderJar)
+ }
+ if !strings.Contains(barJavac.Args["classpath"], fooHeaderJar) {
+ t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], fooHeaderJar)
+ }
+ if len(barTurbineCombined.Inputs) != 2 || barTurbineCombined.Inputs[1].String() != fooHeaderJar {
+ t.Errorf("bar turbine combineJar inputs %v does not contain %q", barTurbineCombined.Inputs, fooHeaderJar)
+ }
+ if !strings.Contains(bazJavac.Args["classpath"], "prebuilts/sdk/14/android.jar") {
+ t.Errorf("baz javac classpath %v does not contain %q", bazJavac.Args["classpath"],
+ "prebuilts/sdk/14/android.jar")
+ }
+}
+
+func TestSharding(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "bar",
+ srcs: ["a.java","b.java","c.java"],
+ javac_shard_size: 1
+ }
+ `)
+
+ barHeaderJar := filepath.Join(buildDir, ".intermediates", "bar", "android_common", "turbine-combined", "bar.jar")
+ for i := 0; i < 3; i++ {
+ barJavac := ctx.ModuleForTests("bar", "android_common").Description("javac" + strconv.Itoa(i))
+ if !strings.Contains(barJavac.Args["classpath"], barHeaderJar) {
+ t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], barHeaderJar)
+ }
+ }
+}
+
+func TestJarGenrules(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: ["a.java"],
+ }
+
+ java_genrule {
+ name: "jargen",
+ tool_files: ["b.java"],
+ cmd: "$(location b.java) $(in) $(out)",
+ out: ["jargen.jar"],
+ srcs: [":foo"],
+ }
+
+ java_library {
+ name: "bar",
+ static_libs: ["jargen"],
+ srcs: ["c.java"],
+ }
+
+ java_library {
+ name: "baz",
+ libs: ["jargen"],
+ srcs: ["c.java"],
+ }
+ `)
+
+ foo := ctx.ModuleForTests("foo", "android_common").Output("javac/foo.jar")
+ jargen := ctx.ModuleForTests("jargen", "android_common").Output("jargen.jar")
+ bar := ctx.ModuleForTests("bar", "android_common").Output("javac/bar.jar")
+ baz := ctx.ModuleForTests("baz", "android_common").Output("javac/baz.jar")
+ barCombined := ctx.ModuleForTests("bar", "android_common").Output("combined/bar.jar")
+
+ if len(jargen.Inputs) != 1 || jargen.Inputs[0].String() != foo.Output.String() {
+ t.Errorf("expected jargen inputs [%q], got %q", foo.Output.String(), jargen.Inputs.Strings())
+ }
+
+ if !strings.Contains(bar.Args["classpath"], jargen.Output.String()) {
+ t.Errorf("bar classpath %v does not contain %q", bar.Args["classpath"], jargen.Output.String())
+ }
+
+ if !strings.Contains(baz.Args["classpath"], jargen.Output.String()) {
+ t.Errorf("baz classpath %v does not contain %q", baz.Args["classpath"], jargen.Output.String())
+ }
+
+ if len(barCombined.Inputs) != 2 ||
+ barCombined.Inputs[0].String() != bar.Output.String() ||
+ barCombined.Inputs[1].String() != jargen.Output.String() {
+ t.Errorf("bar combined jar inputs %v is not [%q, %q]",
+ barCombined.Inputs.Strings(), bar.Output.String(), jargen.Output.String())
}
}
func fail(t *testing.T, errs []error) {
+ t.Helper()
if len(errs) > 0 {
for _, err := range errs {
t.Error(err)
diff --git a/java/proto.go b/java/proto.go
new file mode 100644
index 0000000..17f02a3
--- /dev/null
+++ b/java/proto.go
@@ -0,0 +1,96 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+ "strings"
+
+ "github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
+
+ "android/soong/android"
+)
+
+func init() {
+ pctx.HostBinToolVariable("protocCmd", "aprotoc")
+}
+
+var (
+ proto = pctx.AndroidStaticRule("protoc",
+ blueprint.RuleParams{
+ Command: `rm -rf $outDir && mkdir -p $outDir && ` +
+ `$protocCmd $protoOut=$protoOutFlags:$outDir $protoFlags $in && ` +
+ `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
+ CommandDeps: []string{
+ "$protocCmd",
+ "${config.SoongZipCmd}",
+ },
+ }, "protoFlags", "protoOut", "protoOutFlags", "outDir")
+)
+
+func genProto(ctx android.ModuleContext, outputSrcJar android.WritablePath,
+ protoFiles android.Paths, protoFlags []string, protoOut, protoOutFlags string) {
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: proto,
+ Description: "protoc " + protoFiles[0].Rel(),
+ Output: outputSrcJar,
+ Inputs: protoFiles,
+ Args: map[string]string{
+ "outDir": android.ProtoDir(ctx).String(),
+ "protoOut": protoOut,
+ "protoOutFlags": protoOutFlags,
+ "protoFlags": strings.Join(protoFlags, " "),
+ },
+ })
+}
+
+func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) {
+ switch proptools.String(p.Proto.Type) {
+ case "micro":
+ ctx.AddDependency(ctx.Module(), staticLibTag, "libprotobuf-java-micro")
+ case "nano":
+ ctx.AddDependency(ctx.Module(), staticLibTag, "libprotobuf-java-nano")
+ case "lite", "":
+ ctx.AddDependency(ctx.Module(), staticLibTag, "libprotobuf-java-lite")
+ case "full":
+ if ctx.Host() {
+ ctx.AddDependency(ctx.Module(), staticLibTag, "libprotobuf-java-full")
+ } else {
+ ctx.PropertyErrorf("proto.type", "full java protos only supported on the host")
+ }
+ default:
+ ctx.PropertyErrorf("proto.type", "unknown proto type %q",
+ proptools.String(p.Proto.Type))
+ }
+}
+
+func protoFlags(ctx android.ModuleContext, p *android.ProtoProperties, flags javaBuilderFlags) javaBuilderFlags {
+ switch proptools.String(p.Proto.Type) {
+ case "micro":
+ flags.protoOutFlag = "--javamicro_out"
+ case "nano":
+ flags.protoOutFlag = "--javanano_out"
+ case "lite", "full", "":
+ flags.protoOutFlag = "--java_out"
+ default:
+ ctx.PropertyErrorf("proto.type", "unknown proto type %q",
+ proptools.String(p.Proto.Type))
+ }
+
+ flags.protoFlags = android.ProtoFlags(ctx, p)
+
+ return flags
+}
diff --git a/java/resources.go b/java/resources.go
index 60dc934..a596fd7 100644
--- a/java/resources.go
+++ b/java/resources.go
@@ -15,9 +15,9 @@
package java
import (
+ "fmt"
"path/filepath"
-
- "github.com/google/blueprint/bootstrap"
+ "strings"
"android/soong/android"
)
@@ -31,41 +31,76 @@
"**/*~",
}
-func isStringInSlice(str string, slice []string) bool {
- for _, s := range slice {
- if s == str {
- return true
- }
- }
- return false
-}
-
-func ResourceDirsToJarSpecs(ctx android.ModuleContext, resourceDirs, excludeDirs []string) []jarSpec {
+func ResourceDirsToJarArgs(ctx android.ModuleContext,
+ resourceDirs, excludeDirs []string) (args []string, deps android.Paths) {
var excludes []string
for _, exclude := range excludeDirs {
- excludes = append(excludes, android.PathForModuleSrc(ctx, exclude, "**/*").String())
+ excludes = append(excludes,
+ filepath.Join(android.PathForModuleSrc(ctx, exclude).String(), "**/*"))
}
excludes = append(excludes, resourceExcludes...)
- var jarSpecs []jarSpec
+ for _, dir := range resourceDirs {
+ dir := android.PathForModuleSrc(ctx, dir).String()
+ files := ctx.Glob(filepath.Join(dir, "**/*"), excludes)
- for _, resourceDir := range resourceDirs {
- if isStringInSlice(resourceDir, excludeDirs) {
- continue
- }
- resourceDir := android.PathForModuleSrc(ctx, resourceDir)
- dirs := ctx.Glob(resourceDir.String(), nil)
- for _, dir := range dirs {
- fileListFile := android.ResPathWithName(ctx, dir, "resources.list")
- depFile := fileListFile.String() + ".d"
+ deps = append(deps, files...)
- pattern := filepath.Join(dir.String(), "**/*")
- bootstrap.GlobFile(ctx, pattern, excludes, fileListFile.String(), depFile)
- jarSpecs = append(jarSpecs, jarSpec{fileListFile, dir})
+ if len(files) > 0 {
+ args = append(args, "-C", dir)
+
+ for _, f := range files {
+ path := f.String()
+ if !strings.HasPrefix(path, dir) {
+ panic(fmt.Errorf("path %q does not start with %q", path, dir))
+ }
+ args = append(args, "-f", path)
+ }
}
}
- return jarSpecs
+ return args, deps
+}
+
+// Convert java_resources properties to arguments to soong_zip -jar, ignoring common patterns
+// that should not be treated as resources (including *.java).
+func ResourceFilesToJarArgs(ctx android.ModuleContext,
+ res, exclude []string) (args []string, deps android.Paths) {
+
+ exclude = append([]string(nil), exclude...)
+ exclude = append(exclude, resourceExcludes...)
+ return resourceFilesToJarArgs(ctx, res, exclude)
+}
+
+// Convert java_resources properties to arguments to soong_zip -jar, keeping files that should
+// normally not used as resources like *.java
+func SourceFilesToJarArgs(ctx android.ModuleContext,
+ res, exclude []string) (args []string, deps android.Paths) {
+
+ return resourceFilesToJarArgs(ctx, res, exclude)
+}
+
+func resourceFilesToJarArgs(ctx android.ModuleContext,
+ res, exclude []string) (args []string, deps android.Paths) {
+
+ files := ctx.ExpandSources(res, exclude)
+
+ lastDir := ""
+ for i, f := range files {
+ rel := f.Rel()
+ path := f.String()
+ if !strings.HasSuffix(path, rel) {
+ panic(fmt.Errorf("path %q does not end with %q", path, rel))
+ }
+ dir := filepath.Clean(strings.TrimSuffix(path, rel))
+ if i == 0 || dir != lastDir {
+ args = append(args, "-C", dir)
+ }
+ args = append(args, "-f", path)
+ lastDir = dir
+ }
+
+ return args, files
}
diff --git a/java/system_modules.go b/java/system_modules.go
new file mode 100644
index 0000000..5234d17
--- /dev/null
+++ b/java/system_modules.go
@@ -0,0 +1,145 @@
+// Copyright 2017 Google Inc. All rights reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+ "fmt"
+ "io"
+ "strings"
+
+ "github.com/google/blueprint"
+
+ "android/soong/android"
+)
+
+// OpenJDK 9 introduces the concept of "system modules", which replace the bootclasspath. This
+// file will produce the rules necessary to convert each unique set of bootclasspath jars into
+// system modules in a runtime image using the jmod and jlink tools.
+
+func init() {
+ android.RegisterModuleType("java_system_modules", SystemModulesFactory)
+
+ pctx.SourcePathVariable("moduleInfoJavaPath", "build/soong/scripts/jars-to-module-info-java.sh")
+}
+
+var (
+ jarsTosystemModules = pctx.AndroidStaticRule("jarsTosystemModules", blueprint.RuleParams{
+ Command: `rm -rf ${outDir} ${workDir} && mkdir -p ${workDir}/jmod && ` +
+ `${moduleInfoJavaPath} ${moduleName} $in > ${workDir}/module-info.java && ` +
+ `${config.JavacCmd} --system=none --patch-module=java.base=${classpath} ${workDir}/module-info.java && ` +
+ `${config.SoongZipCmd} -jar -o ${workDir}/classes.jar -C ${workDir} -f ${workDir}/module-info.class && ` +
+ `${config.MergeZipsCmd} -j ${workDir}/module.jar ${workDir}/classes.jar $in && ` +
+ `${config.JmodCmd} create --module-version 9 --target-platform android ` +
+ ` --class-path ${workDir}/module.jar ${workDir}/jmod/${moduleName}.jmod && ` +
+ `${config.JlinkCmd} --module-path ${workDir}/jmod --add-modules ${moduleName} --output ${outDir} && ` +
+ `cp ${config.JrtFsJar} ${outDir}/lib/`,
+ CommandDeps: []string{
+ "${moduleInfoJavaPath}",
+ "${config.JavacCmd}",
+ "${config.SoongZipCmd}",
+ "${config.MergeZipsCmd}",
+ "${config.JmodCmd}",
+ "${config.JlinkCmd}",
+ "${config.JrtFsJar}",
+ },
+ },
+ "moduleName", "classpath", "outDir", "workDir")
+)
+
+func TransformJarsToSystemModules(ctx android.ModuleContext, moduleName string, jars android.Paths) android.WritablePath {
+ outDir := android.PathForModuleOut(ctx, "system")
+ workDir := android.PathForModuleOut(ctx, "modules")
+ outputFile := android.PathForModuleOut(ctx, "system/lib/modules")
+ outputs := android.WritablePaths{
+ outputFile,
+ android.PathForModuleOut(ctx, "system/lib/jrt-fs.jar"),
+ android.PathForModuleOut(ctx, "system/release"),
+ }
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: jarsTosystemModules,
+ Description: "system modules",
+ Outputs: outputs,
+ Inputs: jars,
+ Args: map[string]string{
+ "moduleName": moduleName,
+ "classpath": strings.Join(jars.Strings(), ":"),
+ "workDir": workDir.String(),
+ "outDir": outDir.String(),
+ },
+ })
+
+ return outputFile
+}
+
+func SystemModulesFactory() android.Module {
+ module := &SystemModules{}
+ module.AddProperties(&module.properties)
+ android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon)
+ return module
+}
+
+type SystemModules struct {
+ android.ModuleBase
+
+ properties SystemModulesProperties
+
+ outputFile android.Path
+}
+
+type SystemModulesProperties struct {
+ // List of java library modules that should be included in the system modules
+ Libs []string
+
+ // List of prebuilt jars that should be included in the system modules
+ Jars []string
+
+ // Sdk version that should be included in the system modules
+ Sdk_version *string
+}
+
+func (system *SystemModules) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ var jars android.Paths
+
+ ctx.VisitDirectDeps(func(module android.Module) {
+ if ctx.OtherModuleDependencyTag(module) == libTag {
+ dep, _ := module.(Dependency)
+ jars = append(jars, dep.HeaderJars()...)
+ }
+ })
+
+ jars = append(jars, android.PathsForModuleSrc(ctx, system.properties.Jars)...)
+
+ if ctx.Config().TargetOpenJDK9() {
+ system.outputFile = TransformJarsToSystemModules(ctx, "java.base", jars)
+ }
+}
+
+func (system *SystemModules) DepsMutator(ctx android.BottomUpMutatorContext) {
+ ctx.AddDependency(ctx.Module(), libTag, system.properties.Libs...)
+}
+
+func (system *SystemModules) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ if system.outputFile != nil {
+ makevar := "SOONG_SYSTEM_MODULES_" + name
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, makevar, ":=", system.outputFile.String())
+ fmt.Fprintln(w, ".KATI_READONLY", ":=", makevar)
+ fmt.Fprintln(w, name+":", "$("+makevar+")")
+ }
+ },
+ }
+}
diff --git a/phony/phony.go b/phony/phony.go
index 79f3a1b..a39b5d5 100644
--- a/phony/phony.go
+++ b/phony/phony.go
@@ -48,15 +48,14 @@
}
}
-func (p *phony) AndroidMk() (ret android.AndroidMkData, err error) {
- ret.Custom = func(w io.Writer, name, prefix, moduleDir string) error {
- fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)")
- fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
- fmt.Fprintln(w, "LOCAL_MODULE :=", name)
- fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(p.requiredModuleNames, " "))
- fmt.Fprintln(w, "include $(BUILD_PHONY_PACKAGE)")
-
- return nil
+func (p *phony) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)")
+ fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
+ fmt.Fprintln(w, "LOCAL_MODULE :=", name)
+ fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(p.requiredModuleNames, " "))
+ fmt.Fprintln(w, "include $(BUILD_PHONY_PACKAGE)")
+ },
}
- return
}
diff --git a/python/androidmk.go b/python/androidmk.go
index 0a79e73..5fa01ab 100644
--- a/python/androidmk.go
+++ b/python/androidmk.go
@@ -23,10 +23,10 @@
)
type subAndroidMkProvider interface {
- AndroidMk(*pythonBaseModule, *android.AndroidMkData)
+ AndroidMk(*Module, *android.AndroidMkData)
}
-func (p *pythonBaseModule) subAndroidMk(data *android.AndroidMkData, obj interface{}) {
+func (p *Module) subAndroidMk(data *android.AndroidMkData, obj interface{}) {
if p.subAndroidMkOnce == nil {
p.subAndroidMkOnce = make(map[subAndroidMkProvider]bool)
}
@@ -38,30 +38,46 @@
}
}
-func (p *pythonBaseModule) AndroidMk() (ret android.AndroidMkData, err error) {
+func (p *Module) AndroidMk() android.AndroidMkData {
+ ret := android.AndroidMkData{OutputFile: p.installSource}
+
p.subAndroidMk(&ret, p.installer)
- return ret, nil
+ return ret
}
-func (p *pythonBinaryHostDecorator) AndroidMk(base *pythonBaseModule, ret *android.AndroidMkData) {
+func (p *binaryDecorator) AndroidMk(base *Module, ret *android.AndroidMkData) {
ret.Class = "EXECUTABLES"
- base.subAndroidMk(ret, p.pythonDecorator.baseInstaller)
+
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
+ if len(p.binaryProperties.Test_suites) > 0 {
+ fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=",
+ strings.Join(p.binaryProperties.Test_suites, " "))
+ }
+ })
+ base.subAndroidMk(ret, p.pythonInstaller)
}
-func (p *pythonTestHostDecorator) AndroidMk(base *pythonBaseModule, ret *android.AndroidMkData) {
+func (p *testDecorator) AndroidMk(base *Module, ret *android.AndroidMkData) {
ret.Class = "NATIVE_TESTS"
- base.subAndroidMk(ret, p.pythonDecorator.baseInstaller)
+
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
+ if len(p.binaryDecorator.binaryProperties.Test_suites) > 0 {
+ fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=",
+ strings.Join(p.binaryDecorator.binaryProperties.Test_suites, " "))
+ }
+ })
+ base.subAndroidMk(ret, p.binaryDecorator.pythonInstaller)
}
-func (installer *pythonInstaller) AndroidMk(base *pythonBaseModule, ret *android.AndroidMkData) {
+func (installer *pythonInstaller) AndroidMk(base *Module, ret *android.AndroidMkData) {
// Soong installation is only supported for host modules. Have Make
// installation trigger Soong installation.
if base.Target().Os.Class == android.Host {
ret.OutputFile = android.OptionalPathForPath(installer.path)
}
- ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
+ ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) {
path := installer.path.RelPathString()
dir, file := filepath.Split(path)
stem := strings.TrimSuffix(file, filepath.Ext(file))
@@ -69,6 +85,5 @@
fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+filepath.Ext(file))
fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir))
fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem)
- return nil
})
}
diff --git a/python/binary.go b/python/binary.go
index ae2693b..457c7fa 100644
--- a/python/binary.go
+++ b/python/binary.go
@@ -28,67 +28,63 @@
android.RegisterModuleType("python_binary_host", PythonBinaryHostFactory)
}
-type PythonBinaryBaseProperties struct {
+type BinaryProperties struct {
// the name of the source file that is the main entry point of the program.
// this file must also be listed in srcs.
// If left unspecified, module name is used instead.
// If name doesn’t match any filename in srcs, main must be specified.
- Main string
+ Main *string `android:"arch_variant"`
// set the name of the output binary.
- Stem string
+ Stem *string `android:"arch_variant"`
// append to the name of the output binary.
- Suffix string
+ Suffix *string `android:"arch_variant"`
+
+ // list of compatibility suites (for example "cts", "vts") that the module should be
+ // installed into.
+ Test_suites []string `android:"arch_variant"`
}
-type pythonBinaryBase struct {
- pythonBaseModule
+type binaryDecorator struct {
+ binaryProperties BinaryProperties
- binaryProperties PythonBinaryBaseProperties
-
- // soong_zip arguments from all its dependencies.
- depsParSpecs []parSpec
-
- // Python runfiles paths from all its dependencies.
- depsPyRunfiles []string
+ *pythonInstaller
}
-type PythonBinaryHost struct {
- pythonBinaryBase
-}
-
-var _ PythonSubModule = (*PythonBinaryHost)(nil)
-
-type pythonBinaryHostDecorator struct {
- pythonDecorator
-}
-
-func (p *pythonBinaryHostDecorator) install(ctx android.ModuleContext, file android.Path) {
- p.pythonDecorator.baseInstaller.install(ctx, file)
+type IntermPathProvider interface {
+ IntermPathForModuleOut() android.OptionalPath
}
var (
stubTemplateHost = "build/soong/python/scripts/stub_template_host.txt"
)
-func PythonBinaryHostFactory() android.Module {
- decorator := &pythonBinaryHostDecorator{
- pythonDecorator: pythonDecorator{baseInstaller: NewPythonInstaller("bin")}}
+func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) {
+ module := newModule(hod, android.MultilibFirst)
+ decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "")}
- module := &PythonBinaryHost{}
- module.pythonBaseModule.installer = decorator
- module.AddProperties(&module.binaryProperties)
+ module.bootstrapper = decorator
+ module.installer = decorator
- return InitPythonBaseModule(&module.pythonBinaryBase.pythonBaseModule,
- &module.pythonBinaryBase, android.HostSupportedNoCross)
+ return module, decorator
}
-func (p *pythonBinaryBase) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath {
- p.pythonBaseModule.GeneratePythonBuildActions(ctx)
+func PythonBinaryHostFactory() android.Module {
+ module, _ := NewBinary(android.HostSupportedNoCross)
- // no Python source file for compiling par file.
- if len(p.pythonBaseModule.srcsPathMappings) == 0 && len(p.depsPyRunfiles) == 0 {
+ return module.Init()
+}
+
+func (binary *binaryDecorator) bootstrapperProps() []interface{} {
+ return []interface{}{&binary.binaryProperties}
+}
+
+func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actual_version string,
+ embedded_launcher bool, srcsPathMappings []pathMapping, parSpec parSpec,
+ depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath {
+ // no Python source file for compiling .par file.
+ if len(srcsPathMappings) == 0 {
return android.OptionalPath{}
}
@@ -100,10 +96,10 @@
existingPyPkgSet := make(map[string]bool)
wholePyRunfiles := []string{}
- for _, path := range p.pythonBaseModule.srcsPathMappings {
+ for _, path := range srcsPathMappings {
wholePyRunfiles = append(wholePyRunfiles, path.dest)
}
- wholePyRunfiles = append(wholePyRunfiles, p.depsPyRunfiles...)
+ wholePyRunfiles = append(wholePyRunfiles, depsPyRunfiles...)
// find all the runfiles dirs which have been treated as packages.
for _, path := range wholePyRunfiles {
@@ -130,50 +126,62 @@
populateNewPyPkgs(parentPath, existingPyPkgSet, newPyPkgSet, &newPyPkgs)
}
- main := p.getPyMainFile(ctx)
+ main := binary.getPyMainFile(ctx, srcsPathMappings)
if main == "" {
return android.OptionalPath{}
}
- interp := p.getInterpreter(ctx)
- if interp == "" {
- return android.OptionalPath{}
+
+ var launcher_path android.Path
+ if embedded_launcher {
+ ctx.VisitDirectDeps(func(m android.Module) {
+ if ctx.OtherModuleDependencyTag(m) != launcherTag {
+ return
+ }
+ if provider, ok := m.(IntermPathProvider); ok {
+ if launcher_path != nil {
+ panic(fmt.Errorf("launcher path was found before: %q",
+ launcher_path))
+ }
+ launcher_path = provider.IntermPathForModuleOut().Path()
+ }
+ })
}
- // we need remove "runfiles/" suffix since stub script starts
- // searching for main file in each sub-dir of "runfiles" directory tree.
- binFile := registerBuildActionForParFile(ctx, p.getInterpreter(ctx),
- strings.TrimPrefix(main, runFiles+"/"), p.getStem(ctx),
- newPyPkgs, append(p.depsParSpecs, p.pythonBaseModule.parSpec))
+ binFile := registerBuildActionForParFile(ctx, embedded_launcher, launcher_path,
+ binary.getHostInterpreterName(ctx, actual_version),
+ main, binary.getStem(ctx), newPyPkgs, append(depsParSpecs, parSpec))
return android.OptionalPathForPath(binFile)
}
-// get interpreter path.
-func (p *pythonBinaryBase) getInterpreter(ctx android.ModuleContext) string {
+// get host interpreter name.
+func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext,
+ actual_version string) string {
var interp string
- switch p.pythonBaseModule.properties.ActualVersion {
+ switch actual_version {
case pyVersion2:
- interp = "python2"
+ interp = "python2.7"
case pyVersion3:
interp = "python3"
default:
panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
- p.properties.ActualVersion, ctx.ModuleName()))
+ actual_version, ctx.ModuleName()))
}
return interp
}
// find main program path within runfiles tree.
-func (p *pythonBinaryBase) getPyMainFile(ctx android.ModuleContext) string {
+func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext,
+ srcsPathMappings []pathMapping) string {
var main string
- if p.binaryProperties.Main == "" {
- main = p.BaseModuleName() + pyExt
+ if String(binary.binaryProperties.Main) == "" {
+ main = ctx.ModuleName() + pyExt
} else {
- main = p.binaryProperties.Main
+ main = String(binary.binaryProperties.Main)
}
- for _, path := range p.pythonBaseModule.srcsPathMappings {
+ for _, path := range srcsPathMappings {
if main == path.src.Rel() {
return path.dest
}
@@ -183,13 +191,13 @@
return ""
}
-func (p *pythonBinaryBase) getStem(ctx android.ModuleContext) string {
+func (binary *binaryDecorator) getStem(ctx android.ModuleContext) string {
stem := ctx.ModuleName()
- if p.binaryProperties.Stem != "" {
- stem = p.binaryProperties.Stem
+ if String(binary.binaryProperties.Stem) != "" {
+ stem = String(binary.binaryProperties.Stem)
}
- return stem + p.binaryProperties.Suffix
+ return stem + String(binary.binaryProperties.Suffix)
}
// Sets the given directory and all its ancestor directories as Python packages.
diff --git a/python/builder.go b/python/builder.go
index b823fcb..f194354 100644
--- a/python/builder.go
+++ b/python/builder.go
@@ -17,6 +17,7 @@
// This file contains Ninja build actions for building Python program.
import (
+ "fmt"
"strings"
"android/soong/android"
@@ -28,7 +29,7 @@
var (
pctx = android.NewPackageContext("android/soong/python")
- par = pctx.AndroidStaticRule("par",
+ host_par = pctx.AndroidStaticRule("host_par",
blueprint.RuleParams{
Command: `touch $initFile && ` +
`sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` +
@@ -37,6 +38,16 @@
CommandDeps: []string{"$parCmd", "$template"},
},
"initFile", "interp", "main", "template", "stub", "parCmd", "parFile", "parArgs")
+
+ embedded_par = pctx.AndroidStaticRule("embedded_par",
+ blueprint.RuleParams{
+ Command: `touch $initFile && ` +
+ `echo '$main' > $entry_point && ` +
+ `$parCmd -o $parFile $parArgs && cat $launcher | cat - $parFile > $out && ` +
+ `chmod +x $out && (rm -f $initFile; rm -f $entry_point; rm -f $parFile)`,
+ CommandDeps: []string{"$parCmd"},
+ },
+ "initFile", "main", "entry_point", "parCmd", "parFile", "parArgs", "launcher")
)
func init() {
@@ -58,10 +69,10 @@
}
func (p parSpec) soongParArgs() string {
- ret := "-P " + p.rootPrefix
+ ret := `-P ` + p.rootPrefix
for _, spec := range p.fileListSpecs {
- ret += " -C " + spec.relativeRoot + " -l " + spec.fileList.String()
+ ret += ` -C ` + spec.relativeRoot + ` -l ` + spec.fileList.String()
}
return ret
@@ -76,7 +87,7 @@
content = append(content, file.String())
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ ctx.Build(pctx, android.BuildParams{
Rule: android.WriteFile,
Description: "generate " + fileList.Rel(),
Output: fileList,
@@ -89,22 +100,17 @@
return fileList
}
-func registerBuildActionForParFile(ctx android.ModuleContext,
- interpreter, main, binName string, newPyPkgs []string, parSpecs []parSpec) android.Path {
+func registerBuildActionForParFile(ctx android.ModuleContext, embedded_launcher bool,
+ launcher_path android.Path, interpreter, main, binName string,
+ newPyPkgs []string, parSpecs []parSpec) android.Path {
- // intermediate output path for __init__.py
+ // .intermediate output path for __init__.py
initFile := android.PathForModuleOut(ctx, initFileName).String()
- // the path of stub_template_host.txt from source tree.
- template := android.PathForSource(ctx, stubTemplateHost)
-
- // intermediate output path for __main__.py
- stub := android.PathForModuleOut(ctx, mainFileName).String()
-
- // intermediate output path for par file.
+ // .intermediate output path for par file.
parFile := android.PathForModuleOut(ctx, binName+parFileExt)
- // intermediate output path for bin executable.
+ // .intermediate output path for bin executable.
binFile := android.PathForModuleOut(ctx, binName)
// implicit dependency for parFile build action.
@@ -116,32 +122,68 @@
}
parArgs := []string{}
- parArgs = append(parArgs, "-C "+strings.TrimSuffix(stub, mainFileName)+" -f "+stub)
- parArgs = append(parArgs, "-C "+strings.TrimSuffix(initFile, initFileName)+" -f "+initFile)
+ parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(initFile, initFileName)+` -f `+initFile)
for _, pkg := range newPyPkgs {
- parArgs = append(parArgs, "-P "+pkg+" -f "+initFile)
+ parArgs = append(parArgs, `-P `+pkg+` -f `+initFile)
}
for _, p := range parSpecs {
parArgs = append(parArgs, p.soongParArgs())
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: par,
- Description: "python archive",
- Output: binFile,
- Implicits: implicits,
- Args: map[string]string{
- "initFile": initFile,
- // the "\" isn't being interpreted by regex parser, it's being
- // interpreted in the string literal.
- "interp": strings.Replace(interpreter, "/", `\/`, -1),
- "main": strings.Replace(main, "/", `\/`, -1),
- "template": template.String(),
- "stub": stub,
- "parFile": parFile.String(),
- "parArgs": strings.Join(parArgs, " "),
- },
- })
+ if !embedded_launcher {
+ // the path of stub_template_host.txt from source tree.
+ template := android.PathForSource(ctx, stubTemplateHost)
+
+ // intermediate output path for __main__.py
+ stub := android.PathForModuleOut(ctx, mainFileName).String()
+
+ // added stub file to the soong_zip args.
+ parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(stub, mainFileName)+` -f `+stub)
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: host_par,
+ Description: "host python archive",
+ Output: binFile,
+ Implicits: implicits,
+ Args: map[string]string{
+ "initFile": initFile,
+ "interp": strings.Replace(interpreter, "/", `\/`, -1),
+ // we need remove "runfiles/" suffix since stub script starts
+ // searching for main file in each sub-dir of "runfiles" directory tree.
+ "main": strings.Replace(strings.TrimPrefix(main, runFiles+"/"),
+ "/", `\/`, -1),
+ "template": template.String(),
+ "stub": stub,
+ "parFile": parFile.String(),
+ "parArgs": strings.Join(parArgs, " "),
+ },
+ })
+ } else {
+ // added launcher_path to the implicits Ninja dependencies.
+ implicits = append(implicits, launcher_path)
+
+ // .intermediate output path for entry_point.txt
+ entryPoint := android.PathForModuleOut(ctx, entryPointFile).String()
+
+ // added entry_point file to the soong_zip args.
+ parArgs = append(parArgs, `-P "" `+`-C `+fmt.Sprintf(
+ "%q", strings.TrimSuffix(entryPoint, entryPointFile))+` -f `+entryPoint)
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: embedded_par,
+ Description: "embedded python archive",
+ Output: binFile,
+ Implicits: implicits,
+ Args: map[string]string{
+ "initFile": initFile,
+ "main": main,
+ "entry_point": entryPoint,
+ "parFile": parFile.String(),
+ "parArgs": strings.Join(parArgs, " "),
+ "launcher": launcher_path.String(),
+ },
+ })
+ }
return binFile
}
diff --git a/python/defaults.go b/python/defaults.go
new file mode 100644
index 0000000..641aca4
--- /dev/null
+++ b/python/defaults.go
@@ -0,0 +1,51 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package python
+
+import (
+ "android/soong/android"
+)
+
+func init() {
+ android.RegisterModuleType("python_defaults", defaultsFactory)
+}
+
+type Defaults struct {
+ android.ModuleBase
+ android.DefaultsModuleBase
+}
+
+func (d *Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+}
+
+func (d *Defaults) DepsMutator(ctx android.BottomUpMutatorContext) {
+}
+
+func defaultsFactory() android.Module {
+ return DefaultsFactory()
+}
+
+func DefaultsFactory(props ...interface{}) android.Module {
+ module := &Defaults{}
+
+ module.AddProperties(props...)
+ module.AddProperties(
+ &BaseProperties{},
+ )
+
+ android.InitDefaultsModule(module)
+
+ return module
+}
diff --git a/python/installer.go b/python/installer.go
index 9c12f5f..ab3d9b4 100644
--- a/python/installer.go
+++ b/python/installer.go
@@ -15,25 +15,47 @@
package python
import (
+ "path/filepath"
+
"android/soong/android"
)
// This file handles installing python executables into their final location
+type installLocation int
+
+const (
+ InstallInData installLocation = iota
+)
+
type pythonInstaller struct {
- dir string
+ dir string
+ dir64 string
+ relative string
path android.OutputPath
}
-func NewPythonInstaller(dir string) *pythonInstaller {
+func NewPythonInstaller(dir, dir64 string) *pythonInstaller {
return &pythonInstaller{
- dir: dir,
+ dir: dir,
+ dir64: dir64,
}
}
var _ installer = (*pythonInstaller)(nil)
+func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.OutputPath {
+ dir := installer.dir
+ if ctx.Arch().ArchType.Multilib == "lib64" && installer.dir64 != "" {
+ dir = installer.dir64
+ }
+ if !ctx.Host() && !ctx.Arch().Native {
+ dir = filepath.Join(dir, ctx.Arch().ArchType.String())
+ }
+ return android.PathForModuleInstall(ctx, dir, installer.relative)
+}
+
func (installer *pythonInstaller) install(ctx android.ModuleContext, file android.Path) {
- installer.path = ctx.InstallFile(android.PathForModuleInstall(ctx, installer.dir), file)
+ installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file)
}
diff --git a/python/library.go b/python/library.go
index 2039e56..65c1352 100644
--- a/python/library.go
+++ b/python/library.go
@@ -22,16 +22,17 @@
func init() {
android.RegisterModuleType("python_library_host", PythonLibraryHostFactory)
+ android.RegisterModuleType("python_library", PythonLibraryFactory)
}
-type PythonLibrary struct {
- pythonBaseModule
-}
-
-var _ PythonSubModule = (*PythonLibrary)(nil)
-
func PythonLibraryHostFactory() android.Module {
- module := &PythonLibrary{}
+ module := newModule(android.HostSupportedNoCross, android.MultilibFirst)
- return InitPythonBaseModule(&module.pythonBaseModule, module, android.HostSupportedNoCross)
+ return module.Init()
+}
+
+func PythonLibraryFactory() android.Module {
+ module := newModule(android.HostAndDeviceSupported, android.MultilibBoth)
+
+ return module.Init()
}
diff --git a/python/python.go b/python/python.go
index df5999d..9a3b1bb 100644
--- a/python/python.go
+++ b/python/python.go
@@ -24,6 +24,7 @@
"strings"
"github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
"android/soong/android"
)
@@ -35,58 +36,70 @@
}
// the version properties that apply to python libraries and binaries.
-type PythonVersionProperties struct {
+type VersionProperties struct {
// true, if the module is required to be built with this version.
- Enabled *bool
-
- // if specified, common src files are converted to specific version with converter tool.
- // Converter bool
+ Enabled *bool `android:"arch_variant"`
// non-empty list of .py files under this strict Python version.
// srcs may reference the outputs of other modules that produce source files like genrule
// or filegroup using the syntax ":module".
- Srcs []string
+ Srcs []string `android:"arch_variant"`
+
+ // list of source files that should not be used to build the Python module.
+ // This is most useful in the arch/multilib variants to remove non-common files
+ Exclude_srcs []string `android:"arch_variant"`
// list of the Python libraries under this Python version.
- Libs []string
+ Libs []string `android:"arch_variant"`
+
+ // true, if the binary is required to be built with embedded launcher.
+ // TODO(nanzhang): Remove this flag when embedded Python3 is supported later.
+ Embedded_launcher *bool `android:"arch_variant"`
}
// properties that apply to python libraries and binaries.
-type PythonBaseModuleProperties struct {
+type BaseProperties struct {
// the package path prefix within the output artifact at which to place the source/data
// files of the current module.
// eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
// (from a.b.c import ...) statement.
// if left unspecified, all the source/data files of current module are copied to
// "runfiles/" tree directory directly.
- Pkg_path string
+ Pkg_path *string `android:"arch_variant"`
+
+ // true, if the Python module is used internally, eg, Python std libs.
+ Is_internal *bool `android:"arch_variant"`
// list of source (.py) files compatible both with Python2 and Python3 used to compile the
// Python module.
// srcs may reference the outputs of other modules that produce source files like genrule
// or filegroup using the syntax ":module".
// Srcs has to be non-empty.
- Srcs []string
+ Srcs []string `android:"arch_variant"`
+
+ // list of source files that should not be used to build the C/C++ module.
+ // This is most useful in the arch/multilib variants to remove non-common files
+ Exclude_srcs []string `android:"arch_variant"`
// list of files or filegroup modules that provide data that should be installed alongside
// the test. the file extension can be arbitrary except for (.py).
- Data []string
+ Data []string `android:"arch_variant"`
// list of the Python libraries compatible both with Python2 and Python3.
- Libs []string
+ Libs []string `android:"arch_variant"`
Version struct {
// all the "srcs" or Python dependencies that are to be used only for Python2.
- Py2 PythonVersionProperties
+ Py2 VersionProperties `android:"arch_variant"`
// all the "srcs" or Python dependencies that are to be used only for Python3.
- Py3 PythonVersionProperties
- }
+ Py3 VersionProperties `android:"arch_variant"`
+ } `android:"arch_variant"`
// the actual version each module uses after variations created.
// this property name is hidden from users' perspectives, and soong will populate it during
// runtime.
- ActualVersion string `blueprint:"mutated"`
+ Actual_version string `blueprint:"mutated"`
}
type pathMapping struct {
@@ -94,11 +107,22 @@
src android.Path
}
-type pythonBaseModule struct {
+type Module struct {
android.ModuleBase
- subModule PythonSubModule
+ android.DefaultableModuleBase
- properties PythonBaseModuleProperties
+ properties BaseProperties
+
+ // initialize before calling Init
+ hod android.HostOrDeviceSupported
+ multilib android.Multilib
+
+ // the bootstrapper is used to bootstrap .par executable.
+ // bootstrapper might be nil (Python library module).
+ bootstrapper bootstrapper
+
+ // the installer might be nil.
+ installer installer
// the Python files of current module after expanding source dependencies.
// pathMapping: <dest: runfile_path, src: source_path>
@@ -108,17 +132,37 @@
// pathMapping: <dest: runfile_path, src: source_path>
dataPathMappings []pathMapping
+ // soong_zip arguments of all its dependencies.
+ depsParSpecs []parSpec
+
+ // Python runfiles paths of all its dependencies.
+ depsPyRunfiles []string
+
+ // (.intermediate) module output path as installation source.
+ installSource android.OptionalPath
+
// the soong_zip arguments for zipping current module source/data files.
parSpec parSpec
- // the installer might be nil.
- installer installer
-
subAndroidMkOnce map[subAndroidMkProvider]bool
}
-type PythonSubModule interface {
- GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath
+func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
+ return &Module{
+ hod: hod,
+ multilib: multilib,
+ }
+}
+
+type bootstrapper interface {
+ bootstrapperProps() []interface{}
+ bootstrap(ctx android.ModuleContext, Actual_version string, embedded_launcher bool,
+ srcsPathMappings []pathMapping, parSpec parSpec,
+ depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath
+}
+
+type installer interface {
+ install(ctx android.ModuleContext, path android.Path)
}
type PythonDependency interface {
@@ -127,64 +171,59 @@
GetParSpec() parSpec
}
-type pythonDecorator struct {
- baseInstaller *pythonInstaller
-}
-
-type installer interface {
- install(ctx android.ModuleContext, path android.Path)
-}
-
-func (p *pythonBaseModule) GetSrcsPathMappings() []pathMapping {
+func (p *Module) GetSrcsPathMappings() []pathMapping {
return p.srcsPathMappings
}
-func (p *pythonBaseModule) GetDataPathMappings() []pathMapping {
+func (p *Module) GetDataPathMappings() []pathMapping {
return p.dataPathMappings
}
-func (p *pythonBaseModule) GetParSpec() parSpec {
+func (p *Module) GetParSpec() parSpec {
return p.parSpec
}
-var _ PythonDependency = (*pythonBaseModule)(nil)
+var _ PythonDependency = (*Module)(nil)
-var _ android.AndroidMkDataProvider = (*pythonBaseModule)(nil)
+var _ android.AndroidMkDataProvider = (*Module)(nil)
-func InitPythonBaseModule(baseModule *pythonBaseModule, subModule PythonSubModule,
- hod android.HostOrDeviceSupported) android.Module {
+func (p *Module) Init() android.Module {
- baseModule.subModule = subModule
+ p.AddProperties(&p.properties)
+ if p.bootstrapper != nil {
+ p.AddProperties(p.bootstrapper.bootstrapperProps()...)
+ }
- baseModule.AddProperties(&baseModule.properties)
+ android.InitAndroidArchModule(p, p.hod, p.multilib)
+ android.InitDefaultableModule(p)
- android.InitAndroidArchModule(baseModule, hod, android.MultilibCommon)
-
- return baseModule
+ return p
}
-// the tag used to mark dependencies within "py_libs" attribute.
-type pythonDependencyTag struct {
+type dependencyTag struct {
blueprint.BaseDependencyTag
+ name string
}
-var pyDependencyTag pythonDependencyTag
-
var (
+ pythonLibTag = dependencyTag{name: "pythonLib"}
+ launcherTag = dependencyTag{name: "launcher"}
pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`)
pyExt = ".py"
pyVersion2 = "PY2"
pyVersion3 = "PY3"
initFileName = "__init__.py"
mainFileName = "__main__.py"
+ entryPointFile = "entry_point.txt"
parFileExt = ".zip"
runFiles = "runfiles"
+ internal = "internal"
)
// create version variants for modules.
func versionSplitMutator() func(android.BottomUpMutatorContext) {
return func(mctx android.BottomUpMutatorContext) {
- if base, ok := mctx.Module().(*pythonBaseModule); ok {
+ if base, ok := mctx.Module().(*Module); ok {
versionNames := []string{}
if base.properties.Version.Py2.Enabled != nil &&
*(base.properties.Version.Py2.Enabled) == true {
@@ -197,36 +236,69 @@
modules := mctx.CreateVariations(versionNames...)
for i, v := range versionNames {
// set the actual version for Python module.
- modules[i].(*pythonBaseModule).properties.ActualVersion = v
+ modules[i].(*Module).properties.Actual_version = v
}
}
}
}
-func (p *pythonBaseModule) DepsMutator(ctx android.BottomUpMutatorContext) {
+func (p *Module) HostToolPath() android.OptionalPath {
+ if p.installer == nil {
+ // python_library is just meta module, and doesn't have any installer.
+ return android.OptionalPath{}
+ }
+ return android.OptionalPathForPath(p.installer.(*binaryDecorator).path)
+}
+
+func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool {
+ switch actual_version {
+ case pyVersion2:
+ return proptools.Bool(p.properties.Version.Py2.Embedded_launcher)
+ case pyVersion3:
+ return proptools.Bool(p.properties.Version.Py3.Embedded_launcher)
+ }
+
+ return false
+}
+
+func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
// deps from "data".
android.ExtractSourcesDeps(ctx, p.properties.Data)
// deps from "srcs".
android.ExtractSourcesDeps(ctx, p.properties.Srcs)
- switch p.properties.ActualVersion {
+ switch p.properties.Actual_version {
case pyVersion2:
// deps from "version.py2.srcs" property.
android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs)
- ctx.AddVariationDependencies(nil, pyDependencyTag,
+ ctx.AddVariationDependencies(nil, pythonLibTag,
uniqueLibs(ctx, p.properties.Libs, "version.py2.libs",
p.properties.Version.Py2.Libs)...)
+
+ if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) {
+ ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib")
+ ctx.AddFarVariationDependencies([]blueprint.Variation{
+ {"arch", ctx.Target().String()},
+ }, launcherTag, "py2-launcher")
+ }
+
case pyVersion3:
// deps from "version.py3.srcs" property.
android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs)
- ctx.AddVariationDependencies(nil, pyDependencyTag,
+ ctx.AddVariationDependencies(nil, pythonLibTag,
uniqueLibs(ctx, p.properties.Libs, "version.py3.libs",
p.properties.Version.Py3.Libs)...)
+
+ if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion3) {
+ //TODO(nanzhang): Add embedded launcher for Python3.
+ ctx.PropertyErrorf("version.py3.embedded_launcher",
+ "is not supported yet for Python3.")
+ }
default:
- panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
- p.properties.ActualVersion, ctx.ModuleName()))
+ panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+ p.properties.Actual_version, ctx.ModuleName()))
}
}
@@ -258,27 +330,43 @@
return ret
}
-func (p *pythonBaseModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- installSource := p.subModule.GeneratePythonBuildActions(ctx)
+func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ p.GeneratePythonBuildActions(ctx)
- if p.installer != nil && installSource.Valid() {
- p.installer.install(ctx, installSource.Path())
+ if p.bootstrapper != nil {
+ // TODO(nanzhang): Since embedded launcher is not supported for Python3 for now,
+ // so we initialize "embedded_launcher" to false.
+ embedded_launcher := false
+ if p.properties.Actual_version == pyVersion2 {
+ embedded_launcher = p.isEmbeddedLauncherEnabled(pyVersion2)
+ }
+ p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
+ embedded_launcher, p.srcsPathMappings, p.parSpec, p.depsPyRunfiles,
+ p.depsParSpecs)
}
+
+ if p.installer != nil && p.installSource.Valid() {
+ p.installer.install(ctx, p.installSource.Path())
+ }
+
}
-func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath {
+func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) {
// expand python files from "srcs" property.
srcs := p.properties.Srcs
- switch p.properties.ActualVersion {
+ exclude_srcs := p.properties.Exclude_srcs
+ switch p.properties.Actual_version {
case pyVersion2:
srcs = append(srcs, p.properties.Version.Py2.Srcs...)
+ exclude_srcs = append(exclude_srcs, p.properties.Version.Py2.Exclude_srcs...)
case pyVersion3:
srcs = append(srcs, p.properties.Version.Py3.Srcs...)
+ exclude_srcs = append(exclude_srcs, p.properties.Version.Py3.Exclude_srcs...)
default:
- panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
- p.properties.ActualVersion, ctx.ModuleName()))
+ panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+ p.properties.Actual_version, ctx.ModuleName()))
}
- expandedSrcs := ctx.ExpandSources(srcs, nil)
+ expandedSrcs := ctx.ExpandSources(srcs, exclude_srcs)
if len(expandedSrcs) == 0 {
ctx.ModuleErrorf("doesn't have any source files!")
}
@@ -287,20 +375,31 @@
expandedData := ctx.ExpandSources(p.properties.Data, nil)
// sanitize pkg_path.
- pkg_path := p.properties.Pkg_path
+ pkg_path := String(p.properties.Pkg_path)
if pkg_path != "" {
- pkg_path = filepath.Clean(p.properties.Pkg_path)
+ pkg_path = filepath.Clean(String(p.properties.Pkg_path))
if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") ||
strings.HasPrefix(pkg_path, "/") {
- ctx.PropertyErrorf("pkg_path", "%q is not a valid format.",
- p.properties.Pkg_path)
- return android.OptionalPath{}
+ ctx.PropertyErrorf("pkg_path",
+ "%q must be a relative path contained in par file.",
+ String(p.properties.Pkg_path))
+ return
}
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = filepath.Join(runFiles, pkg_path)
+ if p.properties.Is_internal != nil && *p.properties.Is_internal {
+ // pkg_path starts from "internal/" implicitly.
+ pkg_path = filepath.Join(internal, pkg_path)
+ } else {
+ // pkg_path starts from "runfiles/" implicitly.
+ pkg_path = filepath.Join(runFiles, pkg_path)
+ }
} else {
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = runFiles
+ if p.properties.Is_internal != nil && *p.properties.Is_internal {
+ // pkg_path starts from "runfiles/" implicitly.
+ pkg_path = internal
+ } else {
+ // pkg_path starts from "runfiles/" implicitly.
+ pkg_path = runFiles
+ }
}
p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData)
@@ -308,13 +407,11 @@
p.parSpec = p.dumpFileList(ctx, pkg_path)
p.uniqWholeRunfilesTree(ctx)
-
- return android.OptionalPath{}
}
// generate current module unique pathMappings: <dest: runfiles_path, src: source_path>
// for python/data files.
-func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
+func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
expandedSrcs, expandedData android.Paths) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates.
@@ -355,7 +452,7 @@
}
// register build actions to dump filelist to disk.
-func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
+func (p *Module) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
relativeRootMap := make(map[string]android.Paths)
// the soong_zip params in order to pack current module's Python/data files.
ret := parSpec{rootPrefix: pkg_path}
@@ -365,7 +462,8 @@
// "srcs" or "data" properties may have filegroup so it might happen that
// the relative root for each source path is different.
for _, path := range pathMappings {
- relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
+ var relativeRoot string
+ relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
if v, found := relativeRootMap[relativeRoot]; found {
relativeRootMap[relativeRoot] = append(v, path.src)
} else {
@@ -392,8 +490,19 @@
return ret
}
-// check Python/data files duplicates from current module and its whole dependencies.
-func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
+func isPythonLibModule(module blueprint.Module) bool {
+ if m, ok := module.(*Module); ok {
+ // Python library has no bootstrapper or installer.
+ if m.bootstrapper != nil || m.installer != nil {
+ return false
+ }
+ return true
+ }
+ return false
+}
+
+// check Python source/data files duplicates from current module and its whole dependencies.
+func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates.
destToPySrcs := make(map[string]string)
@@ -407,17 +516,16 @@
}
// visit all its dependencies in depth first.
- ctx.VisitDepsDepthFirst(func(module blueprint.Module) {
- // module can only depend on Python library.
- if base, ok := module.(*pythonBaseModule); ok {
- if _, ok := base.subModule.(*PythonLibrary); !ok {
- panic(fmt.Errorf(
- "the dependency %q of module %q is not Python library!",
- ctx.ModuleName(), ctx.OtherModuleName(module)))
- }
- } else {
+ ctx.VisitDepsDepthFirst(func(module android.Module) {
+ if ctx.OtherModuleDependencyTag(module) != pythonLibTag {
return
}
+ // Python module cannot depend on modules, except for Python library.
+ if !isPythonLibModule(module) {
+ panic(fmt.Errorf(
+ "the dependency %q of module %q is not Python library!",
+ ctx.ModuleName(), ctx.OtherModuleName(module)))
+ }
if dep, ok := module.(PythonDependency); ok {
srcs := dep.GetSrcsPathMappings()
for _, path := range srcs {
@@ -428,9 +536,7 @@
}
// binary needs the Python runfiles paths from all its
// dependencies to fill __init__.py in each runfiles dir.
- if sub, ok := p.subModule.(*pythonBinaryBase); ok {
- sub.depsPyRunfiles = append(sub.depsPyRunfiles, path.dest)
- }
+ p.depsPyRunfiles = append(p.depsPyRunfiles, path.dest)
}
data := dep.GetDataPathMappings()
for _, path := range data {
@@ -440,9 +546,7 @@
}
// binary needs the soong_zip arguments from all its
// dependencies to generate executable par file.
- if sub, ok := p.subModule.(*pythonBinaryBase); ok {
- sub.depsParSpecs = append(sub.depsParSpecs, dep.GetParSpec())
- }
+ p.depsParSpecs = append(p.depsParSpecs, dep.GetParSpec())
}
})
}
@@ -461,3 +565,10 @@
return true
}
+
+func (p *Module) InstallInData() bool {
+ return true
+}
+
+var Bool = proptools.Bool
+var String = proptools.String
diff --git a/python/python_test.go b/python/python_test.go
index 4c30d95..176c6ec 100644
--- a/python/python_test.go
+++ b/python/python_test.go
@@ -28,7 +28,7 @@
"android/soong/android"
)
-type pyBinary struct {
+type pyModule struct {
name string
actualVersion string
pyRunfiles []string
@@ -41,7 +41,7 @@
buildNamePrefix = "soong_python_test"
moduleVariantErrTemplate = "%s: module %q variant %q: "
pkgPathErrTemplate = moduleVariantErrTemplate +
- "pkg_path: %q is not a valid format."
+ "pkg_path: %q must be a relative path contained in par file."
badIdentifierErrTemplate = moduleVariantErrTemplate +
"srcs: the path %q contains invalid token %q."
dupRunfileErrTemplate = moduleVariantErrTemplate +
@@ -58,7 +58,7 @@
mockFiles map[string][]byte
errors []string
- expectedBinaries []pyBinary
+ expectedBinaries []pyModule
}{
{
desc: "module without any src files",
@@ -222,7 +222,28 @@
mockFiles: map[string][]byte{
bpFile: []byte(`subdirs = ["dir"]`),
filepath.Join("dir", bpFile): []byte(
- `python_library_host {
+ `python_defaults {
+ name: "default_lib",
+ srcs: [
+ "default.py",
+ ],
+ version: {
+ py2: {
+ enabled: true,
+ srcs: [
+ "default_py2.py",
+ ],
+ },
+ py3: {
+ enabled: false,
+ srcs: [
+ "default_py3.py",
+ ],
+ },
+ },
+ }
+
+ python_library_host {
name: "lib5",
pkg_path: "a/b/",
srcs: [
@@ -251,6 +272,7 @@
python_binary_host {
name: "bin",
+ defaults: ["default_lib"],
pkg_path: "e/",
srcs: [
"bin.py",
@@ -271,19 +293,24 @@
},
}`,
),
- filepath.Join("dir", "file1.py"): nil,
- filepath.Join("dir", "file2.py"): nil,
- filepath.Join("dir", "bin.py"): nil,
- filepath.Join("dir", "file4.py"): nil,
+ filepath.Join("dir", "default.py"): nil,
+ filepath.Join("dir", "default_py2.py"): nil,
+ filepath.Join("dir", "default_py3.py"): nil,
+ filepath.Join("dir", "file1.py"): nil,
+ filepath.Join("dir", "file2.py"): nil,
+ filepath.Join("dir", "bin.py"): nil,
+ filepath.Join("dir", "file4.py"): nil,
stubTemplateHost: []byte(`PYTHON_BINARY = '%interpreter%'
MAIN_FILE = '%main%'`),
},
- expectedBinaries: []pyBinary{
+ expectedBinaries: []pyModule{
{
name: "bin",
actualVersion: "PY3",
pyRunfiles: []string{
+ "runfiles/e/default.py",
"runfiles/e/bin.py",
+ "runfiles/e/default_py3.py",
"runfiles/e/file4.py",
},
depsPyRunfiles: []string{
@@ -314,6 +341,9 @@
android.ModuleFactoryAdaptor(PythonLibraryHostFactory))
ctx.RegisterModuleType("python_binary_host",
android.ModuleFactoryAdaptor(PythonBinaryHostFactory))
+ ctx.RegisterModuleType("python_defaults",
+ android.ModuleFactoryAdaptor(defaultsFactory))
+ ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
ctx.Register()
ctx.MockFileSystem(d.mockFiles)
_, testErrs := ctx.ParseBlueprintsFiles(bpFile)
@@ -363,14 +393,10 @@
expParSpec string, expDepsParSpecs []string) (testErrs []error) {
module := ctx.ModuleForTests(name, variant)
- base, baseOk := module.Module().(*pythonBaseModule)
+ base, baseOk := module.Module().(*Module)
if !baseOk {
t.Fatalf("%s is not Python module!", name)
}
- sub, subOk := base.subModule.(*pythonBinaryBase)
- if !subOk {
- t.Fatalf("%s is not Python binary!", name)
- }
actPyRunfiles := []string{}
for _, path := range base.srcsPathMappings {
@@ -381,28 +407,28 @@
testErrs = append(testErrs, errors.New(fmt.Sprintf(
`binary "%s" variant "%s" has unexpected pyRunfiles: %q!`,
base.Name(),
- base.properties.ActualVersion,
+ base.properties.Actual_version,
actPyRunfiles)))
}
- if !reflect.DeepEqual(sub.depsPyRunfiles, expDepsPyRunfiles) {
+ if !reflect.DeepEqual(base.depsPyRunfiles, expDepsPyRunfiles) {
testErrs = append(testErrs, errors.New(fmt.Sprintf(
`binary "%s" variant "%s" has unexpected depsPyRunfiles: %q!`,
base.Name(),
- base.properties.ActualVersion,
- sub.depsPyRunfiles)))
+ base.properties.Actual_version,
+ base.depsPyRunfiles)))
}
if base.parSpec.soongParArgs() != strings.Replace(expParSpec, "@prefix@", buildDir, 1) {
testErrs = append(testErrs, errors.New(fmt.Sprintf(
`binary "%s" variant "%s" has unexpected parSpec: %q!`,
base.Name(),
- base.properties.ActualVersion,
+ base.properties.Actual_version,
base.parSpec.soongParArgs())))
}
actDepsParSpecs := []string{}
- for i, p := range sub.depsParSpecs {
+ for i, p := range base.depsParSpecs {
actDepsParSpecs = append(actDepsParSpecs, p.soongParArgs())
expDepsParSpecs[i] = strings.Replace(expDepsParSpecs[i], "@prefix@", buildDir, 1)
}
@@ -411,7 +437,7 @@
testErrs = append(testErrs, errors.New(fmt.Sprintf(
`binary "%s" variant "%s" has unexpected depsParSpecs: %q!`,
base.Name(),
- base.properties.ActualVersion,
+ base.properties.Actual_version,
actDepsParSpecs)))
}
@@ -424,7 +450,7 @@
t.Fatal(err)
}
- config = android.TestConfig(buildDir)
+ config = android.TestConfig(buildDir, nil)
return
}
diff --git a/python/test.go b/python/test.go
index 837eb25..825e63c 100644
--- a/python/test.go
+++ b/python/test.go
@@ -16,39 +16,50 @@
import (
"android/soong/android"
- "path/filepath"
)
// This file contains the module types for building Python test.
func init() {
android.RegisterModuleType("python_test_host", PythonTestHostFactory)
+ android.RegisterModuleType("python_test", PythonTestFactory)
}
-type PythonTestHost struct {
- pythonBinaryBase
+type testDecorator struct {
+ *binaryDecorator
}
-var _ PythonSubModule = (*PythonTestHost)(nil)
+func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) {
+ test.binaryDecorator.pythonInstaller.dir = "nativetest"
+ test.binaryDecorator.pythonInstaller.dir64 = "nativetest64"
-type pythonTestHostDecorator struct {
- pythonDecorator
+ test.binaryDecorator.pythonInstaller.relative = ctx.ModuleName()
+
+ test.binaryDecorator.pythonInstaller.install(ctx, file)
}
-func (p *pythonTestHostDecorator) install(ctx android.ModuleContext, file android.Path) {
- p.pythonDecorator.baseInstaller.dir = filepath.Join("nativetest", ctx.ModuleName())
- p.pythonDecorator.baseInstaller.install(ctx, file)
+func NewTest(hod android.HostOrDeviceSupported) *Module {
+ module, binary := NewBinary(hod)
+
+ binary.pythonInstaller = NewPythonInstaller("nativetest", "nativetest64")
+
+ test := &testDecorator{binaryDecorator: binary}
+
+ module.bootstrapper = test
+ module.installer = test
+
+ return module
}
func PythonTestHostFactory() android.Module {
- decorator := &pythonTestHostDecorator{
- pythonDecorator: pythonDecorator{baseInstaller: NewPythonInstaller("nativetest")}}
+ module := NewTest(android.HostSupportedNoCross)
- module := &PythonBinaryHost{}
- module.pythonBaseModule.installer = decorator
+ return module.Init()
+}
- module.AddProperties(&module.binaryProperties)
+func PythonTestFactory() android.Module {
+ module := NewTest(android.HostAndDeviceSupported)
+ module.multilib = android.MultilibBoth
- return InitPythonBaseModule(&module.pythonBinaryBase.pythonBaseModule,
- &module.pythonBinaryBase, android.HostSupportedNoCross)
+ return module.Init()
}
diff --git a/root.bp b/root.bp
index 08f2ff8..7e0c1ed 100644
--- a/root.bp
+++ b/root.bp
@@ -1,38 +1,4 @@
-subname = "Android.bp"
-
-build = [
- "build/blueprint/Blueprints",
-]
-
-subdirs = [
- "build/soong",
-]
-
-optional_subdirs = [
- "art",
- "bionic",
- "bootable/recovery",
- "build/kati",
- "build/tools/*",
- "dalvik",
- "development/*",
- "device/*/*",
- "external/*",
- "frameworks/*",
- "frameworks/compile/*",
- "frameworks/hardware/interfaces",
- "frameworks/opt/net/wifi",
- "hardware/*",
- "libcore",
- "libnativehelper",
- "packages/apps/*",
- "prebuilts/clang/host/linux-x86",
- "prebuilts/ndk",
- "prebuilts/sdk",
- "system/*",
- "system/hardware/interfaces",
- "system/tools/*",
- "test/vts",
- "test/vts-testcase/*",
- "vendor/*/*",
-]
+// Soong finds all Android.bp and Blueprints files in the source tree,
+// subdirs= and optional_subdirs= are obsolete and this file no longer
+// needs a list of the top level directories that may contain Android.bp
+// files.
diff --git a/scripts/build-ndk-prebuilts.sh b/scripts/build-ndk-prebuilts.sh
index 65a2329..55214b2 100755
--- a/scripts/build-ndk-prebuilts.sh
+++ b/scripts/build-ndk-prebuilts.sh
@@ -1,5 +1,19 @@
#!/bin/bash -ex
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
if [ -z "${OUT_DIR}" ]; then
echo Must set OUT_DIR
exit 1
@@ -9,6 +23,12 @@
source build/envsetup.sh
PLATFORM_SDK_VERSION=$(get_build_var PLATFORM_SDK_VERSION)
+PLATFORM_VERSION_ALL_CODENAMES=$(get_build_var PLATFORM_VERSION_ALL_CODENAMES)
+
+# PLATFORM_VERSION_ALL_CODESNAMES is a comma separated list like O,P. We need to
+# turn this into ["O","P"].
+PLATFORM_VERSION_ALL_CODENAMES=${PLATFORM_VERSION_ALL_CODENAMES/,/'","'}
+PLATFORM_VERSION_ALL_CODENAMES="[\"${PLATFORM_VERSION_ALL_CODENAMES}\"]"
SOONG_OUT=${OUT_DIR}/soong
SOONG_NDK_OUT=${OUT_DIR}/soong/ndk
@@ -16,12 +36,39 @@
mkdir -p ${SOONG_OUT}
cat > ${SOONG_OUT}/soong.config << EOF
{
- "Ndk_abis": true,
- "Platform_sdk_version": ${PLATFORM_SDK_VERSION}
+ "Ndk_abis": true
}
EOF
-BUILDDIR=${SOONG_OUT} ./bootstrap.bash
-${SOONG_OUT}/soong ${SOONG_OUT}/ndk.timestamp
+
+# We only really need to set some of these variables, but soong won't merge this
+# with the defaults, so we need to write out all the defaults with our values
+# added.
+cat > ${SOONG_OUT}/soong.variables << EOF
+{
+ "Platform_sdk_version": ${PLATFORM_SDK_VERSION},
+ "Platform_version_active_codenames": ${PLATFORM_VERSION_ALL_CODENAMES},
+
+ "DeviceName": "flounder",
+ "DeviceArch": "arm64",
+ "DeviceArchVariant": "armv8-a",
+ "DeviceCpuVariant": "denver64",
+ "DeviceAbi": [
+ "arm64-v8a"
+ ],
+ "DeviceUsesClang": true,
+ "DeviceSecondaryArch": "arm",
+ "DeviceSecondaryArchVariant": "armv7-a-neon",
+ "DeviceSecondaryCpuVariant": "denver",
+ "DeviceSecondaryAbi": [
+ "armeabi-v7a"
+ ],
+ "HostArch": "x86_64",
+ "HostSecondaryArch": "x86",
+ "Malloc_not_svelte": false,
+ "Safestack": false
+}
+EOF
+m --skip-make ${SOONG_OUT}/ndk.timestamp
if [ -n "${DIST_DIR}" ]; then
mkdir -p ${DIST_DIR} || true
diff --git a/scripts/copygcclib.sh b/scripts/copygcclib.sh
index 93c52cc..28359fc 100755
--- a/scripts/copygcclib.sh
+++ b/scripts/copygcclib.sh
@@ -1,7 +1,21 @@
#!/bin/bash -e
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
OUT=$1
shift
-LIBPATH=$($@)
+LIBPATH=$($@ | sed -e "s|^$PWD/||")
cp -f $LIBPATH $OUT
echo "$OUT: $LIBPATH" > ${OUT}.d
diff --git a/scripts/diff_build_graphs.sh b/scripts/diff_build_graphs.sh
new file mode 100755
index 0000000..81010f3
--- /dev/null
+++ b/scripts/diff_build_graphs.sh
@@ -0,0 +1,170 @@
+#!/bin/bash -eu
+#
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# This file makes it easy to confirm that a set of changes in source code don't result in any
+# changes to the generated ninja files. This is to reduce the effort required to be confident
+# in the correctness of refactorings
+
+function die() {
+ echo "$@" >&2
+ exit 1
+}
+
+function usage() {
+ violation="$1"
+ die "$violation
+
+ Usage: diff_build_graphs.sh [--products=product1,product2...] <OLD_VERSIONS> <NEW_VERSIONS>
+
+ This file builds and parses the build files (Android.mk, Android.bp, etc) for each requested
+ product and for both sets of versions, and checks whether the ninja files (which implement
+ the build graph) changed between the two versions.
+
+ Example: diff_build_graphs.sh 'build/soong:work^ build/blueprint:work^' 'build/soong:work build/blueprint:work'
+
+ Options:
+ --products=PRODUCTS comma-separated list of products to check"
+}
+
+PRODUCTS_ARG=""
+OLD_VERSIONS=""
+NEW_VERSIONS=""
+function parse_args() {
+ # parse optional arguments
+ while true; do
+ arg="${1-}"
+ case "$arg" in
+ --products=*) PRODUCTS_ARG="$arg";;
+ *) break;;
+ esac
+ shift
+ done
+ # parse required arguments
+ if [ "$#" != "2" ]; then
+ usage ""
+ fi
+ #argument validation
+ OLD_VERSIONS="$1"
+ NEW_VERSIONS="$2"
+
+}
+parse_args "$@"
+
+
+# find some file paths
+cd "$(dirname $0)"
+SCRIPT_DIR="$PWD"
+cd ../../..
+CHECKOUT_ROOT="$PWD"
+OUT_DIR="${OUT_DIR-}"
+if [ -z "$OUT_DIR" ]; then
+ OUT_DIR=out
+fi
+WORK_DIR="$OUT_DIR/diff"
+OUT_DIR_OLD="$WORK_DIR/out_old"
+OUT_DIR_NEW="$WORK_DIR/out_new"
+OUT_DIR_TEMP="$WORK_DIR/out_temp"
+
+
+function checkout() {
+ versionSpecs="$1"
+ for versionSpec in $versionSpecs; do
+ project="$(echo $versionSpec | sed 's|\([^:]*\):\([^:]*\)|\1|')"
+ ref="$(echo $versionSpec | sed 's|\([^:]*\):\([^:]*\)|\2|')"
+ echo "checking out ref $ref in project $project"
+ git -C "$project" checkout "$ref"
+ done
+}
+
+function run_build() {
+ echo
+ echo "Starting build"
+ # rebuild multiproduct_kati, in case it was missing before,
+ # or in case it is affected by some of the changes we're testing
+ make blueprint_tools
+ # find multiproduct_kati and have it build the ninja files for each product
+ builder="$(echo $OUT_DIR/soong/host/*/bin/multiproduct_kati)"
+ BUILD_NUMBER=sample "$builder" $PRODUCTS_ARG --keep --out "$OUT_DIR_TEMP" || true
+ echo
+}
+
+function diffProduct() {
+ product="$1"
+
+ zip1="$OUT_DIR_OLD/${product}.zip"
+ unzipped1="$OUT_DIR_OLD/$product"
+
+ zip2="$OUT_DIR_NEW/${product}.zip"
+ unzipped2="$OUT_DIR_NEW/$product"
+
+ unzip -qq "$zip1" -d "$unzipped1"
+ unzip -qq "$zip2" -d "$unzipped2"
+
+ #do a diff of the ninja files
+ diffFile="$WORK_DIR/diff.txt"
+ diff -r "$unzipped1" "$unzipped2" -x build_date.txt -x build_number.txt -x '\.*' -x '*.log' -x build_fingerprint.txt -x build.ninja.d -x '*.zip' > $diffFile || true
+ if [[ -s "$diffFile" ]]; then
+ # outputs are different, so remove the unzipped versions but keep the zipped versions
+ echo "First few differences (total diff linecount=$(wc -l $diffFile)) for product $product:"
+ cat "$diffFile" | head -n 10
+ echo "End of differences for product $product"
+ rm -rf "$unzipped1" "$unzipped2"
+ else
+ # outputs are the same, so remove all of the outputs
+ rm -rf "$zip1" "$unzipped1" "$zip2" "$unzipped2"
+ fi
+}
+
+function do_builds() {
+ #reset work dir
+ rm -rf "$WORK_DIR"
+ mkdir "$WORK_DIR"
+
+ #build new code
+ checkout "$NEW_VERSIONS"
+ run_build
+ mv "$OUT_DIR_TEMP" "$OUT_DIR_NEW"
+
+ #build old code
+ #TODO do we want to cache old results? Maybe by the time we care to cache old results this will
+ #be running on a remote server somewhere and be completely different
+ checkout "$OLD_VERSIONS"
+ run_build
+ mv "$OUT_DIR_TEMP" "$OUT_DIR_OLD"
+
+ #cleanup
+ echo created "$OUT_DIR_OLD" and "$OUT_DIR_NEW"
+}
+
+function main() {
+ do_builds
+ checkout "$NEW_VERSIONS"
+
+ #find all products
+ productsFile="$WORK_DIR/all_products.txt"
+ find $OUT_DIR_OLD $OUT_DIR_NEW -mindepth 1 -maxdepth 1 -name "*.zip" | sed "s|^$OUT_DIR_OLD/||" | sed "s|^$OUT_DIR_NEW/||" | sed "s|\.zip$||" | sort | uniq > "$productsFile"
+ echo Diffing products
+ for product in $(cat $productsFile); do
+ diffProduct "$product"
+ done
+ echo Done diffing products
+ echo "Any differing outputs can be seen at $OUT_DIR_OLD/*.zip and $OUT_DIR_NEW/*.zip"
+ echo "See $WORK_DIR/diff.txt for the full list of differences for the latest product checked"
+}
+
+main
diff --git a/scripts/extract-srcjars.sh b/scripts/extract-srcjars.sh
new file mode 100755
index 0000000..f81032b
--- /dev/null
+++ b/scripts/extract-srcjars.sh
@@ -0,0 +1,44 @@
+#!/bin/bash -e
+
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Extracts .java files from source jars in a specified directory and writes out a list of the files
+
+if [ -z "$1" -o -z "$2" ]; then
+ echo "usage: $0 <output dir> <output file> [<jar> ...]" >&2
+ exit 1
+fi
+
+output_dir=$1
+shift
+output_file=$1
+shift
+
+rm -f $output_file
+touch $output_file
+
+for j in "$@"; do
+ for f in $(zipinfo -1 $j '*.java'); do
+ echo $output_dir/$f >> $output_file
+ done
+ unzip -qn -d $output_dir $j '*.java'
+done
+
+duplicates=$(cat $output_file | sort | uniq -d | uniq)
+if [ -n "$duplicates" ]; then
+ echo Duplicate source files:
+ echo $duplicates
+ exit 1
+fi
diff --git a/scripts/jar-args.sh b/scripts/jar-args.sh
new file mode 100755
index 0000000..340e9a3
--- /dev/null
+++ b/scripts/jar-args.sh
@@ -0,0 +1,78 @@
+#!/bin/bash -e
+
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script that takes a list of files on stdin and converts it to arguments to jar on stdout
+# Usage:
+# find $dir -type f | sort | jar-args.sh $dir > jar_args.txt
+# jar cf out.jar @jar_args.txt
+
+case $(uname) in
+ Linux)
+ extended_re=-r
+ ;;
+ Darwin)
+ extended_re=-E
+ ;;
+ *) echo "unknown OS:" $(uname) >&2 && exit 1;;
+esac
+
+if [ "$1" == "--test" ]; then
+ in=$(mktemp)
+ expected=$(mktemp)
+ out=$(mktemp)
+ cat > $in <<EOF
+a
+a/b
+a/b/'
+a/b/"
+a/b/\\
+a/b/#
+a/b/a
+EOF
+ cat > $expected <<EOF
+
+-C 'a' 'b'
+-C 'a' 'b/\\''
+-C 'a' 'b/"'
+-C 'a' 'b/\\\\'
+-C 'a' 'b/#'
+-C 'a' 'b/a'
+EOF
+ cat $in | $0 a > $out
+
+ if cmp $out $expected; then
+ status=0
+ echo "PASS"
+ else
+ status=1
+ echo "FAIL"
+ echo "got:"
+ cat $out
+ echo "expected:"
+ cat $expected
+ fi
+ rm -f $in $expected $out
+ exit $status
+fi
+
+# In order, the regexps:
+# - Strip $1/ from the beginning of each line, and everything from lines that just have $1
+# - Escape single and double quotes, '#', ' ', and '\'
+# - Prefix each non-blank line with -C $1
+sed ${extended_re} \
+ -e"s,^$1(/|\$),," \
+ -e"s,(['\\]),\\\\\1,g" \
+ -e"s,^(.+),-C '$1' '\1',"
diff --git a/scripts/jar-wrapper.sh b/scripts/jar-wrapper.sh
new file mode 100644
index 0000000..71c1d90
--- /dev/null
+++ b/scripts/jar-wrapper.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set up prog to be the path of this script, including following symlinks,
+# and set up progdir to be the fully-qualified pathname of its directory.
+
+prog="$0"
+while [ -h "${prog}" ]; do
+ fullprog=`/bin/ls -ld "${prog}"`
+ fullprog=`expr "${fullprog}" : ".* -> \(.*\)$"`
+ if expr "x${fullprog}" : 'x/' >/dev/null; then
+ prog="${fullprog}"
+ else
+ progdir=`dirname "${prog}"`
+ prog="${progdir}/${fullprog}"
+ fi
+done
+
+oldwd=`pwd`
+progdir=`dirname "${prog}"`
+cd "${progdir}"
+progdir=`pwd`
+prog="${progdir}"/`basename "${prog}"`
+cd "${oldwd}"
+
+jarfile=`basename "${prog}"`.jar
+jardir="${progdir}"
+
+if [ ! -r "${jardir}/${jarfile}" ]; then
+ jardir=`dirname "${progdir}"`/framework
+fi
+
+if [ ! -r "${jardir}/${jarfile}" ]; then
+ echo `basename "${prog}"`": can't find ${jarfile}"
+ exit 1
+fi
+
+javaOpts=""
+while expr "x$1" : 'x-J' >/dev/null; do
+ opt=`expr "$1" : '-J\(.*\)'`
+ javaOpts="${javaOpts} -${opt}"
+ shift
+done
+
+exec java ${javaOpts} -jar ${jardir}/${jarfile} "$@"
diff --git a/scripts/jars-to-module-info-java.sh b/scripts/jars-to-module-info-java.sh
new file mode 100755
index 0000000..dd15198
--- /dev/null
+++ b/scripts/jars-to-module-info-java.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -e
+
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Extracts the Java package names of all classes in the .jar files and writes a module-info.java
+# file to stdout that exports all of those packages.
+
+if [ -z "$1" ]; then
+ echo "usage: $0 <module name> <jar1> [<jar2> ...]" >&2
+ exit 1
+fi
+
+module_name=$1
+shift
+
+echo "module ${module_name} {"
+for j in "$@"; do zipinfo -1 $j ; done \
+ | grep -E '/[^/]*\.class$' \
+ | sed 's|\(.*\)/[^/]*\.class$| exports \1;|g' \
+ | sed 's|/|.|g' \
+ | sort -u
+echo "}"
diff --git a/scripts/reverse_path.py b/scripts/reverse_path.py
deleted file mode 100755
index 7b7d621..0000000
--- a/scripts/reverse_path.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-
-import os
-import sys
-
-# Find the best reverse path to reference the current directory from another
-# directory. We use this to find relative paths to and from the source and build
-# directories.
-#
-# If the directory is given as an absolute path, return an absolute path to the
-# current directory.
-#
-# If there's a symlink involved, and the same relative path would not work if
-# the symlink was replace with a regular directory, then return an absolute
-# path. This handles paths like out -> /mnt/ssd/out
-#
-# For symlinks that can use the same relative path (out -> out.1), just return
-# the relative path. That way out.1 can be renamed as long as the symlink is
-# updated.
-#
-# For everything else, just return the relative path. That allows the source and
-# output directories to be moved as long as they stay in the same position
-# relative to each other.
-def reverse_path(path):
- if path.startswith("/"):
- return os.path.abspath('.')
-
- realpath = os.path.relpath(os.path.realpath('.'), os.path.realpath(path))
- relpath = os.path.relpath('.', path)
-
- if realpath != relpath:
- return os.path.abspath('.')
-
- return relpath
-
-
-if __name__ == '__main__':
- print(reverse_path(sys.argv[1]))
diff --git a/scripts/reverse_path_test.py b/scripts/reverse_path_test.py
deleted file mode 100755
index 5577693..0000000
--- a/scripts/reverse_path_test.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-
-import os
-import shutil
-import tempfile
-import unittest
-
-from reverse_path import reverse_path
-
-class TestReversePath(unittest.TestCase):
- def setUp(self):
- self.tmpdir = tempfile.mkdtemp()
- os.chdir(self.tmpdir)
-
- def tearDown(self):
- shutil.rmtree(self.tmpdir)
-
- def test_absolute(self):
- self.assertEqual(self.tmpdir, reverse_path('/out'))
-
- def test_relative(self):
- os.mkdir('a')
- os.mkdir('b')
-
- self.assertEqual('..', reverse_path('a'))
-
- os.chdir('a')
- self.assertEqual('a', reverse_path('..'))
- self.assertEqual('.', reverse_path('../a'))
- self.assertEqual('../a', reverse_path('../b'))
-
- def test_symlink(self):
- os.mkdir('b')
- os.symlink('b', 'a')
- os.mkdir('b/d')
- os.symlink('b/d', 'c')
-
- self.assertEqual('..', reverse_path('a'))
- self.assertEqual('..', reverse_path('b'))
- self.assertEqual(self.tmpdir, reverse_path('c'))
- self.assertEqual('../..', reverse_path('b/d'))
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/scripts/setup_go_workspace_for_soong.sh b/scripts/setup_go_workspace_for_soong.sh
index 4b118ef..e2fb9fa 100755
--- a/scripts/setup_go_workspace_for_soong.sh
+++ b/scripts/setup_go_workspace_for_soong.sh
@@ -1,6 +1,20 @@
#!/bin/bash
set -e
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
#mounts the components of soong into a directory structure that Go tools and editors expect
#move to the script's directory
@@ -39,6 +53,7 @@
bindOne "${ANDROID_PATH}/build/soong" "${OUTPUT_PATH}/src/android/soong"
bindOne "${ANDROID_PATH}/art/build" "${OUTPUT_PATH}/src/android/soong/art"
+ bindOne "${ANDROID_PATH}/external/golang-protobuf" "${OUTPUT_PATH}/src/github.com/golang/protobuf"
bindOne "${ANDROID_PATH}/external/llvm/soong" "${OUTPUT_PATH}/src/android/soong/llvm"
bindOne "${ANDROID_PATH}/external/clang/soong" "${OUTPUT_PATH}/src/android/soong/clang"
echo
@@ -50,7 +65,14 @@
existingPath="$1"
newPath="$2"
mkdir -p "$newPath"
- echoAndDo bindfs "${existingPath}" "${newPath}"
+ case $(uname -s) in
+ Darwin)
+ echoAndDo bindfs -o allow_recursion -n "${existingPath}" "${newPath}"
+ ;;
+ Linux)
+ echoAndDo bindfs "${existingPath}" "${newPath}"
+ ;;
+ esac
}
function echoAndDo() {
diff --git a/scripts/strip.sh b/scripts/strip.sh
index b9ff741..848fe8d 100755
--- a/scripts/strip.sh
+++ b/scripts/strip.sh
@@ -1,5 +1,19 @@
#!/bin/bash -e
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# Script to handle the various ways soong may need to strip binaries
# Inputs:
# Environment:
diff --git a/scripts/toc.sh b/scripts/toc.sh
index 59bf8a3..7b2224c 100755
--- a/scripts/toc.sh
+++ b/scripts/toc.sh
@@ -1,5 +1,19 @@
#!/bin/bash -eu
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# Script to handle generating a .toc file from a .so file
# Inputs:
# Environment:
diff --git a/soong.bash b/soong.bash
index 1dbf4e4..8cf2ec6 100755
--- a/soong.bash
+++ b/soong.bash
@@ -1,40 +1,22 @@
#!/bin/bash
-set -e
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-# Switch to the build directory
-cd $(dirname "${BASH_SOURCE[0]}")
-
-# The source directory path and operating system will get written to
-# .soong.bootstrap by the bootstrap script.
-
-BOOTSTRAP=".soong.bootstrap"
-if [ ! -f "${BOOTSTRAP}" ]; then
- echo "Error: soong script must be located in a directory created by bootstrap.bash"
- exit 1
-fi
-
-source "${BOOTSTRAP}"
-
-# Now switch to the source directory so that all the relative paths from
-# $BOOTSTRAP are correct
-cd ${SRCDIR_FROM_BUILDDIR}
-
-# Ninja can't depend on environment variables, so do a manual comparison
-# of the relevant environment variables from the last build using the
-# soong_env tool and trigger a build manifest regeneration if necessary
-ENVFILE="${BUILDDIR}/.soong.environment"
-ENVTOOL="${BUILDDIR}/.bootstrap/bin/soong_env"
-if [ -f "${ENVFILE}" ]; then
- if [ -x "${ENVTOOL}" ]; then
- if ! "${ENVTOOL}" "${ENVFILE}"; then
- echo "forcing build manifest regeneration"
- rm -f "${ENVFILE}"
- fi
- else
- echo "Missing soong_env tool, forcing build manifest regeneration"
- rm -f "${ENVFILE}"
- fi
-fi
-
-BUILDDIR="${BUILDDIR}" NINJA="prebuilts/build-tools/${PREBUILTOS}/bin/ninja" build/blueprint/blueprint.bash "$@"
+echo '==== ERROR: bootstrap.bash & ./soong are obsolete ====' >&2
+echo 'Use `m --skip-make` with a standalone OUT_DIR instead.' >&2
+echo 'Without envsetup.sh, use:' >&2
+echo ' build/soong/soong_ui.bash --make-mode --skip-make' >&2
+echo '======================================================' >&2
+exit 1
diff --git a/third_party/zip/android.go b/third_party/zip/android.go
index f3b6055..8d387cc 100644
--- a/third_party/zip/android.go
+++ b/third_party/zip/android.go
@@ -19,6 +19,9 @@
"io"
)
+const DataDescriptorFlag = 0x8
+const ExtendedTimeStampTag = 0x5455
+
func (w *Writer) CopyFrom(orig *File, newName string) error {
if w.last != nil && !w.last.closed {
if err := w.last.close(); err != nil {
@@ -30,13 +33,10 @@
fileHeader := orig.FileHeader
fileHeader.Name = newName
fh := &fileHeader
- fh.Flags |= 0x8
- // The zip64 extras change between the Central Directory and Local File Header, while we use
- // the same structure for both. The Local File Haeder is taken care of by us writing a data
- // descriptor with the zip64 values. The Central Directory Entry is written by Close(), where
- // the zip64 extra is automatically created and appended when necessary.
- fh.Extra = stripZip64Extras(fh.Extra)
+ // In some cases, we need strip the extras if it change between Central Directory
+ // and Local File Header.
+ fh.Extra = stripExtras(fh.Extra)
h := &header{
FileHeader: fh,
@@ -47,37 +47,45 @@
if err := writeHeader(w.cw, fh); err != nil {
return err
}
-
- // Copy data
dataOffset, err := orig.DataOffset()
if err != nil {
return err
}
io.Copy(w.cw, io.NewSectionReader(orig.zipr, dataOffset, int64(orig.CompressedSize64)))
- // Write data descriptor.
- var buf []byte
- if fh.isZip64() {
- buf = make([]byte, dataDescriptor64Len)
- } else {
- buf = make([]byte, dataDescriptorLen)
+ if orig.hasDataDescriptor() {
+ // Write data descriptor.
+ var buf []byte
+ if fh.isZip64() {
+ buf = make([]byte, dataDescriptor64Len)
+ } else {
+ buf = make([]byte, dataDescriptorLen)
+ }
+ b := writeBuf(buf)
+ b.uint32(dataDescriptorSignature)
+ b.uint32(fh.CRC32)
+ if fh.isZip64() {
+ b.uint64(fh.CompressedSize64)
+ b.uint64(fh.UncompressedSize64)
+ } else {
+ b.uint32(fh.CompressedSize)
+ b.uint32(fh.UncompressedSize)
+ }
+ _, err = w.cw.Write(buf)
}
- b := writeBuf(buf)
- b.uint32(dataDescriptorSignature)
- b.uint32(fh.CRC32)
- if fh.isZip64() {
- b.uint64(fh.CompressedSize64)
- b.uint64(fh.UncompressedSize64)
- } else {
- b.uint32(fh.CompressedSize)
- b.uint32(fh.UncompressedSize)
- }
- _, err = w.cw.Write(buf)
return err
}
-// Strip any Zip64 extra fields
-func stripZip64Extras(input []byte) []byte {
+// The zip64 extras change between the Central Directory and Local File Header, while we use
+// the same structure for both. The Local File Haeder is taken care of by us writing a data
+// descriptor with the zip64 values. The Central Directory Entry is written by Close(), where
+// the zip64 extra is automatically created and appended when necessary.
+//
+// The extended-timestamp extra block changes between the Central Directory Header and Local
+// File Header.
+// Extended-Timestamp extra(LFH): <tag-size-flag-modtime-actime-changetime>
+// Extended-Timestamp extra(CDH): <tag-size-flag-modtime>
+func stripExtras(input []byte) []byte {
ret := []byte{}
for len(input) >= 4 {
@@ -87,7 +95,7 @@
if int(size) > len(r) {
break
}
- if tag != zip64ExtraId {
+ if tag != zip64ExtraId && tag != ExtendedTimeStampTag {
ret = append(ret, input[:4+size]...)
}
input = input[4+size:]
@@ -122,7 +130,7 @@
return nil, errors.New("archive/zip: invalid duplicate FileHeader")
}
- fh.Flags |= 0x8 // we will write a data descriptor
+ fh.Flags |= DataDescriptorFlag // we will write a data descriptor
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
fh.ReaderVersion = zipVersion20
@@ -149,6 +157,17 @@
return fw, nil
}
+// Updated version of CreateHeader that doesn't enforce writing a data descriptor
+func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) {
+ writeDataDescriptor := fh.Method != Store
+ if writeDataDescriptor {
+ fh.Flags &= DataDescriptorFlag
+ } else {
+ fh.Flags &= ^uint16(DataDescriptorFlag)
+ }
+ return w.createHeaderImpl(fh)
+}
+
type compressedFileWriter struct {
fileWriter
}
diff --git a/third_party/zip/android_test.go b/third_party/zip/android_test.go
index cdf66ff..5154a17 100644
--- a/third_party/zip/android_test.go
+++ b/third_party/zip/android_test.go
@@ -59,11 +59,16 @@
in: []byte{0, 0, 8, 0, 0, 0},
out: []byte{0, 0, 8, 0, 0, 0},
},
+ {
+ name: "zip64 extra and extended-timestamp extra and valid non-zip64 extra",
+ in: []byte{1, 0, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 85, 84, 5, 0, 1, 1, 2, 3, 4, 2, 0, 0, 0},
+ out: []byte{2, 0, 0, 0},
+ },
}
func TestStripZip64Extras(t *testing.T) {
for _, testcase := range stripZip64Testcases {
- got := stripZip64Extras(testcase.in)
+ got := stripExtras(testcase.in)
if !bytes.Equal(got, testcase.out) {
t.Errorf("Failed testcase %s\ninput: %v\n want: %v\n got: %v\n", testcase.name, testcase.in, testcase.out, got)
}
diff --git a/third_party/zip/writer.go b/third_party/zip/writer.go
index 3a9292e..4c5eb78 100644
--- a/third_party/zip/writer.go
+++ b/third_party/zip/writer.go
@@ -192,6 +192,15 @@
return w.CreateHeader(header)
}
+// BEGIN ANDROID CHANGE separate createHeaderImpl from CreateHeader
+// Legacy version of CreateHeader
+func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
+ fh.Flags |= DataDescriptorFlag // writing a data descriptor
+ return w.createHeaderImpl(fh)
+}
+
+// END ANDROID CHANGE
+
// CreateHeader adds a file to the zip file using the provided FileHeader
// for the file metadata.
// It returns a Writer to which the file contents should be written.
@@ -199,7 +208,10 @@
// The file's contents must be written to the io.Writer before the next
// call to Create, CreateHeader, or Close. The provided FileHeader fh
// must not be modified after a call to CreateHeader.
-func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
+
+// BEGIN ANDROID CHANGE separate createHeaderImpl from CreateHeader
+func (w *Writer) createHeaderImpl(fh *FileHeader) (io.Writer, error) {
+ // END ANDROID CHANGE
if w.last != nil && !w.last.closed {
if err := w.last.close(); err != nil {
return nil, err
@@ -209,9 +221,9 @@
// See https://golang.org/issue/11144 confusion.
return nil, errors.New("archive/zip: invalid duplicate FileHeader")
}
-
- fh.Flags |= 0x8 // we will write a data descriptor
-
+ // BEGIN ANDROID CHANGE move the setting of DataDescriptorFlag into CreateHeader
+ // fh.Flags |= 0x8 // we will write a data descriptor
+ // END ANDROID CHANGE
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
fh.ReaderVersion = zipVersion20
@@ -255,9 +267,32 @@
b.uint16(h.Method)
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
- b.uint32(0) // since we are writing a data descriptor crc32,
- b.uint32(0) // compressed size,
- b.uint32(0) // and uncompressed size should be zero
+ // BEGIN ANDROID CHANGE populate header size fields and crc field if not writing a data descriptor
+ if h.Flags&DataDescriptorFlag != 0 {
+ // since we are writing a data descriptor, these fields should be 0
+ b.uint32(0) // crc32,
+ b.uint32(0) // compressed size,
+ b.uint32(0) // uncompressed size
+ } else {
+ b.uint32(h.CRC32)
+
+ if h.CompressedSize64 > uint32max || h.UncompressedSize64 > uint32max {
+ panic("skipping writing the data descriptor for a 64-bit value is not yet supported")
+ }
+ compressedSize := uint32(h.CompressedSize64)
+ if compressedSize == 0 {
+ compressedSize = h.CompressedSize
+ }
+
+ uncompressedSize := uint32(h.UncompressedSize64)
+ if uncompressedSize == 0 {
+ uncompressedSize = h.UncompressedSize
+ }
+
+ b.uint32(compressedSize)
+ b.uint32(uncompressedSize)
+ }
+ // END ANDROID CHANGE
b.uint16(uint16(len(h.Name)))
b.uint16(uint16(len(h.Extra)))
if _, err := w.Write(buf[:]); err != nil {
@@ -306,7 +341,9 @@
return w.rawCount.Write(p)
}
-func (w *fileWriter) close() error {
+// BEGIN ANDROID CHANGE give the return value a name
+func (w *fileWriter) close() (err error) {
+ // END ANDROID CHANGE
if w.closed {
return errors.New("zip: file closed twice")
}
@@ -330,28 +367,32 @@
fh.UncompressedSize = uint32(fh.UncompressedSize64)
}
- // Write data descriptor. This is more complicated than one would
- // think, see e.g. comments in zipfile.c:putextended() and
- // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
- // The approach here is to write 8 byte sizes if needed without
- // adding a zip64 extra in the local header (too late anyway).
- var buf []byte
- if fh.isZip64() {
- buf = make([]byte, dataDescriptor64Len)
- } else {
- buf = make([]byte, dataDescriptorLen)
+ // BEGIN ANDROID CHANGE only write data descriptor if the flag is set
+ if fh.Flags&DataDescriptorFlag != 0 {
+ // Write data descriptor. This is more complicated than one would
+ // think, see e.g. comments in zipfile.c:putextended() and
+ // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
+ // The approach here is to write 8 byte sizes if needed without
+ // adding a zip64 extra in the local header (too late anyway).
+ var buf []byte
+ if fh.isZip64() {
+ buf = make([]byte, dataDescriptor64Len)
+ } else {
+ buf = make([]byte, dataDescriptorLen)
+ }
+ b := writeBuf(buf)
+ b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
+ b.uint32(fh.CRC32)
+ if fh.isZip64() {
+ b.uint64(fh.CompressedSize64)
+ b.uint64(fh.UncompressedSize64)
+ } else {
+ b.uint32(fh.CompressedSize)
+ b.uint32(fh.UncompressedSize)
+ }
+ _, err = w.zipw.Write(buf)
}
- b := writeBuf(buf)
- b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
- b.uint32(fh.CRC32)
- if fh.isZip64() {
- b.uint64(fh.CompressedSize64)
- b.uint64(fh.UncompressedSize64)
- } else {
- b.uint32(fh.CompressedSize)
- b.uint32(fh.UncompressedSize)
- }
- _, err := w.zipw.Write(buf)
+ // END ANDROID CHANGE
return err
}
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 548baee..5809894 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -19,20 +19,24 @@
"soong-ui-logger",
"soong-ui-tracer",
"soong-shared",
+ "soong-finder",
+ "blueprint-microfactory",
],
srcs: [
"build.go",
"cleanbuild.go",
"config.go",
"context.go",
+ "dumpvars.go",
"environment.go",
"exec.go",
+ "finder.go",
"kati.go",
- "make.go",
"ninja.go",
"proc_sync.go",
"signal.go",
"soong.go",
+ "test_build.go",
"util.go",
],
testSrcs: [
diff --git a/ui/build/build.go b/ui/build/build.go
index 32f4ba5..78eb6a3 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -26,20 +26,31 @@
func SetupOutDir(ctx Context, config Config) {
ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "Android.mk"))
ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "CleanSpec.mk"))
- ensureEmptyFileExists(ctx, filepath.Join(config.SoongOutDir(), ".soong.in_make"))
+ if !config.SkipMake() {
+ ensureEmptyFileExists(ctx, filepath.Join(config.SoongOutDir(), ".soong.in_make"))
+ }
// The ninja_build file is used by our buildbots to understand that the output
// can be parsed as ninja output.
ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build"))
+ ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), ".out-dir"))
}
var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(`
builddir = {{.OutDir}}
-include {{.KatiNinjaFile}}
+{{if .HasKatiSuffix}}include {{.KatiNinjaFile}}
+{{end -}}
include {{.SoongNinjaFile}}
build {{.CombinedNinjaFile}}: phony {{.SoongNinjaFile}}
`))
func createCombinedBuildNinjaFile(ctx Context, config Config) {
+ // If we're in SkipMake mode, skip creating this file if it already exists
+ if config.SkipMake() {
+ if _, err := os.Stat(config.CombinedNinjaFile()); err == nil || !os.IsNotExist(err) {
+ return
+ }
+ }
+
file, err := os.Create(config.CombinedNinjaFile())
if err != nil {
ctx.Fatalln("Failed to create combined ninja file:", err)
@@ -57,6 +68,7 @@
BuildSoong = 1 << iota
BuildKati = 1 << iota
BuildNinja = 1 << iota
+ RunBuildTests = 1 << iota
BuildAll = BuildProductConfig | BuildSoong | BuildKati | BuildNinja
)
@@ -92,9 +104,8 @@
}
func help(ctx Context, config Config, what int) {
- cmd := Command(ctx, config, "make",
- "make", "-f", "build/core/help.mk")
- cmd.Sandbox = makeSandbox
+ cmd := Command(ctx, config, "help.sh", "build/make/help.sh")
+ cmd.Sandbox = dumpvarsSandbox
cmd.Stdout = ctx.Stdout()
cmd.Stderr = ctx.Stderr()
cmd.RunOrFatal()
@@ -106,6 +117,11 @@
ctx.Verboseln("Starting build with args:", config.Arguments())
ctx.Verboseln("Environment:", config.Environment().Environ())
+ if config.SkipMake() {
+ ctx.Verboseln("Skipping Make/Kati as requested")
+ what = what & (BuildSoong | BuildNinja)
+ }
+
if inList("help", config.Arguments()) {
help(ctx, config, what)
return
@@ -141,20 +157,33 @@
if what&BuildSoong != 0 {
// Run Soong
- runSoongBootstrap(ctx, config)
runSoong(ctx, config)
}
if what&BuildKati != 0 {
// Run ckati
runKati(ctx, config)
+
+ ioutil.WriteFile(config.LastKatiSuffixFile(), []byte(config.KatiSuffix()), 0777)
+ } else {
+ // Load last Kati Suffix if it exists
+ if katiSuffix, err := ioutil.ReadFile(config.LastKatiSuffixFile()); err == nil {
+ ctx.Verboseln("Loaded previous kati config:", string(katiSuffix))
+ config.SetKatiSuffix(string(katiSuffix))
+ }
+ }
+
+ // Write combined ninja file
+ createCombinedBuildNinjaFile(ctx, config)
+
+ if what&RunBuildTests != 0 {
+ testForDanglingRules(ctx, config)
}
if what&BuildNinja != 0 {
- installCleanIfNecessary(ctx, config)
-
- // Write combined ninja file
- createCombinedBuildNinjaFile(ctx, config)
+ if !config.SkipMake() {
+ installCleanIfNecessary(ctx, config)
+ }
// Run ninja
runNinja(ctx, config)
diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go
index 2967c3a..f2de2cd 100644
--- a/ui/build/cleanbuild.go
+++ b/ui/build/cleanbuild.go
@@ -129,6 +129,8 @@
suffix := "\n"
currentProduct := prefix + config.TargetProduct() + "-" + config.TargetBuildVariant() + suffix
+ ensureDirectoriesExist(ctx, filepath.Dir(configFile))
+
writeConfig := func() {
err := ioutil.WriteFile(configFile, []byte(currentProduct), 0666)
if err != nil {
diff --git a/ui/build/config.go b/ui/build/config.go
index ef06157..df97d80 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -34,10 +34,12 @@
environ *Environment
// From the arguments
- parallel int
- keepGoing int
- verbose bool
- dist bool
+ parallel int
+ keepGoing int
+ verbose bool
+ checkbuild bool
+ dist bool
+ skipMake bool
// From the product config
katiArgs []string
@@ -103,6 +105,9 @@
"MAKEFLAGS",
"MAKELEVEL",
"MFLAGS",
+
+ // Set in envsetup.sh, reset in makefiles
+ "ANDROID_JAVA_TOOLCHAIN",
)
// Tell python not to spam the source tree with .pyc files.
@@ -116,14 +121,12 @@
log.Fatalln("Error verifying tree state:", err)
}
- if srcDir, err := filepath.Abs("."); err == nil {
- if strings.ContainsRune(srcDir, ' ') {
- log.Println("You are building in a directory whose absolute path contains a space character:")
- log.Println()
- log.Printf("%q\n", srcDir)
- log.Println()
- log.Fatalln("Directory names containing spaces are not supported")
- }
+ if srcDir := absPath(ctx, "."); strings.ContainsRune(srcDir, ' ') {
+ log.Println("You are building in a directory whose absolute path contains a space character:")
+ log.Println()
+ log.Printf("%q\n", srcDir)
+ log.Println()
+ log.Fatalln("Directory names containing spaces are not supported")
}
if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
@@ -142,6 +145,27 @@
log.Fatalln("Directory names containing spaces are not supported")
}
+ // Configure Java-related variables, including adding it to $PATH
+ javaHome := func() string {
+ if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
+ return override
+ }
+ if v, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK9"); ok && v != "" && v != "false" {
+ return filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
+ }
+ return filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
+ }()
+ absJavaHome := absPath(ctx, javaHome)
+
+ newPath := []string{filepath.Join(absJavaHome, "bin")}
+ if path, ok := ret.environ.Get("PATH"); ok && path != "" {
+ newPath = append(newPath, path)
+ }
+ ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
+ ret.environ.Set("JAVA_HOME", absJavaHome)
+ ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
+ ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
+
return Config{ret}
}
@@ -149,14 +173,11 @@
for i := 0; i < len(args); i++ {
arg := strings.TrimSpace(args[i])
if arg == "--make-mode" {
- continue
} else if arg == "showcommands" {
c.verbose = true
- continue
- } else if arg == "dist" {
- c.dist = true
- }
- if arg[0] == '-' {
+ } else if arg == "--skip-make" {
+ c.skipMake = true
+ } else if len(arg) > 0 && arg[0] == '-' {
parseArgNum := func(def int) int {
if len(arg) > 2 {
p, err := strconv.ParseUint(arg[2:], 10, 31)
@@ -174,9 +195,9 @@
return def
}
- if arg[1] == 'j' {
+ if len(arg) > 1 && arg[1] == 'j' {
c.parallel = parseArgNum(c.parallel)
- } else if arg[1] == 'k' {
+ } else if len(arg) > 1 && arg[1] == 'k' {
c.keepGoing = parseArgNum(0)
} else {
ctx.Fatalln("Unknown option:", arg)
@@ -184,6 +205,11 @@
} else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
c.environ.Set(k, v)
} else {
+ if arg == "dist" {
+ c.dist = true
+ } else if arg == "checkbuild" {
+ c.checkbuild = true
+ }
c.arguments = append(c.arguments, arg)
}
}
@@ -265,6 +291,9 @@
}
func (c *configImpl) NinjaArgs() []string {
+ if c.skipMake {
+ return c.arguments
+ }
return c.ninjaArgs
}
@@ -276,6 +305,10 @@
return shared.TempDirForOutDir(c.SoongOutDir())
}
+func (c *configImpl) FileListDir() string {
+ return filepath.Join(c.OutDir(), ".module_paths")
+}
+
func (c *configImpl) KatiSuffix() string {
if c.katiSuffix != "" {
return c.katiSuffix
@@ -283,6 +316,12 @@
panic("SetKatiSuffix has not been called")
}
+// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
+// user is interested in additional checks at the expense of build time.
+func (c *configImpl) Checkbuild() bool {
+ return c.checkbuild
+}
+
func (c *configImpl) Dist() bool {
return c.dist
}
@@ -291,6 +330,10 @@
return c.verbose
}
+func (c *configImpl) SkipMake() bool {
+ return c.skipMake
+}
+
func (c *configImpl) TargetProduct() string {
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
return v
@@ -355,6 +398,14 @@
c.katiSuffix = suffix
}
+func (c *configImpl) LastKatiSuffixFile() string {
+ return filepath.Join(c.OutDir(), "last_kati_suffix")
+}
+
+func (c *configImpl) HasKatiSuffix() bool {
+ return c.katiSuffix != ""
+}
+
func (c *configImpl) KatiEnvFile() string {
return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
}
@@ -368,6 +419,9 @@
}
func (c *configImpl) CombinedNinjaFile() string {
+ if c.katiSuffix == "" {
+ return filepath.Join(c.OutDir(), "combined.ninja")
+ }
return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
}
@@ -380,11 +434,7 @@
}
func (c *configImpl) ProductOut() string {
- if buildType, ok := c.environ.Get("TARGET_BUILD_TYPE"); ok && buildType == "debug" {
- return filepath.Join(c.OutDir(), "debug", "target", "product", c.TargetDevice())
- } else {
- return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
- }
+ return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
}
func (c *configImpl) DevicePreviousProductConfig() string {
@@ -392,11 +442,7 @@
}
func (c *configImpl) hostOutRoot() string {
- if buildType, ok := c.environ.Get("HOST_BUILD_TYPE"); ok && buildType == "debug" {
- return filepath.Join(c.OutDir(), "debug", "host")
- } else {
- return filepath.Join(c.OutDir(), "host")
- }
+ return filepath.Join(c.OutDir(), "host")
}
func (c *configImpl) HostOut() string {
@@ -422,21 +468,13 @@
}
}
-func (c *configImpl) HostAsan() bool {
+func (c *configImpl) PrebuiltBuildTool(name string) string {
if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
if sanitize := strings.Fields(v); inList("address", sanitize) {
- return true
- }
- }
- return false
-}
-
-func (c *configImpl) PrebuiltBuildTool(name string) string {
- // (b/36182021) We're seeing rare ckati crashes, so always enable asan kati on the build servers.
- if c.HostAsan() || (c.Dist() && name == "ckati") {
- asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
- if _, err := os.Stat(asan); err == nil {
- return asan
+ asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
+ if _, err := os.Stat(asan); err == nil {
+ return asan
+ }
}
}
return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
diff --git a/ui/build/make.go b/ui/build/dumpvars.go
similarity index 62%
rename from ui/build/make.go
rename to ui/build/dumpvars.go
index edf6d96..96f2274 100644
--- a/ui/build/make.go
+++ b/ui/build/dumpvars.go
@@ -15,8 +15,8 @@
package build
import (
+ "bytes"
"fmt"
- "path/filepath"
"strings"
)
@@ -28,27 +28,29 @@
// Make without actually building them. So all the variables based on
// MAKECMDGOALS can be read.
//
-// extra_targets adds real arguments to the make command, in case other targets
-// actually need to be run (like the Soong config generator).
-//
// vars is the list of variables to read. The values will be put in the
// returned map.
-func DumpMakeVars(ctx Context, config Config, goals, extra_targets, vars []string) (map[string]string, error) {
+func DumpMakeVars(ctx Context, config Config, goals, vars []string) (map[string]string, error) {
+ return dumpMakeVars(ctx, config, goals, vars, false)
+}
+
+func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_vars bool) (map[string]string, error) {
ctx.BeginTrace("dumpvars")
defer ctx.EndTrace()
- cmd := Command(ctx, config, "make",
- "make",
- "--no-print-directory",
- "-f", "build/core/config.mk",
+ cmd := Command(ctx, config, "dumpvars",
+ config.PrebuiltBuildTool("ckati"),
+ "-f", "build/make/core/config.mk",
+ "--color_warnings",
"dump-many-vars",
- "CALLED_FROM_SETUP=true",
- "BUILD_SYSTEM=build/core",
- "MAKECMDGOALS="+strings.Join(goals, " "),
- "DUMP_MANY_VARS="+strings.Join(vars, " "),
- "OUT_DIR="+config.OutDir())
- cmd.Args = append(cmd.Args, extra_targets...)
- cmd.Sandbox = makeSandbox
+ "MAKECMDGOALS="+strings.Join(goals, " "))
+ cmd.Environment.Set("CALLED_FROM_SETUP", "true")
+ cmd.Environment.Set("BUILD_SYSTEM", "build/make/core")
+ if write_soong_vars {
+ cmd.Environment.Set("WRITE_SOONG_VARIABLES", "true")
+ }
+ cmd.Environment.Set("DUMP_MANY_VARS", strings.Join(vars, " "))
+ cmd.Sandbox = dumpvarsSandbox
// TODO: error out when Stderr contains any content
cmd.Stderr = ctx.Stderr()
output, err := cmd.Output()
@@ -77,6 +79,50 @@
return ret, nil
}
+// Variables to print out in the top banner
+var BannerVars = []string{
+ "PLATFORM_VERSION_CODENAME",
+ "PLATFORM_VERSION",
+ "TARGET_PRODUCT",
+ "TARGET_BUILD_VARIANT",
+ "TARGET_BUILD_TYPE",
+ "TARGET_BUILD_APPS",
+ "TARGET_ARCH",
+ "TARGET_ARCH_VARIANT",
+ "TARGET_CPU_VARIANT",
+ "TARGET_2ND_ARCH",
+ "TARGET_2ND_ARCH_VARIANT",
+ "TARGET_2ND_CPU_VARIANT",
+ "HOST_ARCH",
+ "HOST_2ND_ARCH",
+ "HOST_OS",
+ "HOST_OS_EXTRA",
+ "HOST_CROSS_OS",
+ "HOST_CROSS_ARCH",
+ "HOST_CROSS_2ND_ARCH",
+ "HOST_BUILD_TYPE",
+ "BUILD_ID",
+ "OUT_DIR",
+ "AUX_OS_VARIANT_LIST",
+ "TARGET_BUILD_PDK",
+ "PDK_FUSION_PLATFORM_ZIP",
+ "PRODUCT_SOONG_NAMESPACES",
+}
+
+func Banner(make_vars map[string]string) string {
+ b := &bytes.Buffer{}
+
+ fmt.Fprintln(b, "============================================")
+ for _, name := range BannerVars {
+ if make_vars[name] != "" {
+ fmt.Fprintf(b, "%s=%s\n", name, make_vars[name])
+ }
+ }
+ fmt.Fprint(b, "============================================")
+
+ return b.String()
+}
+
func runMakeProductConfig(ctx Context, config Config) {
// Variables to export into the environment of Kati/Ninja
exportEnvVars := []string{
@@ -98,35 +144,6 @@
"CCACHE_CPP2",
}
- // Variables to print out in the top banner
- bannerVars := []string{
- "PLATFORM_VERSION_CODENAME",
- "PLATFORM_VERSION",
- "TARGET_PRODUCT",
- "TARGET_BUILD_VARIANT",
- "TARGET_BUILD_TYPE",
- "TARGET_BUILD_APPS",
- "TARGET_ARCH",
- "TARGET_ARCH_VARIANT",
- "TARGET_CPU_VARIANT",
- "TARGET_2ND_ARCH",
- "TARGET_2ND_ARCH_VARIANT",
- "TARGET_2ND_CPU_VARIANT",
- "HOST_ARCH",
- "HOST_2ND_ARCH",
- "HOST_OS",
- "HOST_OS_EXTRA",
- "HOST_CROSS_OS",
- "HOST_CROSS_ARCH",
- "HOST_CROSS_2ND_ARCH",
- "HOST_BUILD_TYPE",
- "BUILD_ID",
- "OUT_DIR",
- "AUX_OS_VARIANT_LIST",
- "TARGET_BUILD_PDK",
- "PDK_FUSION_PLATFORM_ZIP",
- }
-
allVars := append(append([]string{
// Used to execute Kati and Ninja
"NINJA_GOALS",
@@ -134,23 +151,15 @@
// To find target/product/<DEVICE>
"TARGET_DEVICE",
- }, exportEnvVars...), bannerVars...)
+ }, exportEnvVars...), BannerVars...)
- make_vars, err := DumpMakeVars(ctx, config, config.Arguments(), []string{
- filepath.Join(config.SoongOutDir(), "soong.variables"),
- }, allVars)
+ make_vars, err := dumpMakeVars(ctx, config, config.Arguments(), allVars, true)
if err != nil {
ctx.Fatalln("Error dumping make vars:", err)
}
// Print the banner like make does
- fmt.Fprintln(ctx.Stdout(), "============================================")
- for _, name := range bannerVars {
- if make_vars[name] != "" {
- fmt.Fprintf(ctx.Stdout(), "%s=%s\n", name, make_vars[name])
- }
- }
- fmt.Fprintln(ctx.Stdout(), "============================================")
+ fmt.Fprintln(ctx.Stdout(), Banner(make_vars))
// Populate the environment
env := config.Environment()
diff --git a/ui/build/finder.go b/ui/build/finder.go
new file mode 100644
index 0000000..a0f5d08
--- /dev/null
+++ b/ui/build/finder.go
@@ -0,0 +1,115 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "android/soong/finder"
+ "android/soong/fs"
+ "android/soong/ui/logger"
+ "bytes"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// This file provides an interface to the Finder for use in Soong UI
+// This file stores configuration information about which files to find
+
+// NewSourceFinder returns a new Finder configured to search for source files.
+// Callers of NewSourceFinder should call <f.Shutdown()> when done
+func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
+ ctx.BeginTrace("find modules")
+ defer ctx.EndTrace()
+
+ dir, err := os.Getwd()
+ if err != nil {
+ ctx.Fatalf("No working directory for module-finder: %v", err.Error())
+ }
+ filesystem := fs.OsFs
+
+ // if the root dir is ignored, then the subsequent error messages are very confusing,
+ // so check for that upfront
+ pruneFiles := []string{".out-dir", ".find-ignore"}
+ for _, name := range pruneFiles {
+ prunePath := filepath.Join(dir, name)
+ _, statErr := filesystem.Lstat(prunePath)
+ if statErr == nil {
+ ctx.Fatalf("%v must not exist", prunePath)
+ }
+ }
+
+ cacheParams := finder.CacheParams{
+ WorkingDirectory: dir,
+ RootDirs: []string{"."},
+ ExcludeDirs: []string{".git", ".repo"},
+ PruneFiles: pruneFiles,
+ IncludeFiles: []string{"Android.mk", "Android.bp", "Blueprints", "CleanSpec.mk", "TEST_MAPPING"},
+ }
+ dumpDir := config.FileListDir()
+ f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
+ filepath.Join(dumpDir, "files.db"))
+ if err != nil {
+ ctx.Fatalf("Could not create module-finder: %v", err)
+ }
+ return f
+}
+
+// FindSources searches for source files known to <f> and writes them to the filesystem for
+// use later.
+func FindSources(ctx Context, config Config, f *finder.Finder) {
+ // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder
+ // if a caller such as multiproduct_kati wants to share one Finder among several builds
+ dumpDir := config.FileListDir()
+ os.MkdirAll(dumpDir, 0777)
+
+ androidMks := f.FindFirstNamedAt(".", "Android.mk")
+ err := dumpListToFile(androidMks, filepath.Join(dumpDir, "Android.mk.list"))
+ if err != nil {
+ ctx.Fatalf("Could not export module list: %v", err)
+ }
+
+ cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
+ dumpListToFile(cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
+ if err != nil {
+ ctx.Fatalf("Could not export module list: %v", err)
+ }
+
+ testMappings := f.FindNamedAt(".", "TEST_MAPPING")
+ err = dumpListToFile(testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list"))
+ if err != nil {
+ ctx.Fatalf("Could not find modules: %v", err)
+ }
+
+ androidBps := f.FindNamedAt(".", "Android.bp")
+ androidBps = append(androidBps, f.FindNamedAt("build/blueprint", "Blueprints")...)
+ if len(androidBps) == 0 {
+ ctx.Fatalf("No Android.bp found")
+ }
+ err = dumpListToFile(androidBps, filepath.Join(dumpDir, "Android.bp.list"))
+ if err != nil {
+ ctx.Fatalf("Could not find modules: %v", err)
+ }
+}
+
+func dumpListToFile(list []string, filePath string) (err error) {
+ desiredText := strings.Join(list, "\n")
+ desiredBytes := []byte(desiredText)
+ actualBytes, readErr := ioutil.ReadFile(filePath)
+ if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
+ err = ioutil.WriteFile(filePath, desiredBytes, 0777)
+ }
+ return err
+}
diff --git a/ui/build/kati.go b/ui/build/kati.go
index 48c38d4..7bb721d 100644
--- a/ui/build/kati.go
+++ b/ui/build/kati.go
@@ -59,11 +59,13 @@
}
func runKati(ctx Context, config Config) {
+ genKatiSuffix(ctx, config)
+
+ runKatiCleanSpec(ctx, config)
+
ctx.BeginTrace("kati")
defer ctx.EndTrace()
- genKatiSuffix(ctx, config)
-
executable := config.PrebuiltBuildTool("ckati")
args := []string{
"--ninja",
@@ -75,7 +77,8 @@
"--color_warnings",
"--gen_all_targets",
"--werror_find_emulator",
- "-f", "build/core/main.mk",
+ "--kati_stats",
+ "-f", "build/make/core/main.mk",
}
if !config.Environment().IsFalse("KATI_EMULATE_FIND") {
@@ -101,15 +104,13 @@
}
cmd.Stderr = cmd.Stdout
- // Kati leaks memory, so ensure leak detection is turned off
- cmd.Environment.Set("ASAN_OPTIONS", "detect_leaks=0")
-
cmd.StartOrFatal()
katiRewriteOutput(ctx, pipe)
cmd.WaitOrFatal()
}
var katiIncludeRe = regexp.MustCompile(`^(\[\d+/\d+] )?including [^ ]+ ...$`)
+var katiLogRe = regexp.MustCompile(`^\*kati\*: `)
func katiRewriteOutput(ctx Context, pipe io.ReadCloser) {
haveBlankLine := true
@@ -120,6 +121,12 @@
line := scanner.Text()
verbose := katiIncludeRe.MatchString(line)
+ // Only put kati debug/stat lines in our verbose log
+ if katiLogRe.MatchString(line) {
+ ctx.Verbose(line)
+ continue
+ }
+
// For verbose lines, write them on the current line without a newline,
// then overwrite them if the next thing we're printing is another
// verbose line.
@@ -162,3 +169,33 @@
fmt.Fprintln(ctx.Stdout())
}
}
+
+func runKatiCleanSpec(ctx Context, config Config) {
+ ctx.BeginTrace("kati cleanspec")
+ defer ctx.EndTrace()
+
+ executable := config.PrebuiltBuildTool("ckati")
+ args := []string{
+ "--ninja",
+ "--ninja_dir=" + config.OutDir(),
+ "--ninja_suffix=" + config.KatiSuffix() + "-cleanspec",
+ "--regen",
+ "--detect_android_echo",
+ "--color_warnings",
+ "--gen_all_targets",
+ "--werror_find_emulator",
+ "--use_find_emulator",
+ "-f", "build/make/core/cleanbuild.mk",
+ "BUILDING_WITH_NINJA=true",
+ "SOONG_MAKEVARS_MK=" + config.SoongMakeVarsMk(),
+ }
+
+ cmd := Command(ctx, config, "ckati", executable, args...)
+ cmd.Sandbox = katiCleanSpecSandbox
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+
+ // Kati leaks memory, so ensure leak detection is turned off
+ cmd.Environment.Set("ASAN_OPTIONS", "detect_leaks=0")
+ cmd.RunOrFatal()
+}
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
index 78d1170..96b5e9d 100644
--- a/ui/build/ninja.go
+++ b/ui/build/ninja.go
@@ -53,7 +53,9 @@
args = append(args, "-w", "dupbuild=err")
cmd := Command(ctx, config, "ninja", executable, args...)
- cmd.Environment.AppendFromKati(config.KatiEnvFile())
+ if config.HasKatiSuffix() {
+ cmd.Environment.AppendFromKati(config.KatiEnvFile())
+ }
// Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been
// used in the past to specify extra ninja arguments.
diff --git a/ui/build/sandbox_darwin.go b/ui/build/sandbox_darwin.go
index 54c145c..7e75167 100644
--- a/ui/build/sandbox_darwin.go
+++ b/ui/build/sandbox_darwin.go
@@ -16,17 +16,17 @@
import (
"os/exec"
- "path/filepath"
)
type Sandbox string
const (
- noSandbox = ""
- globalSandbox = "build/soong/ui/build/sandbox/darwin/global.sb"
- makeSandbox = globalSandbox
- soongSandbox = globalSandbox
- katiSandbox = globalSandbox
+ noSandbox = ""
+ globalSandbox = "build/soong/ui/build/sandbox/darwin/global.sb"
+ dumpvarsSandbox = globalSandbox
+ soongSandbox = globalSandbox
+ katiSandbox = globalSandbox
+ katiCleanSpecSandbox = globalSandbox
)
var sandboxExecPath string
@@ -49,14 +49,8 @@
func (c *Cmd) wrapSandbox() {
homeDir, _ := c.Environment.Get("HOME")
- outDir, err := filepath.Abs(c.config.OutDir())
- if err != nil {
- c.ctx.Fatalln("Failed to get absolute path of OUT_DIR:", err)
- }
- distDir, err := filepath.Abs(c.config.DistDir())
- if err != nil {
- c.ctx.Fatalln("Failed to get absolute path of DIST_DIR:", err)
- }
+ outDir := absPath(c.ctx, c.config.OutDir())
+ distDir := absPath(c.ctx, c.config.DistDir())
c.Args[0] = c.Path
c.Path = sandboxExecPath
diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go
index fb48b9c..f2bfac2 100644
--- a/ui/build/sandbox_linux.go
+++ b/ui/build/sandbox_linux.go
@@ -17,11 +17,12 @@
type Sandbox bool
const (
- noSandbox = false
- globalSandbox = false
- makeSandbox = false
- soongSandbox = false
- katiSandbox = false
+ noSandbox = false
+ globalSandbox = false
+ dumpvarsSandbox = false
+ soongSandbox = false
+ katiSandbox = false
+ katiCleanSpecSandbox = false
)
func (c *Cmd) sandboxSupported() bool {
diff --git a/ui/build/soong.go b/ui/build/soong.go
index d242805..6dafd27 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,35 +15,100 @@
package build
import (
+ "os"
"path/filepath"
+ "strconv"
+ "time"
+
+ "github.com/google/blueprint/microfactory"
)
-func runSoongBootstrap(ctx Context, config Config) {
- ctx.BeginTrace("bootstrap soong")
- defer ctx.EndTrace()
-
- cmd := Command(ctx, config, "soong bootstrap", "./bootstrap.bash")
- cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
- cmd.Environment.Set("NINJA_BUILDDIR", config.OutDir())
- cmd.Sandbox = soongSandbox
- cmd.Stdout = ctx.Stdout()
- cmd.Stderr = ctx.Stderr()
- cmd.RunOrFatal()
-}
-
func runSoong(ctx Context, config Config) {
ctx.BeginTrace("soong")
defer ctx.EndTrace()
- cmd := Command(ctx, config, "soong",
- filepath.Join(config.SoongOutDir(), "soong"), "-w", "dupbuild=err")
- if config.IsVerbose() {
- cmd.Args = append(cmd.Args, "-v")
+ func() {
+ ctx.BeginTrace("blueprint bootstrap")
+ defer ctx.EndTrace()
+
+ cmd := Command(ctx, config, "blueprint bootstrap", "build/blueprint/bootstrap.bash", "-t")
+ cmd.Environment.Set("BLUEPRINTDIR", "./build/blueprint")
+ cmd.Environment.Set("BOOTSTRAP", "./build/blueprint/bootstrap.bash")
+ cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
+ cmd.Environment.Set("GOROOT", filepath.Join("./prebuilts/go", config.HostPrebuiltTag()))
+ cmd.Environment.Set("BLUEPRINT_LIST_FILE", filepath.Join(config.FileListDir(), "Android.bp.list"))
+ cmd.Environment.Set("NINJA_BUILDDIR", config.OutDir())
+ cmd.Environment.Set("SRCDIR", ".")
+ cmd.Environment.Set("TOPNAME", "Android.bp")
+ cmd.Sandbox = soongSandbox
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ cmd.RunOrFatal()
+ }()
+
+ func() {
+ ctx.BeginTrace("environment check")
+ defer ctx.EndTrace()
+
+ envFile := filepath.Join(config.SoongOutDir(), ".soong.environment")
+ envTool := filepath.Join(config.SoongOutDir(), ".bootstrap/bin/soong_env")
+ if _, err := os.Stat(envFile); err == nil {
+ if _, err := os.Stat(envTool); err == nil {
+ cmd := Command(ctx, config, "soong_env", envTool, envFile)
+ cmd.Sandbox = soongSandbox
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ if err := cmd.Run(); err != nil {
+ ctx.Verboseln("soong_env failed, forcing manifest regeneration")
+ os.Remove(envFile)
+ }
+ } else {
+ ctx.Verboseln("Missing soong_env tool, forcing manifest regeneration")
+ os.Remove(envFile)
+ }
+ } else if !os.IsNotExist(err) {
+ ctx.Fatalf("Failed to stat %f: %v", envFile, err)
+ }
+ }()
+
+ func() {
+ ctx.BeginTrace("minibp")
+ defer ctx.EndTrace()
+
+ var cfg microfactory.Config
+ cfg.Map("github.com/google/blueprint", "build/blueprint")
+
+ cfg.TrimPath = absPath(ctx, ".")
+
+ minibp := filepath.Join(config.SoongOutDir(), ".minibootstrap/minibp")
+ if _, err := microfactory.Build(&cfg, minibp, "github.com/google/blueprint/bootstrap/minibp"); err != nil {
+ ctx.Fatalln("Failed to build minibp:", err)
+ }
+ }()
+
+ ninja := func(name, file string) {
+ ctx.BeginTrace(name)
+ defer ctx.EndTrace()
+
+ cmd := Command(ctx, config, "soong "+name,
+ config.PrebuiltBuildTool("ninja"),
+ "-d", "keepdepfile",
+ "-w", "dupbuild=err",
+ "-j", strconv.Itoa(config.Parallel()),
+ "-f", filepath.Join(config.SoongOutDir(), file))
+ if config.IsVerbose() {
+ cmd.Args = append(cmd.Args, "-v")
+ }
+ cmd.Environment.Set("GOROOT", filepath.Join("./prebuilts/go", config.HostPrebuiltTag()))
+ cmd.Sandbox = soongSandbox
+ cmd.Stdin = ctx.Stdin()
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+
+ defer ctx.ImportNinjaLog(filepath.Join(config.OutDir(), ".ninja_log"), time.Now())
+ cmd.RunOrFatal()
}
- cmd.Environment.Set("SKIP_NINJA", "true")
- cmd.Sandbox = soongSandbox
- cmd.Stdin = ctx.Stdin()
- cmd.Stdout = ctx.Stdout()
- cmd.Stderr = ctx.Stderr()
- cmd.RunOrFatal()
+
+ ninja("minibootstrap", ".minibootstrap/build.ninja")
+ ninja("bootstrap", ".bootstrap/build.ninja")
}
diff --git a/ui/build/test_build.go b/ui/build/test_build.go
new file mode 100644
index 0000000..940f0c8
--- /dev/null
+++ b/ui/build/test_build.go
@@ -0,0 +1,85 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "bufio"
+ "path/filepath"
+ "runtime"
+ "strings"
+)
+
+// Checks for files in the out directory that have a rule that depends on them but no rule to
+// create them. This catches a common set of build failures where a rule to generate a file is
+// deleted (either by deleting a module in an Android.mk file, or by modifying the build system
+// incorrectly). These failures are often not caught by a local incremental build because the
+// previously built files are still present in the output directory.
+func testForDanglingRules(ctx Context, config Config) {
+ // Many modules are disabled on mac. Checking for dangling rules would cause lots of build
+ // breakages, and presubmit wouldn't catch them, so just disable the check.
+ if runtime.GOOS != "linux" {
+ return
+ }
+
+ ctx.BeginTrace("test for dangling rules")
+ defer ctx.EndTrace()
+
+ // Get a list of leaf nodes in the dependency graph from ninja
+ executable := config.PrebuiltBuildTool("ninja")
+
+ args := []string{}
+ args = append(args, config.NinjaArgs()...)
+ args = append(args, "-f", config.CombinedNinjaFile())
+ args = append(args, "-t", "targets", "rule")
+
+ cmd := Command(ctx, config, "ninja", executable, args...)
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ ctx.Fatal(err)
+ }
+
+ cmd.StartOrFatal()
+
+ outDir := config.OutDir()
+ bootstrapDir := filepath.Join(outDir, "soong", ".bootstrap")
+ miniBootstrapDir := filepath.Join(outDir, "soong", ".minibootstrap")
+
+ var danglingRules []string
+
+ scanner := bufio.NewScanner(stdout)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if !strings.HasPrefix(line, outDir) {
+ // Leaf node is not in the out directory.
+ continue
+ }
+ if strings.HasPrefix(line, bootstrapDir) || strings.HasPrefix(line, miniBootstrapDir) {
+ // Leaf node is in one of Soong's bootstrap directories, which do not have
+ // full build rules in the primary build.ninja file.
+ continue
+ }
+ danglingRules = append(danglingRules, line)
+ }
+
+ cmd.WaitOrFatal()
+
+ if len(danglingRules) > 0 {
+ ctx.Println("Dependencies in out found with no rule to create them:")
+ for _, dep := range danglingRules {
+ ctx.Println(dep)
+ }
+ ctx.Fatal("")
+ }
+}
diff --git a/ui/build/util.go b/ui/build/util.go
index 2555e8a..f698ccd 100644
--- a/ui/build/util.go
+++ b/ui/build/util.go
@@ -24,6 +24,14 @@
"unsafe"
)
+func absPath(ctx Context, p string) string {
+ ret, err := filepath.Abs(p)
+ if err != nil {
+ ctx.Fatalf("Failed to get absolute path: %v", err)
+ }
+ return ret
+}
+
// indexList finds the index of a string in a []string
func indexList(s string, list []string) int {
for i, l := range list {
diff --git a/ui/logger/logger.go b/ui/logger/logger.go
index db7e82a..15c413d 100644
--- a/ui/logger/logger.go
+++ b/ui/logger/logger.go
@@ -38,6 +38,7 @@
"path/filepath"
"strconv"
"sync"
+ "syscall"
)
type Logger interface {
@@ -93,10 +94,19 @@
// existing files to <filename>.#.<ext>, keeping up to maxCount files.
// <filename>.1.<ext> is the most recent backup, <filename>.2.<ext> is the
// second most recent backup, etc.
-//
-// TODO: This function is not guaranteed to be atomic, if there are multiple
-// users attempting to do the same operation, the result is undefined.
func CreateFileWithRotation(filename string, maxCount int) (*os.File, error) {
+ lockFileName := filepath.Join(filepath.Dir(filename), ".lock_"+filepath.Base(filename))
+ lockFile, err := os.OpenFile(lockFileName, os.O_RDWR|os.O_CREATE, 0666)
+ if err != nil {
+ return nil, err
+ }
+ defer lockFile.Close()
+
+ err = syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX)
+ if err != nil {
+ return nil, err
+ }
+
if _, err := os.Lstat(filename); err == nil {
ext := filepath.Ext(filename)
basename := filename[:len(filename)-len(ext)]
@@ -145,13 +155,14 @@
// SetVerbose controls whether Verbose[f|ln] logs to stderr as well as the
// file-backed log.
-func (s *stdLogger) SetVerbose(v bool) {
+func (s *stdLogger) SetVerbose(v bool) *stdLogger {
s.verbose = v
+ return s
}
// SetOutput controls where the file-backed log will be saved. It will keep
// some number of backups of old log files.
-func (s *stdLogger) SetOutput(path string) {
+func (s *stdLogger) SetOutput(path string) *stdLogger {
if f, err := CreateFileWithRotation(path, 5); err == nil {
s.mutex.Lock()
defer s.mutex.Unlock()
@@ -164,6 +175,7 @@
} else {
s.Fatal(err.Error())
}
+ return s
}
// Close disables logging to the file and closes the file handle.
diff --git a/ui/logger/logger_test.go b/ui/logger/logger_test.go
index 0f88ab3..dc6f2e9 100644
--- a/ui/logger/logger_test.go
+++ b/ui/logger/logger_test.go
@@ -67,7 +67,7 @@
t.Fatalf("Failed to read dir: %v", err)
}
sort.Strings(names)
- expected := []string{"build.1.log", "build.2.log", "build.3.log", "build.log"}
+ expected := []string{".lock_build.log", "build.1.log", "build.2.log", "build.3.log", "build.log"}
if !reflect.DeepEqual(names, expected) {
t.Errorf("File list does not match.")
t.Errorf(" got: %v", names)
diff --git a/ui/tracer/microfactory.go b/ui/tracer/microfactory.go
index 320d9d8..acb9be4 100644
--- a/ui/tracer/microfactory.go
+++ b/ui/tracer/microfactory.go
@@ -28,7 +28,7 @@
f, err := os.Open(filename)
if err != nil {
- t.log.Println("Error opening microfactory trace:", err)
+ t.log.Verboseln("Error opening microfactory trace:", err)
return
}
defer f.Close()
diff --git a/cmd/soong_zip/Android.bp b/zip/Android.bp
similarity index 74%
copy from cmd/soong_zip/Android.bp
copy to zip/Android.bp
index 10896ce..3bb4f25 100644
--- a/cmd/soong_zip/Android.bp
+++ b/zip/Android.bp
@@ -12,11 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-blueprint_go_binary {
- name: "soong_zip",
- deps: ["android-archive-zip"],
+subdirs = ["cmd"]
+
+bootstrap_go_package {
+ name: "soong-zip",
+ pkgPath: "android/soong/zip",
+ deps: [
+ "android-archive-zip",
+ "blueprint-pathtools",
+ "soong-jar",
+ ],
srcs: [
- "soong_zip.go",
+ "zip.go",
"rate_limit.go",
],
}
+
diff --git a/cmd/soong_zip/Android.bp b/zip/cmd/Android.bp
similarity index 88%
rename from cmd/soong_zip/Android.bp
rename to zip/cmd/Android.bp
index 10896ce..6029a69 100644
--- a/cmd/soong_zip/Android.bp
+++ b/zip/cmd/Android.bp
@@ -14,9 +14,10 @@
blueprint_go_binary {
name: "soong_zip",
- deps: ["android-archive-zip"],
+ deps: [
+ "soong-zip",
+ ],
srcs: [
- "soong_zip.go",
- "rate_limit.go",
+ "main.go",
],
}
diff --git a/zip/cmd/main.go b/zip/cmd/main.go
new file mode 100644
index 0000000..c0418f7
--- /dev/null
+++ b/zip/cmd/main.go
@@ -0,0 +1,173 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+
+ "android/soong/zip"
+)
+
+type byteReaderCloser struct {
+ *bytes.Reader
+ io.Closer
+}
+
+type pathMapping struct {
+ dest, src string
+ zipMethod uint16
+}
+
+type uniqueSet map[string]bool
+
+func (u *uniqueSet) String() string {
+ return `""`
+}
+
+func (u *uniqueSet) Set(s string) error {
+ if _, found := (*u)[s]; found {
+ return fmt.Errorf("File %q was specified twice as a file to not deflate", s)
+ } else {
+ (*u)[s] = true
+ }
+
+ return nil
+}
+
+type file struct{}
+
+type listFiles struct{}
+
+type dir struct{}
+
+func (f *file) String() string {
+ return `""`
+}
+
+func (f *file) Set(s string) error {
+ if *relativeRoot == "" {
+ return fmt.Errorf("must pass -C before -f")
+ }
+
+ fArgs = append(fArgs, zip.FileArg{
+ PathPrefixInZip: filepath.Clean(*rootPrefix),
+ SourcePrefixToStrip: filepath.Clean(*relativeRoot),
+ SourceFiles: []string{s},
+ })
+
+ return nil
+}
+
+func (l *listFiles) String() string {
+ return `""`
+}
+
+func (l *listFiles) Set(s string) error {
+ if *relativeRoot == "" {
+ return fmt.Errorf("must pass -C before -l")
+ }
+
+ list, err := ioutil.ReadFile(s)
+ if err != nil {
+ return err
+ }
+
+ fArgs = append(fArgs, zip.FileArg{
+ PathPrefixInZip: filepath.Clean(*rootPrefix),
+ SourcePrefixToStrip: filepath.Clean(*relativeRoot),
+ SourceFiles: strings.Split(string(list), "\n"),
+ })
+
+ return nil
+}
+
+func (d *dir) String() string {
+ return `""`
+}
+
+func (d *dir) Set(s string) error {
+ if *relativeRoot == "" {
+ return fmt.Errorf("must pass -C before -D")
+ }
+
+ fArgs = append(fArgs, zip.FileArg{
+ PathPrefixInZip: filepath.Clean(*rootPrefix),
+ SourcePrefixToStrip: filepath.Clean(*relativeRoot),
+ GlobDir: filepath.Clean(s),
+ })
+
+ return nil
+}
+
+var (
+ out = flag.String("o", "", "file to write zip file to")
+ manifest = flag.String("m", "", "input jar manifest file name")
+ directories = flag.Bool("d", false, "include directories in zip")
+ rootPrefix = flag.String("P", "", "path prefix within the zip at which to place files")
+ relativeRoot = flag.String("C", "", "path to use as relative root of files in following -f, -l, or -D arguments")
+ parallelJobs = flag.Int("j", runtime.NumCPU(), "number of parallel threads to use")
+ compLevel = flag.Int("L", 5, "deflate compression level (0-9)")
+ emulateJar = flag.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'")
+ writeIfChanged = flag.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
+
+ fArgs zip.FileArgs
+ nonDeflatedFiles = make(uniqueSet)
+
+ cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file")
+ traceFile = flag.String("trace", "", "write trace to file")
+)
+
+func init() {
+ flag.Var(&listFiles{}, "l", "file containing list of .class files")
+ flag.Var(&dir{}, "D", "directory to include in zip")
+ flag.Var(&file{}, "f", "file to include in zip")
+ flag.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression")
+}
+
+func usage() {
+ fmt.Fprintf(os.Stderr, "usage: zip -o zipfile [-m manifest] -C dir [-f|-l file]...\n")
+ flag.PrintDefaults()
+ os.Exit(2)
+}
+
+func main() {
+ flag.Parse()
+
+ err := zip.Run(zip.ZipArgs{
+ FileArgs: fArgs,
+ OutputFilePath: *out,
+ CpuProfileFilePath: *cpuProfile,
+ TraceFilePath: *traceFile,
+ EmulateJar: *emulateJar,
+ AddDirectoryEntriesToZip: *directories,
+ CompressionLevel: *compLevel,
+ ManifestSourcePath: *manifest,
+ NumParallelJobs: *parallelJobs,
+ NonDeflatedFiles: nonDeflatedFiles,
+ WriteIfChanged: *writeIfChanged,
+ })
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err.Error())
+ os.Exit(1)
+ }
+}
diff --git a/zip/rate_limit.go b/zip/rate_limit.go
new file mode 100644
index 0000000..0ea2ef0
--- /dev/null
+++ b/zip/rate_limit.go
@@ -0,0 +1,152 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zip
+
+import (
+ "fmt"
+ "runtime"
+)
+
+type RateLimit struct {
+ requests chan request
+ completions chan int64
+
+ stop chan struct{}
+}
+
+type request struct {
+ size int64
+ serviced chan struct{}
+}
+
+// NewRateLimit starts a new rate limiter that permits the usage of up to <capacity> at once,
+// except when no capacity is in use, in which case the first caller is always permitted
+func NewRateLimit(capacity int64) *RateLimit {
+ ret := &RateLimit{
+ requests: make(chan request),
+ completions: make(chan int64),
+
+ stop: make(chan struct{}),
+ }
+
+ go ret.monitorChannels(capacity)
+
+ return ret
+}
+
+// RequestExecution blocks until another execution of size <size> can be allowed to run.
+func (r *RateLimit) Request(size int64) {
+ request := request{
+ size: size,
+ serviced: make(chan struct{}, 1),
+ }
+
+ // wait for the request to be received
+ r.requests <- request
+
+ // wait for the request to be accepted
+ <-request.serviced
+}
+
+// Finish declares the completion of an execution of size <size>
+func (r *RateLimit) Finish(size int64) {
+ r.completions <- size
+}
+
+// Stop the background goroutine
+func (r *RateLimit) Stop() {
+ close(r.stop)
+}
+
+// monitorChannels processes incoming requests from channels
+func (r *RateLimit) monitorChannels(capacity int64) {
+ var usedCapacity int64
+ var currentRequest *request
+
+ for {
+ var requests chan request
+ if currentRequest == nil {
+ // If we don't already have a queued request, then we should check for a new request
+ requests = r.requests
+ }
+
+ select {
+ case newRequest := <-requests:
+ currentRequest = &newRequest
+ case amountCompleted := <-r.completions:
+ usedCapacity -= amountCompleted
+
+ if usedCapacity < 0 {
+ panic(fmt.Sprintf("usedCapacity < 0: %v (decreased by %v)", usedCapacity, amountCompleted))
+ }
+ case <-r.stop:
+ return
+ }
+
+ if currentRequest != nil {
+ accepted := false
+ if usedCapacity == 0 {
+ accepted = true
+ } else {
+ if capacity >= usedCapacity+currentRequest.size {
+ accepted = true
+ }
+ }
+ if accepted {
+ usedCapacity += currentRequest.size
+ currentRequest.serviced <- struct{}{}
+ currentRequest = nil
+ }
+ }
+ }
+}
+
+// A CPURateLimiter limits the number of active calls based on CPU requirements
+type CPURateLimiter struct {
+ impl *RateLimit
+}
+
+func NewCPURateLimiter(capacity int64) *CPURateLimiter {
+ if capacity <= 0 {
+ capacity = int64(runtime.NumCPU())
+ }
+ impl := NewRateLimit(capacity)
+ return &CPURateLimiter{impl: impl}
+}
+
+func (e CPURateLimiter) Request() {
+ e.impl.Request(1)
+}
+
+func (e CPURateLimiter) Finish() {
+ e.impl.Finish(1)
+}
+
+func (e CPURateLimiter) Stop() {
+ e.impl.Stop()
+}
+
+// A MemoryRateLimiter limits the number of active calls based on Memory requirements
+type MemoryRateLimiter struct {
+ *RateLimit
+}
+
+func NewMemoryRateLimiter(capacity int64) *MemoryRateLimiter {
+ if capacity <= 0 {
+ capacity = 512 * 1024 * 1024 // 512MB
+ }
+ impl := NewRateLimit(capacity)
+ return &MemoryRateLimiter{RateLimit: impl}
+}
diff --git a/zip/zip.go b/zip/zip.go
new file mode 100644
index 0000000..c878a0c
--- /dev/null
+++ b/zip/zip.go
@@ -0,0 +1,782 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zip
+
+import (
+ "bytes"
+ "compress/flate"
+ "errors"
+ "fmt"
+ "hash/crc32"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+ "runtime/pprof"
+ "runtime/trace"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/google/blueprint/pathtools"
+
+ "android/soong/jar"
+ "android/soong/third_party/zip"
+)
+
+// Block size used during parallel compression of a single file.
+const parallelBlockSize = 1 * 1024 * 1024 // 1MB
+
+// Minimum file size to use parallel compression. It requires more
+// flate.Writer allocations, since we can't change the dictionary
+// during Reset
+const minParallelFileSize = parallelBlockSize * 6
+
+// Size of the ZIP compression window (32KB)
+const windowSize = 32 * 1024
+
+type nopCloser struct {
+ io.Writer
+}
+
+func (nopCloser) Close() error {
+ return nil
+}
+
+type byteReaderCloser struct {
+ *bytes.Reader
+ io.Closer
+}
+
+type pathMapping struct {
+ dest, src string
+ zipMethod uint16
+}
+
+type uniqueSet map[string]bool
+
+func (u *uniqueSet) String() string {
+ return `""`
+}
+
+func (u *uniqueSet) Set(s string) error {
+ if _, found := (*u)[s]; found {
+ return fmt.Errorf("File %q was specified twice as a file to not deflate", s)
+ } else {
+ (*u)[s] = true
+ }
+
+ return nil
+}
+
+type FileArg struct {
+ PathPrefixInZip, SourcePrefixToStrip string
+ SourceFiles []string
+ GlobDir string
+}
+
+type FileArgs []FileArg
+
+type ZipWriter struct {
+ time time.Time
+ createdFiles map[string]string
+ createdDirs map[string]string
+ directories bool
+
+ errors chan error
+ writeOps chan chan *zipEntry
+
+ cpuRateLimiter *CPURateLimiter
+ memoryRateLimiter *MemoryRateLimiter
+
+ compressorPool sync.Pool
+ compLevel int
+}
+
+type zipEntry struct {
+ fh *zip.FileHeader
+
+ // List of delayed io.Reader
+ futureReaders chan chan io.Reader
+
+ // Only used for passing into the MemoryRateLimiter to ensure we
+ // release as much memory as much as we request
+ allocatedSize int64
+}
+
+type ZipArgs struct {
+ FileArgs FileArgs
+ OutputFilePath string
+ CpuProfileFilePath string
+ TraceFilePath string
+ EmulateJar bool
+ AddDirectoryEntriesToZip bool
+ CompressionLevel int
+ ManifestSourcePath string
+ NumParallelJobs int
+ NonDeflatedFiles map[string]bool
+ WriteIfChanged bool
+}
+
+func Run(args ZipArgs) (err error) {
+ if args.CpuProfileFilePath != "" {
+ f, err := os.Create(args.CpuProfileFilePath)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err.Error())
+ os.Exit(1)
+ }
+ defer f.Close()
+ pprof.StartCPUProfile(f)
+ defer pprof.StopCPUProfile()
+ }
+
+ if args.TraceFilePath != "" {
+ f, err := os.Create(args.TraceFilePath)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err.Error())
+ os.Exit(1)
+ }
+ defer f.Close()
+ err = trace.Start(f)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err.Error())
+ os.Exit(1)
+ }
+ defer trace.Stop()
+ }
+
+ if args.OutputFilePath == "" {
+ return fmt.Errorf("output file path must be nonempty")
+ }
+
+ if args.EmulateJar {
+ args.AddDirectoryEntriesToZip = true
+ }
+
+ w := &ZipWriter{
+ time: jar.DefaultTime,
+ createdDirs: make(map[string]string),
+ createdFiles: make(map[string]string),
+ directories: args.AddDirectoryEntriesToZip,
+ compLevel: args.CompressionLevel,
+ }
+ pathMappings := []pathMapping{}
+
+ for _, fa := range args.FileArgs {
+ srcs := fa.SourceFiles
+ if fa.GlobDir != "" {
+ srcs = append(srcs, recursiveGlobFiles(fa.GlobDir)...)
+ }
+ for _, src := range srcs {
+ if err := fillPathPairs(fa.PathPrefixInZip,
+ fa.SourcePrefixToStrip, src, &pathMappings, args.NonDeflatedFiles); err != nil {
+ log.Fatal(err)
+ }
+ }
+ }
+
+ buf := &bytes.Buffer{}
+ var out io.Writer = buf
+
+ if !args.WriteIfChanged {
+ f, err := os.Create(args.OutputFilePath)
+ if err != nil {
+ return err
+ }
+
+ defer f.Close()
+ defer func() {
+ if err != nil {
+ os.Remove(args.OutputFilePath)
+ }
+ }()
+
+ out = f
+ }
+
+ err = w.write(out, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
+ if err != nil {
+ return err
+ }
+
+ if args.WriteIfChanged {
+ err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func fillPathPairs(prefix, rel, src string, pathMappings *[]pathMapping, nonDeflatedFiles map[string]bool) error {
+ src = strings.TrimSpace(src)
+ if src == "" {
+ return nil
+ }
+ src = filepath.Clean(src)
+ dest, err := filepath.Rel(rel, src)
+ if err != nil {
+ return err
+ }
+ dest = filepath.Join(prefix, dest)
+
+ zipMethod := zip.Deflate
+ if _, found := nonDeflatedFiles[dest]; found {
+ zipMethod = zip.Store
+ }
+ *pathMappings = append(*pathMappings,
+ pathMapping{dest: dest, src: src, zipMethod: zipMethod})
+
+ return nil
+}
+
+func jarSort(mappings []pathMapping) {
+ less := func(i int, j int) (smaller bool) {
+ return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
+ }
+ sort.SliceStable(mappings, less)
+}
+
+type readerSeekerCloser interface {
+ io.Reader
+ io.ReaderAt
+ io.Closer
+ io.Seeker
+}
+
+func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
+ z.errors = make(chan error)
+ defer close(z.errors)
+
+ // This channel size can be essentially unlimited -- it's used as a fifo
+ // queue decouple the CPU and IO loads. Directories don't require any
+ // compression time, but still cost some IO. Similar with small files that
+ // can be very fast to compress. Some files that are more difficult to
+ // compress won't take a corresponding longer time writing out.
+ //
+ // The optimum size here depends on your CPU and IO characteristics, and
+ // the the layout of your zip file. 1000 was chosen mostly at random as
+ // something that worked reasonably well for a test file.
+ //
+ // The RateLimit object will put the upper bounds on the number of
+ // parallel compressions and outstanding buffers.
+ z.writeOps = make(chan chan *zipEntry, 1000)
+ z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
+ z.memoryRateLimiter = NewMemoryRateLimiter(0)
+ defer func() {
+ z.cpuRateLimiter.Stop()
+ z.memoryRateLimiter.Stop()
+ }()
+
+ if manifest != "" && !emulateJar {
+ return errors.New("must specify --jar when specifying a manifest via -m")
+ }
+
+ if emulateJar {
+ // manifest may be empty, in which case addManifest will fill in a default
+ pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
+
+ jarSort(pathMappings)
+ }
+
+ go func() {
+ var err error
+ defer close(z.writeOps)
+
+ for _, ele := range pathMappings {
+ if emulateJar && ele.dest == jar.ManifestFile {
+ err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
+ } else {
+ err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar)
+ }
+ if err != nil {
+ z.errors <- err
+ return
+ }
+ }
+ }()
+
+ zipw := zip.NewWriter(f)
+
+ var currentWriteOpChan chan *zipEntry
+ var currentWriter io.WriteCloser
+ var currentReaders chan chan io.Reader
+ var currentReader chan io.Reader
+ var done bool
+
+ for !done {
+ var writeOpsChan chan chan *zipEntry
+ var writeOpChan chan *zipEntry
+ var readersChan chan chan io.Reader
+
+ if currentReader != nil {
+ // Only read and process errors
+ } else if currentReaders != nil {
+ readersChan = currentReaders
+ } else if currentWriteOpChan != nil {
+ writeOpChan = currentWriteOpChan
+ } else {
+ writeOpsChan = z.writeOps
+ }
+
+ select {
+ case writeOp, ok := <-writeOpsChan:
+ if !ok {
+ done = true
+ }
+
+ currentWriteOpChan = writeOp
+
+ case op := <-writeOpChan:
+ currentWriteOpChan = nil
+
+ var err error
+ if op.fh.Method == zip.Deflate {
+ currentWriter, err = zipw.CreateCompressedHeader(op.fh)
+ } else {
+ var zw io.Writer
+
+ op.fh.CompressedSize64 = op.fh.UncompressedSize64
+
+ zw, err = zipw.CreateHeaderAndroid(op.fh)
+ currentWriter = nopCloser{zw}
+ }
+ if err != nil {
+ return err
+ }
+
+ currentReaders = op.futureReaders
+ if op.futureReaders == nil {
+ currentWriter.Close()
+ currentWriter = nil
+ }
+ z.memoryRateLimiter.Finish(op.allocatedSize)
+
+ case futureReader, ok := <-readersChan:
+ if !ok {
+ // Done with reading
+ currentWriter.Close()
+ currentWriter = nil
+ currentReaders = nil
+ }
+
+ currentReader = futureReader
+
+ case reader := <-currentReader:
+ _, err := io.Copy(currentWriter, reader)
+ if err != nil {
+ return err
+ }
+
+ currentReader = nil
+
+ case err := <-z.errors:
+ return err
+ }
+ }
+
+ // One last chance to catch an error
+ select {
+ case err := <-z.errors:
+ return err
+ default:
+ zipw.Close()
+ return nil
+ }
+}
+
+// imports (possibly with compression) <src> into the zip at sub-path <dest>
+func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error {
+ var fileSize int64
+ var executable bool
+
+ if s, err := os.Lstat(src); err != nil {
+ return err
+ } else if s.IsDir() {
+ if z.directories {
+ return z.writeDirectory(dest, src, emulateJar)
+ }
+ return nil
+ } else {
+ if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
+ return err
+ }
+
+ if prev, exists := z.createdDirs[dest]; exists {
+ return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
+ }
+ if prev, exists := z.createdFiles[dest]; exists {
+ return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
+ }
+
+ z.createdFiles[dest] = src
+
+ if s.Mode()&os.ModeSymlink != 0 {
+ return z.writeSymlink(dest, src)
+ } else if !s.Mode().IsRegular() {
+ return fmt.Errorf("%s is not a file, directory, or symlink", src)
+ }
+
+ fileSize = s.Size()
+ executable = s.Mode()&0100 != 0
+ }
+
+ r, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+
+ header := &zip.FileHeader{
+ Name: dest,
+ Method: method,
+ UncompressedSize64: uint64(fileSize),
+ }
+
+ if executable {
+ header.SetMode(0700)
+ }
+
+ return z.writeFileContents(header, r)
+}
+
+func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
+ if prev, exists := z.createdDirs[dest]; exists {
+ return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
+ }
+ if prev, exists := z.createdFiles[dest]; exists {
+ return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
+ }
+
+ if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
+ return err
+ }
+
+ fh, buf, err := jar.ManifestFileContents(src)
+ if err != nil {
+ return err
+ }
+
+ reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
+
+ return z.writeFileContents(fh, reader)
+}
+
+func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r readerSeekerCloser) (err error) {
+
+ header.SetModTime(z.time)
+
+ compressChan := make(chan *zipEntry, 1)
+ z.writeOps <- compressChan
+
+ // Pre-fill a zipEntry, it will be sent in the compressChan once
+ // we're sure about the Method and CRC.
+ ze := &zipEntry{
+ fh: header,
+ }
+
+ ze.allocatedSize = int64(header.UncompressedSize64)
+ z.cpuRateLimiter.Request()
+ z.memoryRateLimiter.Request(ze.allocatedSize)
+
+ fileSize := int64(header.UncompressedSize64)
+ if fileSize == 0 {
+ fileSize = int64(header.UncompressedSize)
+ }
+
+ if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
+ wg := new(sync.WaitGroup)
+
+ // Allocate enough buffer to hold all readers. We'll limit
+ // this based on actual buffer sizes in RateLimit.
+ ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
+
+ // Calculate the CRC in the background, since reading the entire
+ // file could take a while.
+ //
+ // We could split this up into chunks as well, but it's faster
+ // than the compression. Due to the Go Zip API, we also need to
+ // know the result before we can begin writing the compressed
+ // data out to the zipfile.
+ wg.Add(1)
+ go z.crcFile(r, ze, compressChan, wg)
+
+ for start := int64(0); start < fileSize; start += parallelBlockSize {
+ sr := io.NewSectionReader(r, start, parallelBlockSize)
+ resultChan := make(chan io.Reader, 1)
+ ze.futureReaders <- resultChan
+
+ z.cpuRateLimiter.Request()
+
+ last := !(start+parallelBlockSize < fileSize)
+ var dict []byte
+ if start >= windowSize {
+ dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
+ if err != nil {
+ return err
+ }
+ }
+
+ wg.Add(1)
+ go z.compressPartialFile(sr, dict, last, resultChan, wg)
+ }
+
+ close(ze.futureReaders)
+
+ // Close the file handle after all readers are done
+ go func(wg *sync.WaitGroup, closer io.Closer) {
+ wg.Wait()
+ closer.Close()
+ }(wg, r)
+ } else {
+ go func() {
+ z.compressWholeFile(ze, r, compressChan)
+ r.Close()
+ }()
+ }
+
+ return nil
+}
+
+func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
+ defer wg.Done()
+ defer z.cpuRateLimiter.Finish()
+
+ crc := crc32.NewIEEE()
+ _, err := io.Copy(crc, r)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+
+ ze.fh.CRC32 = crc.Sum32()
+ resultChan <- ze
+ close(resultChan)
+}
+
+func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ result, err := z.compressBlock(r, dict, last)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+
+ z.cpuRateLimiter.Finish()
+
+ resultChan <- result
+}
+
+func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
+ buf := new(bytes.Buffer)
+ var fw *flate.Writer
+ var err error
+ if len(dict) > 0 {
+ // There's no way to Reset a Writer with a new dictionary, so
+ // don't use the Pool
+ fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
+ } else {
+ var ok bool
+ if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
+ fw.Reset(buf)
+ } else {
+ fw, err = flate.NewWriter(buf, z.compLevel)
+ }
+ defer z.compressorPool.Put(fw)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ _, err = io.Copy(fw, r)
+ if err != nil {
+ return nil, err
+ }
+ if last {
+ fw.Close()
+ } else {
+ fw.Flush()
+ }
+
+ return buf, nil
+}
+
+func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
+
+ crc := crc32.NewIEEE()
+ _, err := io.Copy(crc, r)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+
+ ze.fh.CRC32 = crc.Sum32()
+
+ _, err = r.Seek(0, 0)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+
+ readFile := func(reader io.ReadSeeker) ([]byte, error) {
+ _, err := reader.Seek(0, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ buf, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return nil, err
+ }
+
+ return buf, nil
+ }
+
+ ze.futureReaders = make(chan chan io.Reader, 1)
+ futureReader := make(chan io.Reader, 1)
+ ze.futureReaders <- futureReader
+ close(ze.futureReaders)
+
+ if ze.fh.Method == zip.Deflate {
+ compressed, err := z.compressBlock(r, nil, true)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+ if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
+ futureReader <- compressed
+ } else {
+ buf, err := readFile(r)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+ ze.fh.Method = zip.Store
+ futureReader <- bytes.NewReader(buf)
+ }
+ } else {
+ buf, err := readFile(r)
+ if err != nil {
+ z.errors <- err
+ return
+ }
+ ze.fh.Method = zip.Store
+ futureReader <- bytes.NewReader(buf)
+ }
+
+ z.cpuRateLimiter.Finish()
+
+ close(futureReader)
+
+ compressChan <- ze
+ close(compressChan)
+}
+
+// writeDirectory annotates that dir is a directory created for the src file or directory, and adds
+// the directory entry to the zip file if directories are enabled.
+func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
+ // clean the input
+ dir = filepath.Clean(dir)
+
+ // discover any uncreated directories in the path
+ zipDirs := []string{}
+ for dir != "" && dir != "." {
+ if _, exists := z.createdDirs[dir]; exists {
+ break
+ }
+
+ if prev, exists := z.createdFiles[dir]; exists {
+ return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
+ }
+
+ z.createdDirs[dir] = src
+ // parent directories precede their children
+ zipDirs = append([]string{dir}, zipDirs...)
+
+ dir = filepath.Dir(dir)
+ }
+
+ if z.directories {
+ // make a directory entry for each uncreated directory
+ for _, cleanDir := range zipDirs {
+ var dirHeader *zip.FileHeader
+
+ if emulateJar && cleanDir+"/" == jar.MetaDir {
+ dirHeader = jar.MetaDirFileHeader()
+ } else {
+ dirHeader = &zip.FileHeader{
+ Name: cleanDir + "/",
+ }
+ dirHeader.SetMode(0700 | os.ModeDir)
+ }
+
+ dirHeader.SetModTime(z.time)
+
+ ze := make(chan *zipEntry, 1)
+ ze <- &zipEntry{
+ fh: dirHeader,
+ }
+ close(ze)
+ z.writeOps <- ze
+ }
+ }
+
+ return nil
+}
+
+func (z *ZipWriter) writeSymlink(rel, file string) error {
+ fileHeader := &zip.FileHeader{
+ Name: rel,
+ }
+ fileHeader.SetModTime(z.time)
+ fileHeader.SetMode(0700 | os.ModeSymlink)
+
+ dest, err := os.Readlink(file)
+ if err != nil {
+ return err
+ }
+
+ ze := make(chan *zipEntry, 1)
+ futureReaders := make(chan chan io.Reader, 1)
+ futureReader := make(chan io.Reader, 1)
+ futureReaders <- futureReader
+ close(futureReaders)
+ futureReader <- bytes.NewBufferString(dest)
+ close(futureReader)
+
+ ze <- &zipEntry{
+ fh: fileHeader,
+ futureReaders: futureReaders,
+ }
+ close(ze)
+ z.writeOps <- ze
+
+ return nil
+}
+
+func recursiveGlobFiles(path string) []string {
+ var files []string
+ filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+ if !info.IsDir() {
+ files = append(files, path)
+ }
+ return nil
+ })
+
+ return files
+}