Remove python2 support from soong rules
Python2 has been removed from the android build.
Removing the python version transition mutator should lead to some
slight analysis time performance improvements.
I also made embedded_launcher in base_properties no longer
blueprint:"mutated", so that we can migrate usages to the main property
struct and remove the `version` property struct.
Many python tests were not being run due to a
`if d.desc != "module with duplicate runfile path" { continue }` line,
I removed that and fixed the tests. I also removed the test for an
error when there are no source files, because that check has been
relaxed since, and I think it's legitimate to have a python library that
only has data files, not source files. And there are some of those
libraries in the source tree.
Bug: 203436762
Test: m nothing --no-skip-soong-tests
Change-Id: Idf18e39233418237ecc83f70f65d83d4ea496633
diff --git a/python/binary.go b/python/binary.go
index a3acb34..4d6e118 100644
--- a/python/binary.go
+++ b/python/binary.go
@@ -23,6 +23,7 @@
"android/soong/android"
"android/soong/cc"
+
"github.com/google/blueprint"
)
@@ -143,8 +144,7 @@
}
srcsZips = append(srcsZips, depsSrcsZips...)
p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath,
- p.getHostInterpreterName(ctx, p.properties.Actual_version),
- main, p.getStem(ctx), srcsZips)
+ "python3", main, p.getStem(ctx), srcsZips)
var sharedLibs []string
// if embedded launcher is enabled, we need to collect the shared library dependencies of the
@@ -205,23 +205,6 @@
return BoolDefault(b.binaryProperties.Autorun, true)
}
-// get host interpreter name.
-func (p *PythonBinaryModule) getHostInterpreterName(ctx android.ModuleContext,
- actualVersion string) string {
- var interp string
- switch actualVersion {
- case pyVersion2:
- interp = "python2.7"
- case pyVersion3:
- interp = "python3"
- default:
- panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
- actualVersion, ctx.ModuleName()))
- }
-
- return interp
-}
-
// find main program path within runfiles tree.
func (p *PythonBinaryModule) getPyMainFile(ctx android.ModuleContext,
srcsPathMappings []pathMapping) string {
diff --git a/python/defaults.go b/python/defaults.go
index 3dc5bc4..b5ee2bc 100644
--- a/python/defaults.go
+++ b/python/defaults.go
@@ -18,10 +18,6 @@
"android/soong/android"
)
-func init() {
- android.RegisterModuleType("python_defaults", DefaultsFactory)
-}
-
type Defaults struct {
android.ModuleBase
android.DefaultsModuleBase
diff --git a/python/library.go b/python/library.go
index 7cdb80b..c197028 100644
--- a/python/library.go
+++ b/python/library.go
@@ -27,6 +27,7 @@
func registerPythonLibraryComponents(ctx android.RegistrationContext) {
ctx.RegisterModuleType("python_library_host", PythonLibraryHostFactory)
ctx.RegisterModuleType("python_library", PythonLibraryFactory)
+ ctx.RegisterModuleType("python_defaults", DefaultsFactory)
}
func PythonLibraryHostFactory() android.Module {
diff --git a/python/python.go b/python/python.go
index 09af62e..10c11ad 100644
--- a/python/python.go
+++ b/python/python.go
@@ -23,6 +23,7 @@
"strings"
"android/soong/cc"
+
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
@@ -39,19 +40,6 @@
var PythonLibraryInfoProvider = blueprint.NewProvider[PythonLibraryInfo]()
-func init() {
- registerPythonMutators(android.InitRegistrationContext)
-}
-
-func registerPythonMutators(ctx android.RegistrationContext) {
- ctx.PreDepsMutators(RegisterPythonPreDepsMutators)
-}
-
-// Exported to support other packages using Python modules in tests.
-func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) {
- ctx.Transition("python_version", &versionSplitTransitionMutator{})
-}
-
// the version-specific properties that apply to python modules.
type VersionProperties struct {
// whether the module is required to be built with this version.
@@ -127,18 +115,14 @@
Py3 VersionProperties `android:"arch_variant"`
} `android:"arch_variant"`
- // the actual version each module uses after variations created.
- // this property name is hidden from users' perspectives, and soong will populate it during
- // runtime.
- Actual_version string `blueprint:"mutated"`
-
- // whether the module is required to be built with actual_version.
- // this is set by the python version mutator based on version-specific properties
+ // This enabled property is to accept the collapsed enabled property from the VersionProperties.
+ // It is unused now, as all builds should be python3.
Enabled *bool `blueprint:"mutated"`
- // whether the binary is required to be built with embedded launcher for this actual_version.
- // this is set by the python version mutator based on version-specific properties
- Embedded_launcher *bool `blueprint:"mutated"`
+ // whether the binary is required to be built with an embedded python interpreter, defaults to
+ // true. This allows taking the resulting binary outside of the build and running it on machines
+ // that don't have python installed or may have an older version of python.
+ Embedded_launcher *bool
}
// Used to store files of current module after expanding dependencies
@@ -252,8 +236,6 @@
pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
pyExt = ".py"
protoExt = ".proto"
- pyVersion2 = "PY2"
- pyVersion3 = "PY3"
internalPath = "internal"
)
@@ -261,71 +243,6 @@
getBaseProperties() *BaseProperties
}
-type versionSplitTransitionMutator struct{}
-
-func (versionSplitTransitionMutator) Split(ctx android.BaseModuleContext) []string {
- if base, ok := ctx.Module().(basePropertiesProvider); ok {
- props := base.getBaseProperties()
- var variants []string
- // PY3 is first so that we alias the PY3 variant rather than PY2 if both
- // are available
- if proptools.BoolDefault(props.Version.Py3.Enabled, true) {
- variants = append(variants, pyVersion3)
- }
- if proptools.BoolDefault(props.Version.Py2.Enabled, false) {
- if ctx.ModuleName() != "py2-cmd" &&
- ctx.ModuleName() != "py2-stdlib" {
- ctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3.")
- }
- variants = append(variants, pyVersion2)
- }
- return variants
- }
- return []string{""}
-}
-
-func (versionSplitTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string {
- return ""
-}
-
-func (versionSplitTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string {
- if incomingVariation != "" {
- return incomingVariation
- }
- if base, ok := ctx.Module().(basePropertiesProvider); ok {
- props := base.getBaseProperties()
- if proptools.BoolDefault(props.Version.Py3.Enabled, true) {
- return pyVersion3
- } else {
- return pyVersion2
- }
- }
-
- return ""
-}
-
-func (versionSplitTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) {
- if variation == "" {
- return
- }
- if base, ok := ctx.Module().(basePropertiesProvider); ok {
- props := base.getBaseProperties()
- props.Actual_version = variation
-
- var versionProps *VersionProperties
- if variation == pyVersion3 {
- versionProps = &props.Version.Py3
- } else if variation == pyVersion2 {
- versionProps = &props.Version.Py2
- }
-
- err := proptools.AppendMatchingProperties([]interface{}{props}, versionProps, nil)
- if err != nil {
- panic(err)
- }
- }
-}
-
func anyHasExt(paths []string, ext string) bool {
for _, p := range paths {
if filepath.Ext(p) == ext {
@@ -345,19 +262,26 @@
// - if required, specifies launcher and adds launcher dependencies,
// - applies python version mutations to Python dependencies
func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
- android.ProtoDeps(ctx, &p.protoProperties)
+ // Flatten the version.py3 props down into the main property struct. Leftover from when
+ // there was both python2 and 3 in the build, and properties could be different between them.
+ if base, ok := ctx.Module().(basePropertiesProvider); ok {
+ props := base.getBaseProperties()
- versionVariation := []blueprint.Variation{
- {"python_version", p.properties.Actual_version},
+ err := proptools.AppendMatchingProperties([]interface{}{props}, &props.Version.Py3, nil)
+ if err != nil {
+ panic(err)
+ }
}
+ android.ProtoDeps(ctx, &p.protoProperties)
+
// If sources contain a proto file, add dependency on libprotobuf-python
if p.anySrcHasExt(ctx, protoExt) && p.Name() != "libprotobuf-python" {
- ctx.AddVariationDependencies(versionVariation, pythonLibTag, "libprotobuf-python")
+ ctx.AddDependency(ctx.Module(), pythonLibTag, "libprotobuf-python")
}
// Add python library dependencies for this python version variation
- ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
+ ctx.AddDependency(ctx.Module(), pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
// Emulate the data property for java_data but with the arch variation overridden to "common"
// so that it can point to java modules.
@@ -394,55 +318,38 @@
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
}
- switch p.properties.Actual_version {
- case pyVersion2:
- stdLib = "py2-stdlib"
-
- launcherModule = "py2-launcher"
- if autorun {
- launcherModule = "py2-launcher-autorun"
- }
-
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
- case pyVersion3:
- var prebuiltStdLib bool
- if targetForDeps.Os.Bionic() {
- prebuiltStdLib = false
- } else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") {
- prebuiltStdLib = false
- } else {
- prebuiltStdLib = true
- }
-
- if prebuiltStdLib {
- stdLib = "py3-stdlib-prebuilt"
- } else {
- stdLib = "py3-stdlib"
- }
-
- launcherModule = "py3-launcher"
- if autorun {
- launcherModule = "py3-launcher-autorun"
- }
- if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl {
- launcherModule += "-static"
- }
- if ctx.Device() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
- }
- default:
- panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
- p.properties.Actual_version, ctx.ModuleName()))
+ var prebuiltStdLib bool
+ if targetForDeps.Os.Bionic() {
+ prebuiltStdLib = false
+ } else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") {
+ prebuiltStdLib = false
+ } else {
+ prebuiltStdLib = true
}
+
+ if prebuiltStdLib {
+ stdLib = "py3-stdlib-prebuilt"
+ } else {
+ stdLib = "py3-stdlib"
+ }
+
+ launcherModule = "py3-launcher"
+ if autorun {
+ launcherModule = "py3-launcher-autorun"
+ }
+ if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl {
+ launcherModule += "-static"
+ }
+ if ctx.Device() {
+ launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
+ }
+
targetVariations := targetForDeps.Variations()
if ctx.ModuleName() != stdLib {
- stdLibVariations := make([]blueprint.Variation, 0, len(targetVariations)+1)
- stdLibVariations = append(stdLibVariations, blueprint.Variation{Mutator: "python_version", Variation: p.properties.Actual_version})
- stdLibVariations = append(stdLibVariations, targetVariations...)
// Using AddFarVariationDependencies for all of these because they can be for a different
// platform, like if the python module itself was being compiled for device, we may want
// the python interpreter built for host so that we can precompile python sources.
- ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib)
+ ctx.AddFarVariationDependencies(targetVariations, stdLibTag, stdLib)
}
ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule)
ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...)
@@ -450,6 +357,9 @@
// GenerateAndroidBuildActions performs build actions common to all Python modules
func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ if proptools.BoolDefault(p.properties.Version.Py2.Enabled, false) {
+ ctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3.")
+ }
expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs)
// Keep before any early returns.
android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{
diff --git a/python/python_test.go b/python/python_test.go
index 6a6bd1d..0e97626 100644
--- a/python/python_test.go
+++ b/python/python_test.go
@@ -36,10 +36,8 @@
}
var (
- buildNamePrefix = "soong_python_test"
- // We allow maching almost anything before the actual variant so that the os/arch variant
- // is matched.
- moduleVariantErrTemplate = `%s: module %q variant "[a-zA-Z0-9_]*%s": `
+ buildNamePrefix = "soong_python_test"
+ moduleVariantErrTemplate = `%s: module %q variant "[a-zA-Z0-9_]*": `
pkgPathErrTemplate = moduleVariantErrTemplate +
"pkg_path: %q must be a relative path contained in par file."
badIdentifierErrTemplate = moduleVariantErrTemplate +
@@ -48,9 +46,8 @@
"found two files to be placed at the same location within zip %q." +
" First file: in module %s at path %q." +
" Second file: in module %s at path %q."
- noSrcFileErr = moduleVariantErrTemplate + "doesn't have any source files!"
- badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py|.proto) file: %q!"
- badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py) file: %q!"
+ badSrcFileExtErr = moduleVariantErrTemplate + `srcs: found non \(.py\|.proto\) file: %q!`
+ badDataFileExtErr = moduleVariantErrTemplate + `data: found \(.py\) file: %q!`
bpFile = "Android.bp"
data = []struct {
@@ -61,20 +58,6 @@
expectedBinaries []pyModule
}{
{
- desc: "module without any src files",
- mockFiles: map[string][]byte{
- filepath.Join("dir", bpFile): []byte(
- `python_library_host {
- name: "lib1",
- }`,
- ),
- },
- errors: []string{
- fmt.Sprintf(noSrcFileErr,
- "dir/Android.bp:1:1", "lib1", "PY3"),
- },
- },
- {
desc: "module with bad src file ext",
mockFiles: map[string][]byte{
filepath.Join("dir", bpFile): []byte(
@@ -89,7 +72,7 @@
},
errors: []string{
fmt.Sprintf(badSrcFileExtErr,
- "dir/Android.bp:3:11", "lib1", "PY3", "dir/file1.exe"),
+ "dir/Android.bp:3:11", "lib1", "dir/file1.exe"),
},
},
{
@@ -111,7 +94,7 @@
},
errors: []string{
fmt.Sprintf(badDataFileExtErr,
- "dir/Android.bp:6:11", "lib1", "PY3", "dir/file2.py"),
+ "dir/Android.bp:6:11", "lib1", "dir/file2.py"),
},
},
{
@@ -146,9 +129,9 @@
},
errors: []string{
fmt.Sprintf(pkgPathErrTemplate,
- "dir/Android.bp:11:15", "lib2", "PY3", "a/c/../../../"),
+ "dir/Android.bp:11:15", "lib2", "a/c/../../../"),
fmt.Sprintf(pkgPathErrTemplate,
- "dir/Android.bp:19:15", "lib3", "PY3", "/a/c/../../"),
+ "dir/Android.bp:19:15", "lib3", "/a/c/../../"),
},
},
{
@@ -171,11 +154,11 @@
},
errors: []string{
fmt.Sprintf(badIdentifierErrTemplate, "dir/Android.bp:4:11",
- "lib1", "PY3", "a/b/c/-e/f/file1.py", "-e"),
+ "lib1", "a/b/c/-e/f/file1.py", "-e"),
fmt.Sprintf(badIdentifierErrTemplate, "dir/Android.bp:4:11",
- "lib1", "PY3", "a/b/c/.file1.py", ".file1"),
+ "lib1", "a/b/c/.file1.py", ".file1"),
fmt.Sprintf(badIdentifierErrTemplate, "dir/Android.bp:4:11",
- "lib1", "PY3", "a/b/c/123/file1.py", "123"),
+ "lib1", "a/b/c/123/file1.py", "123"),
},
},
{
@@ -219,115 +202,15 @@
},
errors: []string{
fmt.Sprintf(dupRunfileErrTemplate, "dir/Android.bp:20:6",
- "bin", "PY3", "a/b/c/file1.py", "bin", "dir/file1.py",
+ "bin", "a/b/c/file1.py", "bin", "dir/file1.py",
"lib1", "dir/c/file1.py"),
},
},
- {
- desc: "module for testing dependencies",
- mockFiles: map[string][]byte{
- filepath.Join("dir", bpFile): []byte(
- `python_defaults {
- name: "default_lib",
- srcs: [
- "default.py",
- ],
- version: {
- py2: {
- enabled: true,
- srcs: [
- "default_py2.py",
- ],
- },
- py3: {
- enabled: false,
- srcs: [
- "default_py3.py",
- ],
- },
- },
- }
-
- python_library_host {
- name: "lib5",
- pkg_path: "a/b/",
- srcs: [
- "file1.py",
- ],
- version: {
- py2: {
- enabled: true,
- },
- py3: {
- enabled: true,
- },
- },
- }
-
- python_library_host {
- name: "lib6",
- pkg_path: "c/d/",
- srcs: [
- "file2.py",
- ],
- libs: [
- "lib5",
- ],
- }
-
- python_binary_host {
- name: "bin",
- defaults: ["default_lib"],
- pkg_path: "e/",
- srcs: [
- "bin.py",
- ],
- libs: [
- "lib5",
- ],
- version: {
- py3: {
- enabled: true,
- srcs: [
- "file4.py",
- ],
- libs: [
- "lib6",
- ],
- },
- },
- }`,
- ),
- filepath.Join("dir", "default.py"): nil,
- filepath.Join("dir", "default_py2.py"): nil,
- filepath.Join("dir", "default_py3.py"): nil,
- filepath.Join("dir", "file1.py"): nil,
- filepath.Join("dir", "file2.py"): nil,
- filepath.Join("dir", "bin.py"): nil,
- filepath.Join("dir", "file4.py"): nil,
- },
- expectedBinaries: []pyModule{
- {
- name: "bin",
- actualVersion: "PY3",
- pyRunfiles: []string{
- "e/default.py",
- "e/bin.py",
- "e/default_py3.py",
- "e/file4.py",
- },
- srcsZip: "out/soong/.intermediates/dir/bin/PY3/bin.py.srcszip",
- },
- },
- },
}
)
func TestPythonModule(t *testing.T) {
for _, d := range data {
- if d.desc != "module with duplicate runfile path" {
- continue
- }
d.mockFiles[filepath.Join("common", bpFile)] = []byte(`
python_library {
name: "py3-stdlib",
@@ -416,10 +299,6 @@
for i, bp := range testCases {
ctx := android.GroupFixturePreparers(
PrepareForTestWithPythonBuildComponents,
- android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
-
- ctx.RegisterModuleType("python_defaults", DefaultsFactory)
- }),
android.PrepareForTestWithAllowMissingDependencies).
ExtendWithErrorHandler(android.FixtureIgnoreErrors).
RunTestWithBp(t, bp)
diff --git a/python/testing.go b/python/testing.go
index ce1a5ab..fe53ee5 100644
--- a/python/testing.go
+++ b/python/testing.go
@@ -20,5 +20,4 @@
android.FixtureRegisterWithContext(registerPythonBinaryComponents),
android.FixtureRegisterWithContext(registerPythonLibraryComponents),
android.FixtureRegisterWithContext(registerPythonTestComponents),
- android.FixtureRegisterWithContext(registerPythonMutators),
)