Supported python build in host side.
The base module handles all the common functionalites, such as version
compatibilty check, version variations split, source file format check,
source/data file duplicate check.
The library/binary module focuses on how to generate binary build actions,
such as setting up stub script, zipping, filling in __init__.py in
runfiles dir tree.
Bug: b/31676493
Test: go test under python package
Change-Id: I06608369f350f7195873d459e1c8d1bdb811e77e
diff --git a/python/python_test.go b/python/python_test.go
new file mode 100644
index 0000000..c6b8451
--- /dev/null
+++ b/python/python_test.go
@@ -0,0 +1,456 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package python
+
+import (
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "strings"
+ "testing"
+
+ "android/soong/android"
+
+ "github.com/google/blueprint"
+)
+
+type pyBinary struct {
+ name string
+ actualVersion string
+ pyRunfiles []string
+ depsPyRunfiles []string
+ parSpec string
+ depsParSpecs []string
+}
+
+var (
+ buildNamePrefix = "soong_python_test"
+ moduleVariantErrTemplate = "%s: module %q variant %q: "
+ pkgPathErrTemplate = moduleVariantErrTemplate +
+ "pkg_path: %q is not a valid format."
+ badIdentifierErrTemplate = moduleVariantErrTemplate +
+ "srcs: the path %q contains invalid token %q."
+ dupRunfileErrTemplate = moduleVariantErrTemplate +
+ "found two files to be placed at the same runfiles location %q." +
+ " First file: in module %s at path %q." +
+ " Second file: in module %s at path %q."
+ noSrcFileErr = moduleVariantErrTemplate + "doesn't have any source files!"
+ badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py) file: %q!"
+ badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py) file: %q!"
+ bpFile = "Blueprints"
+
+ data = []struct {
+ desc string
+ mockFiles map[string][]byte
+
+ errors []string
+ expectedBinaries []pyBinary
+ }{
+ {
+ desc: "module without any src files",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib1",
+ }`,
+ ),
+ },
+ errors: []string{
+ fmt.Sprintf(noSrcFileErr,
+ "dir/Blueprints:1:1", "lib1", "PY3"),
+ },
+ },
+ {
+ desc: "module with bad src file ext",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib1",
+ srcs: [
+ "file1.exe",
+ ],
+ }`,
+ ),
+ "dir/file1.exe": nil,
+ },
+ errors: []string{
+ fmt.Sprintf(badSrcFileExtErr,
+ "dir/Blueprints:3:11", "lib1", "PY3", "dir/file1.exe"),
+ },
+ },
+ {
+ desc: "module with bad data file ext",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib1",
+ srcs: [
+ "file1.py",
+ ],
+ data: [
+ "file2.py",
+ ],
+ }`,
+ ),
+ "dir/file1.py": nil,
+ "dir/file2.py": nil,
+ },
+ errors: []string{
+ fmt.Sprintf(badDataFileExtErr,
+ "dir/Blueprints:6:11", "lib1", "PY3", "dir/file2.py"),
+ },
+ },
+ {
+ desc: "module with bad pkg_path format",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib1",
+ pkg_path: "a/c/../../",
+ srcs: [
+ "file1.py",
+ ],
+ }
+
+ python_library_host {
+ name: "lib2",
+ pkg_path: "a/c/../../../",
+ srcs: [
+ "file1.py",
+ ],
+ }
+
+ python_library_host {
+ name: "lib3",
+ pkg_path: "/a/c/../../",
+ srcs: [
+ "file1.py",
+ ],
+ }`,
+ ),
+ "dir/file1.py": nil,
+ },
+ errors: []string{
+ fmt.Sprintf(pkgPathErrTemplate,
+ "dir/Blueprints:11:15", "lib2", "PY3", "a/c/../../../"),
+ fmt.Sprintf(pkgPathErrTemplate,
+ "dir/Blueprints:19:15", "lib3", "PY3", "/a/c/../../"),
+ },
+ },
+ {
+ desc: "module with bad runfile src path format",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib1",
+ pkg_path: "a/b/c/",
+ srcs: [
+ ".file1.py",
+ "123/file1.py",
+ "-e/f/file1.py",
+ ],
+ }`,
+ ),
+ "dir/.file1.py": nil,
+ "dir/123/file1.py": nil,
+ "dir/-e/f/file1.py": nil,
+ },
+ errors: []string{
+ fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11",
+ "lib1", "PY3", "runfiles/a/b/c/-e/f/file1.py", "-e"),
+ fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11",
+ "lib1", "PY3", "runfiles/a/b/c/.file1.py", ".file1"),
+ fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11",
+ "lib1", "PY3", "runfiles/a/b/c/123/file1.py", "123"),
+ },
+ },
+ {
+ desc: "module with duplicate runfile path",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib1",
+ pkg_path: "a/b/",
+ srcs: [
+ "c/file1.py",
+ ],
+ }
+
+ python_library_host {
+ name: "lib2",
+ pkg_path: "a/b/c/",
+ srcs: [
+ "file1.py",
+ ],
+ libs: [
+ "lib1",
+ ],
+ }
+ `,
+ ),
+ "dir/c/file1.py": nil,
+ "dir/file1.py": nil,
+ },
+ errors: []string{
+ fmt.Sprintf(dupRunfileErrTemplate, "dir/Blueprints:9:6",
+ "lib2", "PY3", "runfiles/a/b/c/file1.py", "lib2", "dir/file1.py",
+ "lib1", "dir/c/file1.py"),
+ },
+ },
+ {
+ desc: "module for testing dependencies",
+ mockFiles: map[string][]byte{
+ bpFile: []byte(`subdirs = ["dir"]`),
+ filepath.Join("dir", bpFile): []byte(
+ `python_library_host {
+ name: "lib5",
+ pkg_path: "a/b/",
+ srcs: [
+ "file1.py",
+ ],
+ version: {
+ py2: {
+ enabled: true,
+ },
+ py3: {
+ enabled: true,
+ },
+ },
+ }
+
+ python_library_host {
+ name: "lib6",
+ pkg_path: "c/d/",
+ srcs: [
+ "file2.py",
+ ],
+ libs: [
+ "lib5",
+ ],
+ }
+
+ python_binary_host {
+ name: "bin",
+ pkg_path: "e/",
+ srcs: [
+ "bin.py",
+ ],
+ libs: [
+ "lib5",
+ ],
+ version: {
+ py3: {
+ enabled: true,
+ srcs: [
+ "file4.py",
+ ],
+ libs: [
+ "lib6",
+ ],
+ },
+ },
+ }`,
+ ),
+ filepath.Join("dir", "file1.py"): nil,
+ filepath.Join("dir", "file2.py"): nil,
+ filepath.Join("dir", "bin.py"): nil,
+ filepath.Join("dir", "file4.py"): nil,
+ stubTemplateHost: []byte(`PYTHON_BINARY = '%interpreter%'
+ MAIN_FILE = '%main%'`),
+ },
+ expectedBinaries: []pyBinary{
+ {
+ name: "bin",
+ actualVersion: "PY3",
+ pyRunfiles: []string{
+ "runfiles/e/bin.py",
+ "runfiles/e/file4.py",
+ },
+ depsPyRunfiles: []string{
+ "runfiles/a/b/file1.py",
+ "runfiles/c/d/file2.py",
+ },
+ parSpec: "-P runfiles/e -C dir/ -l @prefix@/.intermediates/dir/bin/PY3/dir_.list",
+ depsParSpecs: []string{
+ "-P runfiles/a/b -C dir/ -l @prefix@/.intermediates/dir/lib5/PY3/dir_.list",
+ "-P runfiles/c/d -C dir/ -l @prefix@/.intermediates/dir/lib6/PY3/dir_.list",
+ },
+ },
+ },
+ },
+ }
+)
+
+func TestPythonModule(t *testing.T) {
+ config, buildDir := setupBuildEnv(t)
+ defer tearDownBuildEnv()
+ android.TestPreDepsMutators(func(ctx android.RegisterMutatorsContext) {
+ ctx.BottomUp("version_split", versionSplitMutator()).Parallel()
+ })
+ for _, d := range data {
+ t.Run(d.desc, func(t *testing.T) {
+ ctx := blueprint.NewContext()
+ android.RegisterTestMutators(ctx)
+ ctx.RegisterModuleType("python_library_host", PythonLibraryHostFactory)
+ ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory)
+ ctx.MockFileSystem(d.mockFiles)
+ _, testErrs := ctx.ParseBlueprintsFiles(bpFile)
+ fail(t, testErrs)
+ _, actErrs := ctx.PrepareBuildActions(config)
+ if len(actErrs) > 0 {
+ testErrs = append(testErrs, expectErrors(t, actErrs, d.errors)...)
+ } else {
+ for _, e := range d.expectedBinaries {
+ testErrs = append(testErrs,
+ expectModule(t, ctx, buildDir, e.name,
+ e.actualVersion,
+ e.pyRunfiles, e.depsPyRunfiles,
+ e.parSpec, e.depsParSpecs)...)
+ }
+ }
+ fail(t, testErrs)
+ })
+ }
+}
+
+func expectErrors(t *testing.T, actErrs []error, expErrs []string) (testErrs []error) {
+ actErrStrs := []string{}
+ for _, v := range actErrs {
+ actErrStrs = append(actErrStrs, v.Error())
+ }
+ sort.Strings(actErrStrs)
+ if len(actErrStrs) != len(expErrs) {
+ t.Errorf("got (%d) errors, expected (%d) errors!", len(actErrStrs), len(expErrs))
+ for _, v := range actErrStrs {
+ testErrs = append(testErrs, errors.New(v))
+ }
+ } else {
+ sort.Strings(expErrs)
+ for i, v := range actErrStrs {
+ if v != expErrs[i] {
+ testErrs = append(testErrs, errors.New(v))
+ }
+ }
+ }
+
+ return
+}
+
+func expectModule(t *testing.T, ctx *blueprint.Context, buildDir, name, variant string,
+ expPyRunfiles, expDepsPyRunfiles []string,
+ expParSpec string, expDepsParSpecs []string) (testErrs []error) {
+ module := findModule(ctx, name, variant)
+ if module == nil {
+ t.Fatalf("failed to find module %s!", name)
+ }
+
+ base, baseOk := module.(*pythonBaseModule)
+ if !baseOk {
+ t.Fatalf("%s is not Python module!", name)
+ }
+ sub, subOk := base.subModule.(*PythonBinary)
+ if !subOk {
+ t.Fatalf("%s is not Python binary!", name)
+ }
+
+ actPyRunfiles := []string{}
+ for _, path := range base.srcsPathMappings {
+ actPyRunfiles = append(actPyRunfiles, path.dest)
+ }
+
+ if !reflect.DeepEqual(actPyRunfiles, expPyRunfiles) {
+ testErrs = append(testErrs, errors.New(fmt.Sprintf(
+ `binary "%s" variant "%s" has unexpected pyRunfiles: %q!`,
+ base.Name(),
+ base.properties.ActualVersion,
+ actPyRunfiles)))
+ }
+
+ if !reflect.DeepEqual(sub.depsPyRunfiles, expDepsPyRunfiles) {
+ testErrs = append(testErrs, errors.New(fmt.Sprintf(
+ `binary "%s" variant "%s" has unexpected depsPyRunfiles: %q!`,
+ base.Name(),
+ base.properties.ActualVersion,
+ sub.depsPyRunfiles)))
+ }
+
+ if base.parSpec.soongParArgs() != strings.Replace(expParSpec, "@prefix@", buildDir, 1) {
+ testErrs = append(testErrs, errors.New(fmt.Sprintf(
+ `binary "%s" variant "%s" has unexpected parSpec: %q!`,
+ base.Name(),
+ base.properties.ActualVersion,
+ base.parSpec.soongParArgs())))
+ }
+
+ actDepsParSpecs := []string{}
+ for i, p := range sub.depsParSpecs {
+ actDepsParSpecs = append(actDepsParSpecs, p.soongParArgs())
+ expDepsParSpecs[i] = strings.Replace(expDepsParSpecs[i], "@prefix@", buildDir, 1)
+ }
+
+ if !reflect.DeepEqual(actDepsParSpecs, expDepsParSpecs) {
+ testErrs = append(testErrs, errors.New(fmt.Sprintf(
+ `binary "%s" variant "%s" has unexpected depsParSpecs: %q!`,
+ base.Name(),
+ base.properties.ActualVersion,
+ actDepsParSpecs)))
+ }
+
+ return
+}
+
+func setupBuildEnv(t *testing.T) (config android.Config, buildDir string) {
+ buildDir, err := ioutil.TempDir("", buildNamePrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ config = android.TestConfig(buildDir)
+
+ return
+}
+
+func tearDownBuildEnv() {
+ os.RemoveAll(buildNamePrefix)
+}
+
+func findModule(ctx *blueprint.Context, name, variant string) blueprint.Module {
+ var ret blueprint.Module
+ ctx.VisitAllModules(func(m blueprint.Module) {
+ if ctx.ModuleName(m) == name && ctx.ModuleSubDir(m) == variant {
+ ret = m
+ }
+ })
+ return ret
+}
+
+func fail(t *testing.T, errs []error) {
+ if len(errs) > 0 {
+ for _, err := range errs {
+ t.Error(err)
+ }
+ t.FailNow()
+ }
+}