Multithread symlink forest removal.
This makes symlink forest creation ca. 2x faster again, taking 2-3
seconds instead of 5.
Who would have thought that os.RemoveAll() is slow.
Test: Presubmits.
Change-Id: I91e41319c972dbf1113cf723e383c785433c18b9
diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go
index 86f3d42..8000477 100644
--- a/bp2build/java_binary_host_conversion_test.go
+++ b/bp2build/java_binary_host_conversion_test.go
@@ -32,7 +32,7 @@
}, tc)
}
-var fs = map[string]string{
+var testFs = map[string]string{
"test.mf": "Main-Class: com.android.test.MainClass",
"other/Android.bp": `cc_library_host_shared {
name: "jni-lib-1",
@@ -43,7 +43,7 @@
func TestJavaBinaryHost(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, exclude_srcs, jni_libs, javacflags, and manifest.",
- Filesystem: fs,
+ Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-1",
srcs: ["a.java", "b.java"],
@@ -76,7 +76,7 @@
func TestJavaBinaryHostRuntimeDeps(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, exclude_srcs, jni_libs, javacflags, and manifest.",
- Filesystem: fs,
+ Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-1",
static_libs: ["java-dep-1"],
diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go
index 45817e3..0fa9c50 100644
--- a/bp2build/symlink_forest.go
+++ b/bp2build/symlink_forest.go
@@ -15,7 +15,9 @@
package bp2build
import (
+ "errors"
"fmt"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -317,6 +319,45 @@
}
}
+func removeParallelRecursive(path string, fi os.FileInfo, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ if fi.IsDir() {
+ children := readdirToMap(path)
+ childrenWg := &sync.WaitGroup{}
+ childrenWg.Add(len(children))
+
+ for child, childFi := range children {
+ go removeParallelRecursive(shared.JoinPath(path, child), childFi, childrenWg)
+ }
+
+ childrenWg.Wait()
+ }
+
+ if err := os.Remove(path); err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot unlink '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+}
+
+func removeParallel(path string) {
+ fi, err := os.Lstat(path)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return
+ }
+
+ fmt.Fprintf(os.Stderr, "Cannot lstat '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+
+ wg := &sync.WaitGroup{}
+ wg.Add(1)
+ removeParallelRecursive(path, fi, wg)
+
+ wg.Wait()
+}
+
// Creates a symlink forest by merging the directory tree at "buildFiles" and
// "srcDir" while excluding paths listed in "exclude". Returns the set of paths
// under srcDir on which readdir() had to be called to produce the symlink
@@ -330,7 +371,7 @@
context.okay.Store(true)
- os.RemoveAll(shared.JoinPath(topdir, forest))
+ removeParallel(shared.JoinPath(topdir, forest))
instructions := instructionsFromExcludePathList(exclude)
go func() {