WriteFileRule: Chunk long content and merge them to result
sbox.textproto files are created when handling genrule. The command
for the genrule and output files are written to this file in the
following format:
commands <
command: "...."
copy_after: <
from: "out/asm-generic/auxvec.h"
to: "out/soong/.intermediates/kernel/msm-4.19/qti_generate_kernel_headers_arm/gen/asm-generic/auxvec.h"
>
copy_after: <
from: "out/asm-generic/bitsperlong.h"
to:
"out/soong/.intermediates/kernel/msm-4.19/qti_generate_kernel_headers_arm/gen/asm-generic/bitsperlong.h"
>
....
>
This file grow by one copy_after entry for each output file.
When generating kenrnel headers where the number of output files are
~1000 we run into problems as the contents of sbox.textproto files
are written to disk by generating a shell script using the following
template: /bin/bash -c 'echo -e "$$0" > $out' $content
If $content is very long as in the case of generating kernel headers we
run into the issue where the command line is so long that the shell
script return E2BIG.
Fix this issue by chuking contents into smaller files and then merge
them as a final step.
Test: Build
Issue: 174444967
Change-Id: I1a74023b4222d19672e4df7edb19810a9cf2136f
diff --git a/android/defs.go b/android/defs.go
index 631dfe8..f5bd362 100644
--- a/android/defs.go
+++ b/android/defs.go
@@ -15,6 +15,7 @@
package android
import (
+ "fmt"
"strings"
"testing"
@@ -97,7 +98,7 @@
// content to file.
writeFile = pctx.AndroidStaticRule("writeFile",
blueprint.RuleParams{
- Command: `/bin/bash -c 'echo -e "$$0" > $out' $content`,
+ Command: `/bin/bash -c 'echo -e -n "$$0" > $out' $content`,
Description: "writing file $out",
},
"content")
@@ -133,9 +134,7 @@
shellUnescaper = strings.NewReplacer(`'\''`, `'`)
)
-// WriteFileRule creates a ninja rule to write contents to a file. The contents will be escaped
-// so that the file contains exactly the contents passed to the function, plus a trailing newline.
-func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
+func buildWriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
content = echoEscaper.Replace(content)
content = proptools.ShellEscape(content)
if content == "" {
@@ -151,6 +150,31 @@
})
}
+// WriteFileRule creates a ninja rule to write contents to a file. The contents will be escaped
+// so that the file contains exactly the contents passed to the function, plus a trailing newline.
+func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
+ // This is MAX_ARG_STRLEN subtracted with some safety to account for shell escapes
+ const SHARD_SIZE = 131072 - 10000
+
+ content += "\n"
+ if len(content) > SHARD_SIZE {
+ var chunks WritablePaths
+ for i, c := range ShardString(content, SHARD_SIZE) {
+ tempPath := outputFile.ReplaceExtension(ctx, fmt.Sprintf("%s.%d", outputFile.Ext(), i))
+ buildWriteFileRule(ctx, tempPath, c)
+ chunks = append(chunks, tempPath)
+ }
+ ctx.Build(pctx, BuildParams{
+ Rule: Cat,
+ Inputs: chunks.Paths(),
+ Output: outputFile,
+ Description: "Merging to " + outputFile.Base(),
+ })
+ return
+ }
+ buildWriteFileRule(ctx, outputFile, content)
+}
+
// shellUnescape reverses proptools.ShellEscape
func shellUnescape(s string) string {
// Remove leading and trailing quotes if present