Add a dependency fixer for proto deps
protoc dependency files, at least for C++ outputs, uses the form of:
a/b.c \
a/b.h: <dep1> <dep2>...
Ninja will fail the command when it parses a dep file and there's more
than one output file (even though it doesn't care what the output file
name is). So this tool will parse the original file, and output a
version with only a single output file.
Bug: 67329638
Test: NINJA_ARGS="-t deps ...pb.c" m
Test: NINJA_ARGS="-t deps ...srcjar" m
Test: NINJA_ARGS="-t deps ...srcszip" m
Test: Run dep_fixer across all of taimen's dep files, no failures.
Test: Run dep_fixer against the processed files, no changes.
Test: Run androidmk across all of our Android.mk files, inspect the diffs
Change-Id: I4263b7d5faea37285afa6b24dedf5964aa7d19dc
diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go
index e6885a8..4b782a2 100644
--- a/androidmk/parser/make_strings.go
+++ b/androidmk/parser/make_strings.go
@@ -90,10 +90,10 @@
if len(ms.Strings) == 0 {
return ""
} else {
- ret := ms.Strings[0]
+ ret := unescape(ms.Strings[0])
for i := range ms.Strings[1:] {
ret += ms.Variables[i].Value(scope)
- ret += ms.Strings[i+1]
+ ret += unescape(ms.Strings[i+1])
}
return ret
}
@@ -125,6 +125,16 @@
}
func (ms *MakeString) SplitN(sep string, n int) []*MakeString {
+ return ms.splitNFunc(n, func(s string, n int) []string {
+ return splitAnyN(s, sep, n)
+ })
+}
+
+func (ms *MakeString) Words() []*MakeString {
+ return ms.splitNFunc(-1, splitWords)
+}
+
+func (ms *MakeString) splitNFunc(n int, splitFunc func(s string, n int) []string) []*MakeString {
ret := []*MakeString{}
curMs := SimpleMakeString("", ms.Pos())
@@ -133,7 +143,7 @@
var s string
for i, s = range ms.Strings {
if n != 0 {
- split := splitAnyN(s, sep, n)
+ split := splitFunc(s, n)
if n != -1 {
if len(split) > n {
panic("oops!")
@@ -156,7 +166,9 @@
}
}
- ret = append(ret, curMs)
+ if !curMs.Empty() {
+ ret = append(ret, curMs)
+ }
return ret
}
@@ -206,3 +218,64 @@
ret = append(ret, s)
return ret
}
+
+func splitWords(s string, n int) []string {
+ ret := []string{}
+ preserve := ""
+ for n == -1 || n > 1 {
+ index := strings.IndexAny(s, " \t")
+ if index == 0 && len(preserve) == 0 {
+ s = s[1:]
+ } else if index >= 0 {
+ escapeCount := 0
+ for i := index - 1; i >= 0; i-- {
+ if s[i] != '\\' {
+ break
+ }
+ escapeCount += 1
+ }
+
+ if escapeCount%2 == 1 {
+ preserve += s[0 : index+1]
+ s = s[index+1:]
+ continue
+ }
+
+ ret = append(ret, preserve+s[0:index])
+ s = s[index+1:]
+ preserve = ""
+ if n > 0 {
+ n--
+ }
+ } else {
+ break
+ }
+ }
+ if preserve != "" || s != "" || len(ret) == 0 {
+ ret = append(ret, preserve+s)
+ }
+ return ret
+}
+
+func unescape(s string) string {
+ ret := ""
+ for {
+ index := strings.IndexByte(s, '\\')
+ if index < 0 {
+ break
+ }
+
+ if index+1 == len(s) {
+ break
+ }
+
+ switch s[index+1] {
+ case ' ', '\\', '#', ':', '*', '[', '|', '\t', '\n', '\r':
+ ret += s[:index] + s[index+1:index+2]
+ default:
+ ret += s[:index+2]
+ }
+ s = s[index+2:]
+ }
+ return ret + s
+}
diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go
index 8ad3d74..6995e89 100644
--- a/androidmk/parser/make_strings_test.go
+++ b/androidmk/parser/make_strings_test.go
@@ -99,6 +99,78 @@
}
}
+var valueTestCases = []struct {
+ in *MakeString
+ expected string
+}{
+ {
+ in: SimpleMakeString("a b", NoPos),
+ expected: "a b",
+ },
+ {
+ in: SimpleMakeString("a\\ \\\tb\\\\", NoPos),
+ expected: "a \tb\\",
+ },
+ {
+ in: SimpleMakeString("a\\b\\", NoPos),
+ expected: "a\\b\\",
+ },
+}
+
+func TestMakeStringValue(t *testing.T) {
+ for _, test := range valueTestCases {
+ got := test.in.Value(nil)
+ if got != test.expected {
+ t.Errorf("\nwith: %q\nwant: %q\n got: %q", test.in.Dump(), test.expected, got)
+ }
+ }
+}
+
+var splitWordsTestCases = []struct {
+ in *MakeString
+ expected []*MakeString
+}{
+ {
+ in: SimpleMakeString("", NoPos),
+ expected: []*MakeString{},
+ },
+ {
+ in: SimpleMakeString(" a b\\ c d", NoPos),
+ expected: []*MakeString{
+ SimpleMakeString("a", NoPos),
+ SimpleMakeString("b\\ c", NoPos),
+ SimpleMakeString("d", NoPos),
+ },
+ },
+ {
+ in: SimpleMakeString(" a\tb\\\t\\ c d ", NoPos),
+ expected: []*MakeString{
+ SimpleMakeString("a", NoPos),
+ SimpleMakeString("b\\\t\\ c", NoPos),
+ SimpleMakeString("d", NoPos),
+ },
+ },
+ {
+ in: SimpleMakeString(`a\\ b\\\ c d`, NoPos),
+ expected: []*MakeString{
+ SimpleMakeString(`a\\`, NoPos),
+ SimpleMakeString(`b\\\ c`, NoPos),
+ SimpleMakeString("d", NoPos),
+ },
+ },
+}
+
+func TestMakeStringWords(t *testing.T) {
+ for _, test := range splitWordsTestCases {
+ got := test.in.Words()
+ gotString := dumpArray(got)
+ expectedString := dumpArray(test.expected)
+ if gotString != expectedString {
+ t.Errorf("with:\n%q\nexpected:\n%s\ngot:\n%s", test.in.Dump(), expectedString, gotString)
+ }
+ }
+}
+
func dumpArray(a []*MakeString) string {
ret := make([]string, len(a))
diff --git a/androidmk/parser/parser.go b/androidmk/parser/parser.go
index 89ee308..89c1af9 100644
--- a/androidmk/parser/parser.go
+++ b/androidmk/parser/parser.go
@@ -35,6 +35,10 @@
return fmt.Sprintf("%s: %s", e.Pos, e.Err)
}
+const builtinDollar = "__builtin_dollar"
+
+var builtinDollarName = SimpleMakeString(builtinDollar, NoPos)
+
func (p *parser) Parse() ([]Node, []error) {
defer func() {
if r := recover(); r != nil {
@@ -326,7 +330,11 @@
case '$':
var variable Variable
variable = p.parseVariable()
- value.appendVariable(variable)
+ if variable.Name == builtinDollarName {
+ value.appendString("$")
+ } else {
+ value.appendVariable(variable)
+ }
case scanner.EOF:
break loop
case '(':
@@ -357,7 +365,8 @@
case '{':
return p.parseBracketedVariable('{', '}', pos)
case '$':
- name = SimpleMakeString("__builtin_dollar", NoPos)
+ name = builtinDollarName
+ p.accept(p.tok)
case scanner.EOF:
p.errorf("expected variable name, found %s",
scanner.TokenString(p.tok))
@@ -457,6 +466,8 @@
case '=':
p.parseAssignment("=", target, prerequisites)
return nil, true
+ case scanner.EOF:
+ // do nothing
default:
p.errorf("unexpected token %s after rule prerequisites", scanner.TokenString(p.tok))
}
diff --git a/androidmk/parser/parser_test.go b/androidmk/parser/parser_test.go
new file mode 100644
index 0000000..f562c29
--- /dev/null
+++ b/androidmk/parser/parser_test.go
@@ -0,0 +1,61 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+ "bytes"
+ "testing"
+)
+
+var parserTestCases = []struct {
+ name string
+ in string
+ out []Node
+}{
+ {
+ name: "Escaped $",
+ in: `a$$ b: c`,
+ out: []Node{
+ &Rule{
+ Target: SimpleMakeString("a$ b", NoPos),
+ Prerequisites: SimpleMakeString("c", NoPos),
+ },
+ },
+ },
+}
+
+func TestParse(t *testing.T) {
+ for _, test := range parserTestCases {
+ t.Run(test.name, func(t *testing.T) {
+ p := NewParser(test.name, bytes.NewBufferString(test.in))
+ got, errs := p.Parse()
+
+ if len(errs) != 0 {
+ t.Fatalf("Unexpected errors while parsing: %v", errs)
+ }
+
+ if len(got) != len(test.out) {
+ t.Fatalf("length mismatch, expected %d nodes, got %d", len(test.out), len(got))
+ }
+
+ for i := range got {
+ if got[i].Dump() != test.out[i].Dump() {
+ t.Errorf("incorrect node %d:\nexpected: %#v (%s)\n got: %#v (%s)",
+ i, test.out[i], test.out[i].Dump(), got[i], got[i].Dump())
+ }
+ }
+ })
+ }
+}
diff --git a/androidmk/parser/scope.go b/androidmk/parser/scope.go
index 7a514fa..167e470 100644
--- a/androidmk/parser/scope.go
+++ b/androidmk/parser/scope.go
@@ -71,7 +71,7 @@
func init() {
builtinScope := make(map[string]string)
- builtinScope["__builtin_dollar"] = "$"
+ builtinScope[builtinDollar] = "$"
}
func (v Variable) EvalFunction(scope Scope) (string, bool) {