blob: c82b464ad5428254106990fcf29bd98f5d91cb2b [file] [log] [blame]
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -05001// Copyright 2020 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package bazel
16
17import (
18 "encoding/json"
Chris Parsonsaffbb602020-12-23 12:02:11 -050019 "fmt"
20 "path/filepath"
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050021 "strings"
22
23 "github.com/google/blueprint/proptools"
24)
25
26// artifact contains relevant portions of Bazel's aquery proto, Artifact.
27// Represents a single artifact, whether it's a source file or a derived output file.
28type artifact struct {
Chris Parsonsaffbb602020-12-23 12:02:11 -050029 Id int
30 PathFragmentId int
31}
32
33type pathFragment struct {
34 Id int
35 Label string
36 ParentId int
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050037}
38
39// KeyValuePair represents Bazel's aquery proto, KeyValuePair.
40type KeyValuePair struct {
41 Key string
42 Value string
43}
44
45// depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
46// Represents a data structure containing one or more files. Depsets in Bazel are an efficient
47// data structure for storing large numbers of file paths.
48type depSetOfFiles struct {
Chris Parsons943f2432021-01-19 11:36:50 -050049 Id int
50 DirectArtifactIds []int
51 TransitiveDepSetIds []int
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050052}
53
54// action contains relevant portions of Bazel's aquery proto, Action.
55// Represents a single command line invocation in the Bazel build graph.
56type action struct {
57 Arguments []string
58 EnvironmentVariables []KeyValuePair
Chris Parsonsaffbb602020-12-23 12:02:11 -050059 InputDepSetIds []int
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050060 Mnemonic string
Chris Parsonsaffbb602020-12-23 12:02:11 -050061 OutputIds []int
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050062}
63
64// actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
65// An aquery response from Bazel contains a single ActionGraphContainer proto.
66type actionGraphContainer struct {
67 Artifacts []artifact
68 Actions []action
69 DepSetOfFiles []depSetOfFiles
Chris Parsonsaffbb602020-12-23 12:02:11 -050070 PathFragments []pathFragment
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050071}
72
73// BuildStatement contains information to register a build statement corresponding (one to one)
74// with a Bazel action from Bazel's action graph.
75type BuildStatement struct {
76 Command string
77 OutputPaths []string
78 InputPaths []string
79 Env []KeyValuePair
80 Mnemonic string
81}
82
83// AqueryBuildStatements returns an array of BuildStatements which should be registered (and output
84// to a ninja file) to correspond one-to-one with the given action graph json proto (from a bazel
85// aquery invocation).
Chris Parsons4f069892021-01-15 12:22:41 -050086func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050087 buildStatements := []BuildStatement{}
88
89 var aqueryResult actionGraphContainer
Chris Parsons4f069892021-01-15 12:22:41 -050090 err := json.Unmarshal(aqueryJsonProto, &aqueryResult)
91
92 if err != nil {
93 return nil, err
94 }
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050095
Chris Parsonsaffbb602020-12-23 12:02:11 -050096 pathFragments := map[int]pathFragment{}
97 for _, pathFragment := range aqueryResult.PathFragments {
98 pathFragments[pathFragment.Id] = pathFragment
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -050099 }
Chris Parsonsaffbb602020-12-23 12:02:11 -0500100 artifactIdToPath := map[int]string{}
101 for _, artifact := range aqueryResult.Artifacts {
102 artifactPath, err := expandPathFragment(artifact.PathFragmentId, pathFragments)
103 if err != nil {
Chris Parsons4f069892021-01-15 12:22:41 -0500104 return nil, err
Chris Parsonsaffbb602020-12-23 12:02:11 -0500105 }
106 artifactIdToPath[artifact.Id] = artifactPath
107 }
Chris Parsons943f2432021-01-19 11:36:50 -0500108
109 depsetIdToDepset := map[int]depSetOfFiles{}
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500110 for _, depset := range aqueryResult.DepSetOfFiles {
Chris Parsons943f2432021-01-19 11:36:50 -0500111 depsetIdToDepset[depset.Id] = depset
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500112 }
113
Chris Parsons943f2432021-01-19 11:36:50 -0500114 // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
115 // may be an expensive operation.
116 depsetIdToArtifactIdsCache := map[int][]int{}
117
Chris Parsons8d6e4332021-02-22 16:13:50 -0500118 // Do a pass through all actions to identify which artifacts are middleman artifacts.
119 // These will be omitted from the inputs of other actions.
120 // TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
121 // headers may cause build failures.
122 middlemanArtifactIds := map[int]bool{}
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500123 for _, actionEntry := range aqueryResult.Actions {
Chris Parsons8d6e4332021-02-22 16:13:50 -0500124 if actionEntry.Mnemonic == "Middleman" {
125 for _, outputId := range actionEntry.OutputIds {
126 middlemanArtifactIds[outputId] = true
127 }
128 }
129 }
130
131 for _, actionEntry := range aqueryResult.Actions {
132 if shouldSkipAction(actionEntry) {
133 continue
134 }
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500135 outputPaths := []string{}
136 for _, outputId := range actionEntry.OutputIds {
Chris Parsons4f069892021-01-15 12:22:41 -0500137 outputPath, exists := artifactIdToPath[outputId]
138 if !exists {
139 return nil, fmt.Errorf("undefined outputId %d", outputId)
140 }
141 outputPaths = append(outputPaths, outputPath)
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500142 }
143 inputPaths := []string{}
144 for _, inputDepSetId := range actionEntry.InputDepSetIds {
Chris Parsons943f2432021-01-19 11:36:50 -0500145 inputArtifacts, err :=
146 artifactIdsFromDepsetId(depsetIdToDepset, depsetIdToArtifactIdsCache, inputDepSetId)
147 if err != nil {
148 return nil, err
Chris Parsons4f069892021-01-15 12:22:41 -0500149 }
150 for _, inputId := range inputArtifacts {
Chris Parsons8d6e4332021-02-22 16:13:50 -0500151 if _, isMiddlemanArtifact := middlemanArtifactIds[inputId]; isMiddlemanArtifact {
152 // Omit middleman artifacts.
153 continue
154 }
Chris Parsons4f069892021-01-15 12:22:41 -0500155 inputPath, exists := artifactIdToPath[inputId]
156 if !exists {
157 return nil, fmt.Errorf("undefined input artifactId %d", inputId)
158 }
159 inputPaths = append(inputPaths, inputPath)
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500160 }
161 }
162 buildStatement := BuildStatement{
163 Command: strings.Join(proptools.ShellEscapeList(actionEntry.Arguments), " "),
164 OutputPaths: outputPaths,
165 InputPaths: inputPaths,
166 Env: actionEntry.EnvironmentVariables,
167 Mnemonic: actionEntry.Mnemonic}
Chris Parsons8d6e4332021-02-22 16:13:50 -0500168 if len(actionEntry.Arguments) < 1 {
169 return nil, fmt.Errorf("received action with no command: [%s]", buildStatement)
170 continue
171 }
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500172 buildStatements = append(buildStatements, buildStatement)
173 }
174
Chris Parsons4f069892021-01-15 12:22:41 -0500175 return buildStatements, nil
Chris Parsonsdbcb1ff2020-12-10 17:19:18 -0500176}
Chris Parsonsaffbb602020-12-23 12:02:11 -0500177
Chris Parsons8d6e4332021-02-22 16:13:50 -0500178func shouldSkipAction(a action) bool {
179 // TODO(b/180945121): Handle symlink actions.
180 if a.Mnemonic == "Symlink" || a.Mnemonic == "SourceSymlinkManifest" || a.Mnemonic == "SymlinkTree" {
181 return true
182 }
183 // TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
184 // headers may cause build failures.
185 if a.Mnemonic == "Middleman" {
186 return true
187 }
188 // Skip "Fail" actions, which are placeholder actions designed to always fail.
189 if a.Mnemonic == "Fail" {
190 return true
191 }
192 // TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information
193 // about the contents that are written.
194 if a.Mnemonic == "FileWrite" {
195 return true
196 }
197 return false
198}
199
Chris Parsons943f2432021-01-19 11:36:50 -0500200func artifactIdsFromDepsetId(depsetIdToDepset map[int]depSetOfFiles,
201 depsetIdToArtifactIdsCache map[int][]int, depsetId int) ([]int, error) {
202 if result, exists := depsetIdToArtifactIdsCache[depsetId]; exists {
203 return result, nil
204 }
205 if depset, exists := depsetIdToDepset[depsetId]; exists {
206 result := depset.DirectArtifactIds
207 for _, childId := range depset.TransitiveDepSetIds {
208 childArtifactIds, err :=
209 artifactIdsFromDepsetId(depsetIdToDepset, depsetIdToArtifactIdsCache, childId)
210 if err != nil {
211 return nil, err
212 }
213 result = append(result, childArtifactIds...)
214 }
215 depsetIdToArtifactIdsCache[depsetId] = result
216 return result, nil
217 } else {
218 return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
219 }
220}
221
Chris Parsonsaffbb602020-12-23 12:02:11 -0500222func expandPathFragment(id int, pathFragmentsMap map[int]pathFragment) (string, error) {
223 labels := []string{}
224 currId := id
225 // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node.
226 for currId > 0 {
227 currFragment, ok := pathFragmentsMap[currId]
228 if !ok {
Chris Parsons4f069892021-01-15 12:22:41 -0500229 return "", fmt.Errorf("undefined path fragment id %d", currId)
Chris Parsonsaffbb602020-12-23 12:02:11 -0500230 }
231 labels = append([]string{currFragment.Label}, labels...)
232 currId = currFragment.ParentId
233 }
234 return filepath.Join(labels...), nil
235}