Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 1 | // Copyright 2020 Google Inc. All rights reserved. |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | package bazel |
| 16 | |
| 17 | import ( |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 18 | "crypto/sha256" |
Usta Shrestha | 2ccdb42 | 2022-06-02 10:19:13 -0400 | [diff] [blame] | 19 | "encoding/base64" |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 20 | "fmt" |
| 21 | "path/filepath" |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 22 | "reflect" |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 23 | "sort" |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 24 | "strings" |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 25 | "sync" |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 26 | |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 27 | analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" |
| 28 | |
| 29 | "github.com/google/blueprint/metrics" |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 30 | "github.com/google/blueprint/proptools" |
Jason Wu | 118fd2b | 2022-10-27 18:41:15 +0000 | [diff] [blame] | 31 | "google.golang.org/protobuf/proto" |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 32 | ) |
| 33 | |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 34 | type artifactId int |
| 35 | type depsetId int |
| 36 | type pathFragmentId int |
| 37 | |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 38 | // artifact contains relevant portions of Bazel's aquery proto, Artifact. |
| 39 | // Represents a single artifact, whether it's a source file or a derived output file. |
| 40 | type artifact struct { |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 41 | Id artifactId |
| 42 | PathFragmentId pathFragmentId |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 43 | } |
| 44 | |
| 45 | type pathFragment struct { |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 46 | Id pathFragmentId |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 47 | Label string |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 48 | ParentId pathFragmentId |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 49 | } |
| 50 | |
| 51 | // KeyValuePair represents Bazel's aquery proto, KeyValuePair. |
| 52 | type KeyValuePair struct { |
| 53 | Key string |
| 54 | Value string |
| 55 | } |
| 56 | |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 57 | // AqueryDepset is a depset definition from Bazel's aquery response. This is |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 58 | // akin to the `depSetOfFiles` in the response proto, except: |
Colin Cross | d079e0b | 2022-08-16 10:27:33 -0700 | [diff] [blame] | 59 | // - direct artifacts are enumerated by full path instead of by ID |
| 60 | // - it has a hash of the depset contents, instead of an int ID (for determinism) |
| 61 | // |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 62 | // A depset is a data structure for efficient transitive handling of artifact |
| 63 | // paths. A single depset consists of one or more artifact paths and one or |
| 64 | // more "child" depsets. |
| 65 | type AqueryDepset struct { |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 66 | ContentHash string |
| 67 | DirectArtifacts []string |
| 68 | TransitiveDepSetHashes []string |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 69 | } |
| 70 | |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 71 | // depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles. |
| 72 | // Represents a data structure containing one or more files. Depsets in Bazel are an efficient |
| 73 | // data structure for storing large numbers of file paths. |
| 74 | type depSetOfFiles struct { |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 75 | Id depsetId |
| 76 | DirectArtifactIds []artifactId |
| 77 | TransitiveDepSetIds []depsetId |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 78 | } |
| 79 | |
| 80 | // action contains relevant portions of Bazel's aquery proto, Action. |
| 81 | // Represents a single command line invocation in the Bazel build graph. |
| 82 | type action struct { |
| 83 | Arguments []string |
| 84 | EnvironmentVariables []KeyValuePair |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 85 | InputDepSetIds []depsetId |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 86 | Mnemonic string |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 87 | OutputIds []artifactId |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 88 | TemplateContent string |
| 89 | Substitutions []KeyValuePair |
Sasha Smundak | 1da064c | 2022-06-08 16:36:16 -0700 | [diff] [blame] | 90 | FileContents string |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 91 | } |
| 92 | |
| 93 | // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer. |
| 94 | // An aquery response from Bazel contains a single ActionGraphContainer proto. |
| 95 | type actionGraphContainer struct { |
| 96 | Artifacts []artifact |
| 97 | Actions []action |
| 98 | DepSetOfFiles []depSetOfFiles |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 99 | PathFragments []pathFragment |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 100 | } |
| 101 | |
| 102 | // BuildStatement contains information to register a build statement corresponding (one to one) |
| 103 | // with a Bazel action from Bazel's action graph. |
| 104 | type BuildStatement struct { |
Liz Kammer | c49e682 | 2021-06-08 15:04:11 -0400 | [diff] [blame] | 105 | Command string |
| 106 | Depfile *string |
| 107 | OutputPaths []string |
Liz Kammer | c49e682 | 2021-06-08 15:04:11 -0400 | [diff] [blame] | 108 | SymlinkPaths []string |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 109 | Env []*analysis_v2_proto.KeyValuePair |
Liz Kammer | c49e682 | 2021-06-08 15:04:11 -0400 | [diff] [blame] | 110 | Mnemonic string |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 111 | |
| 112 | // Inputs of this build statement, either as unexpanded depsets or expanded |
| 113 | // input paths. There should be no overlap between these fields; an input |
| 114 | // path should either be included as part of an unexpanded depset or a raw |
| 115 | // input path string, but not both. |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 116 | InputDepsetHashes []string |
| 117 | InputPaths []string |
Cole Faust | e9ae480 | 2023-07-18 19:36:41 -0700 | [diff] [blame^] | 118 | OrderOnlyInputs []string |
Sasha Smundak | 1da064c | 2022-06-08 16:36:16 -0700 | [diff] [blame] | 119 | FileContents string |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 120 | } |
| 121 | |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 122 | // A helper type for aquery processing which facilitates retrieval of path IDs from their |
| 123 | // less readable Bazel structures (depset and path fragment). |
| 124 | type aqueryArtifactHandler struct { |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 125 | // Maps depset id to AqueryDepset, a representation of depset which is |
| 126 | // post-processed for middleman artifact handling, unhandled artifact |
| 127 | // dropping, content hashing, etc. |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 128 | depsetIdToAqueryDepset map[depsetId]AqueryDepset |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 129 | emptyDepsetIds map[depsetId]struct{} |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 130 | // Maps content hash to AqueryDepset. |
| 131 | depsetHashToAqueryDepset map[string]AqueryDepset |
| 132 | |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 133 | // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening |
| 134 | // may be an expensive operation. |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 135 | depsetHashToArtifactPathsCache sync.Map |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 136 | // Maps artifact ids to fully expanded paths. |
| 137 | artifactIdToPath map[artifactId]string |
Cole Faust | e9ae480 | 2023-07-18 19:36:41 -0700 | [diff] [blame^] | 138 | |
| 139 | extraBuildStatements []*BuildStatement |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 140 | } |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 141 | |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 142 | // The tokens should be substituted with the value specified here, instead of the |
| 143 | // one returned in 'substitutions' of TemplateExpand action. |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 144 | var templateActionOverriddenTokens = map[string]string{ |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 145 | // Uses "python3" for %python_binary% instead of the value returned by aquery |
| 146 | // which is "py3wrapper.sh". See removePy3wrapperScript. |
| 147 | "%python_binary%": "python3", |
| 148 | } |
| 149 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 150 | const ( |
| 151 | middlemanMnemonic = "Middleman" |
| 152 | // The file name of py3wrapper.sh, which is used by py_binary targets. |
| 153 | py3wrapperFileName = "/py3wrapper.sh" |
| 154 | ) |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 155 | |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 156 | func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V { |
| 157 | m := map[K]V{} |
| 158 | for _, v := range values { |
| 159 | m[keyFn(v)] = v |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 160 | } |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 161 | return m |
| 162 | } |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 163 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 164 | func newAqueryHandler(aqueryResult *analysis_v2_proto.ActionGraphContainer) (*aqueryArtifactHandler, error) { |
| 165 | pathFragments := indexBy(aqueryResult.PathFragments, func(pf *analysis_v2_proto.PathFragment) pathFragmentId { |
| 166 | return pathFragmentId(pf.Id) |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 167 | }) |
| 168 | |
Cole Faust | e9ae480 | 2023-07-18 19:36:41 -0700 | [diff] [blame^] | 169 | var extraBuildStatements []*BuildStatement |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 170 | artifactIdToPath := make(map[artifactId]string, len(aqueryResult.Artifacts)) |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 171 | for _, artifact := range aqueryResult.Artifacts { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 172 | artifactPath, err := expandPathFragment(pathFragmentId(artifact.PathFragmentId), pathFragments) |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 173 | if err != nil { |
Chris Parsons | 4f06989 | 2021-01-15 12:22:41 -0500 | [diff] [blame] | 174 | return nil, err |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 175 | } |
Cole Faust | e9ae480 | 2023-07-18 19:36:41 -0700 | [diff] [blame^] | 176 | if strings.HasSuffix(artifactPath, "DumpPlatformClassPath.java") { |
| 177 | // TODO(b/291828210): This is a workaround for the fact that DumpPlatformClassPath.java |
| 178 | // has a timestamp in 2033. We'll copy it to a new file using a order-only dep, so that |
| 179 | // the file is not recopied every build. Technically the order-only dep would produce |
| 180 | // incremental build issues if this was a regular file produced as part of the build, |
| 181 | // but this file is actually created by bazel during analysis, so it's not an issue. |
| 182 | outputPath := "hack_for_b291828210/DumpPlatformClassPath.java" |
| 183 | extraBuildStatements = append(extraBuildStatements, &BuildStatement{ |
| 184 | Command: fmt.Sprintf("cp %s %s", artifactPath, outputPath), |
| 185 | OutputPaths: []string{outputPath}, |
| 186 | OrderOnlyInputs: []string{artifactPath}, |
| 187 | }) |
| 188 | artifactIdToPath[artifactId(artifact.Id)] = outputPath |
| 189 | } else { |
| 190 | artifactIdToPath[artifactId(artifact.Id)] = artifactPath |
| 191 | } |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 192 | } |
Chris Parsons | 943f243 | 2021-01-19 11:36:50 -0500 | [diff] [blame] | 193 | |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 194 | // Map middleman artifact ContentHash to input artifact depset ID. |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 195 | // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, |
Usta Shrestha | 16ac135 | 2022-06-22 11:01:55 -0400 | [diff] [blame] | 196 | // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then, |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 197 | // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for |
| 198 | // that action instead. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 199 | middlemanIdToDepsetIds := map[artifactId][]uint32{} |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 200 | for _, actionEntry := range aqueryResult.Actions { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 201 | if actionEntry.Mnemonic == middlemanMnemonic { |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 202 | for _, outputId := range actionEntry.OutputIds { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 203 | middlemanIdToDepsetIds[artifactId(outputId)] = actionEntry.InputDepSetIds |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 204 | } |
| 205 | } |
| 206 | } |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 207 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 208 | depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d *analysis_v2_proto.DepSetOfFiles) depsetId { |
| 209 | return depsetId(d.Id) |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 210 | }) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 211 | |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 212 | aqueryHandler := aqueryArtifactHandler{ |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 213 | depsetIdToAqueryDepset: map[depsetId]AqueryDepset{}, |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 214 | depsetHashToAqueryDepset: map[string]AqueryDepset{}, |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 215 | depsetHashToArtifactPathsCache: sync.Map{}, |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 216 | emptyDepsetIds: make(map[depsetId]struct{}, 0), |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 217 | artifactIdToPath: artifactIdToPath, |
Cole Faust | e9ae480 | 2023-07-18 19:36:41 -0700 | [diff] [blame^] | 218 | extraBuildStatements: extraBuildStatements, |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 219 | } |
| 220 | |
| 221 | // Validate and adjust aqueryResult.DepSetOfFiles values. |
| 222 | for _, depset := range aqueryResult.DepSetOfFiles { |
| 223 | _, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset) |
| 224 | if err != nil { |
| 225 | return nil, err |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | return &aqueryHandler, nil |
| 230 | } |
| 231 | |
| 232 | // Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given |
| 233 | // depset. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 234 | func (a *aqueryArtifactHandler) populateDepsetMaps(depset *analysis_v2_proto.DepSetOfFiles, middlemanIdToDepsetIds map[artifactId][]uint32, depsetIdToDepset map[depsetId]*analysis_v2_proto.DepSetOfFiles) (*AqueryDepset, error) { |
| 235 | if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depsetId(depset.Id)]; containsDepset { |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 236 | return &aqueryDepset, nil |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 237 | } |
| 238 | transitiveDepsetIds := depset.TransitiveDepSetIds |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 239 | directArtifactPaths := make([]string, 0, len(depset.DirectArtifactIds)) |
| 240 | for _, id := range depset.DirectArtifactIds { |
| 241 | aId := artifactId(id) |
| 242 | path, pathExists := a.artifactIdToPath[aId] |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 243 | if !pathExists { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 244 | return nil, fmt.Errorf("undefined input artifactId %d", aId) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 245 | } |
| 246 | // Filter out any inputs which are universally dropped, and swap middleman |
| 247 | // artifacts with their corresponding depsets. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 248 | if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[aId]; isMiddleman { |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 249 | // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts. |
| 250 | transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...) |
Usta Shrestha | ef92225 | 2022-06-02 14:23:02 -0400 | [diff] [blame] | 251 | } else if strings.HasSuffix(path, py3wrapperFileName) || |
Usta Shrestha | ef92225 | 2022-06-02 14:23:02 -0400 | [diff] [blame] | 252 | strings.HasPrefix(path, "../bazel_tools") { |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 253 | continue |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 254 | // Drop these artifacts. |
| 255 | // See go/python-binary-host-mixed-build for more details. |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 256 | // 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the |
| 257 | // TemplateExpandAction handles everything necessary to launch a Pythin application. |
| 258 | // 2) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the |
Usta Shrestha | ef92225 | 2022-06-02 14:23:02 -0400 | [diff] [blame] | 259 | // containing depset to always be considered newer than their outputs. |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 260 | } else { |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 261 | directArtifactPaths = append(directArtifactPaths, path) |
| 262 | } |
| 263 | } |
| 264 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 265 | childDepsetHashes := make([]string, 0, len(transitiveDepsetIds)) |
| 266 | for _, id := range transitiveDepsetIds { |
| 267 | childDepsetId := depsetId(id) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 268 | childDepset, exists := depsetIdToDepset[childDepsetId] |
| 269 | if !exists { |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 270 | if _, empty := a.emptyDepsetIds[childDepsetId]; empty { |
| 271 | continue |
| 272 | } else { |
| 273 | return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id) |
| 274 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 275 | } |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 276 | if childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset); err != nil { |
| 277 | return nil, err |
| 278 | } else if childAqueryDepset == nil { |
| 279 | continue |
| 280 | } else { |
| 281 | childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 282 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 283 | } |
Usta Shrestha | ef92225 | 2022-06-02 14:23:02 -0400 | [diff] [blame] | 284 | if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 285 | a.emptyDepsetIds[depsetId(depset.Id)] = struct{}{} |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 286 | return nil, nil |
Usta Shrestha | ef92225 | 2022-06-02 14:23:02 -0400 | [diff] [blame] | 287 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 288 | aqueryDepset := AqueryDepset{ |
| 289 | ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes), |
| 290 | DirectArtifacts: directArtifactPaths, |
| 291 | TransitiveDepSetHashes: childDepsetHashes, |
| 292 | } |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 293 | a.depsetIdToAqueryDepset[depsetId(depset.Id)] = aqueryDepset |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 294 | a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 295 | return &aqueryDepset, nil |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 296 | } |
| 297 | |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 298 | // getInputPaths flattens the depsets of the given IDs and returns all transitive |
| 299 | // input paths contained in these depsets. |
| 300 | // This is a potentially expensive operation, and should not be invoked except |
| 301 | // for actions which need specialized input handling. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 302 | func (a *aqueryArtifactHandler) getInputPaths(depsetIds []uint32) ([]string, error) { |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 303 | var inputPaths []string |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 304 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 305 | for _, id := range depsetIds { |
| 306 | inputDepSetId := depsetId(id) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 307 | depset := a.depsetIdToAqueryDepset[inputDepSetId] |
| 308 | inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash) |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 309 | if err != nil { |
| 310 | return nil, err |
| 311 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 312 | for _, inputPath := range inputArtifacts { |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 313 | inputPaths = append(inputPaths, inputPath) |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 314 | } |
| 315 | } |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 316 | |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 317 | return inputPaths, nil |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 318 | } |
| 319 | |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 320 | func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) { |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 321 | if result, exists := a.depsetHashToArtifactPathsCache.Load(depsetHash); exists { |
| 322 | return result.([]string), nil |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 323 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 324 | if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists { |
| 325 | result := depset.DirectArtifacts |
| 326 | for _, childHash := range depset.TransitiveDepSetHashes { |
| 327 | childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash) |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 328 | if err != nil { |
| 329 | return nil, err |
| 330 | } |
| 331 | result = append(result, childArtifactIds...) |
| 332 | } |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 333 | a.depsetHashToArtifactPathsCache.Store(depsetHash, result) |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 334 | return result, nil |
| 335 | } else { |
Usta Shrestha | 2ccdb42 | 2022-06-02 10:19:13 -0400 | [diff] [blame] | 336 | return nil, fmt.Errorf("undefined input depset hash %s", depsetHash) |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 337 | } |
| 338 | } |
| 339 | |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 340 | // AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 341 | // which should be registered (and output to a ninja file) to correspond with Bazel's |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 342 | // action graph, as described by the given action graph json proto. |
| 343 | // BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets |
| 344 | // are one-to-one with Bazel's depSetOfFiles objects. |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 345 | func AqueryBuildStatements(aqueryJsonProto []byte, eventHandler *metrics.EventHandler) ([]*BuildStatement, []AqueryDepset, error) { |
Jason Wu | 118fd2b | 2022-10-27 18:41:15 +0000 | [diff] [blame] | 346 | aqueryProto := &analysis_v2_proto.ActionGraphContainer{} |
| 347 | err := proto.Unmarshal(aqueryJsonProto, aqueryProto) |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 348 | if err != nil { |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 349 | return nil, nil, err |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 350 | } |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 351 | |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 352 | var aqueryHandler *aqueryArtifactHandler |
| 353 | { |
| 354 | eventHandler.Begin("init_handler") |
| 355 | defer eventHandler.End("init_handler") |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 356 | aqueryHandler, err = newAqueryHandler(aqueryProto) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 357 | if err != nil { |
| 358 | return nil, nil, err |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 359 | } |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 360 | } |
| 361 | |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 362 | // allocate both length and capacity so each goroutine can write to an index independently without |
| 363 | // any need for synchronization for slice access. |
| 364 | buildStatements := make([]*BuildStatement, len(aqueryProto.Actions)) |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 365 | { |
| 366 | eventHandler.Begin("build_statements") |
| 367 | defer eventHandler.End("build_statements") |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 368 | wg := sync.WaitGroup{} |
| 369 | var errOnce sync.Once |
| 370 | |
| 371 | for i, actionEntry := range aqueryProto.Actions { |
| 372 | wg.Add(1) |
| 373 | go func(i int, actionEntry *analysis_v2_proto.Action) { |
| 374 | buildStatement, aErr := aqueryHandler.actionToBuildStatement(actionEntry) |
| 375 | if aErr != nil { |
| 376 | errOnce.Do(func() { |
| 377 | err = aErr |
| 378 | }) |
| 379 | } else { |
| 380 | // set build statement at an index rather than appending such that each goroutine does not |
| 381 | // impact other goroutines |
| 382 | buildStatements[i] = buildStatement |
| 383 | } |
| 384 | wg.Done() |
| 385 | }(i, actionEntry) |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 386 | } |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 387 | wg.Wait() |
| 388 | } |
| 389 | if err != nil { |
| 390 | return nil, nil, err |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 391 | } |
Cole Faust | e9ae480 | 2023-07-18 19:36:41 -0700 | [diff] [blame^] | 392 | buildStatements = append(buildStatements, aqueryHandler.extraBuildStatements...) |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 393 | |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 394 | depsetsByHash := map[string]AqueryDepset{} |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 395 | depsets := make([]AqueryDepset, 0, len(aqueryHandler.depsetIdToAqueryDepset)) |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 396 | { |
| 397 | eventHandler.Begin("depsets") |
| 398 | defer eventHandler.End("depsets") |
| 399 | for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset { |
| 400 | if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey { |
| 401 | // Two depsets collide on hash. Ensure that their contents are identical. |
| 402 | if !reflect.DeepEqual(aqueryDepset, prevEntry) { |
| 403 | return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset) |
| 404 | } |
| 405 | } else { |
| 406 | depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset |
| 407 | depsets = append(depsets, aqueryDepset) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 408 | } |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 409 | } |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 410 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 411 | |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 412 | eventHandler.Do("build_statement_sort", func() { |
| 413 | // Build Statements and depsets must be sorted by their content hash to |
| 414 | // preserve determinism between builds (this will result in consistent ninja file |
| 415 | // output). Note they are not sorted by their original IDs nor their Bazel ordering, |
| 416 | // as Bazel gives nondeterministic ordering / identifiers in aquery responses. |
| 417 | sort.Slice(buildStatements, func(i, j int) bool { |
Liz Kammer | a4655a9 | 2023-02-10 17:17:28 -0500 | [diff] [blame] | 418 | // Sort all nil statements to the end of the slice |
| 419 | if buildStatements[i] == nil { |
| 420 | return false |
| 421 | } else if buildStatements[j] == nil { |
| 422 | return true |
| 423 | } |
| 424 | //For build statements, compare output lists. In Bazel, each output file |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 425 | // may only have one action which generates it, so this will provide |
| 426 | // a deterministic ordering. |
| 427 | outputs_i := buildStatements[i].OutputPaths |
| 428 | outputs_j := buildStatements[j].OutputPaths |
| 429 | if len(outputs_i) != len(outputs_j) { |
| 430 | return len(outputs_i) < len(outputs_j) |
| 431 | } |
| 432 | if len(outputs_i) == 0 { |
| 433 | // No outputs for these actions, so compare commands. |
| 434 | return buildStatements[i].Command < buildStatements[j].Command |
| 435 | } |
| 436 | // There may be multiple outputs, but the output ordering is deterministic. |
| 437 | return outputs_i[0] < outputs_j[0] |
| 438 | }) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 439 | }) |
Liz Kammer | 690fbac | 2023-02-10 11:11:17 -0500 | [diff] [blame] | 440 | eventHandler.Do("depset_sort", func() { |
| 441 | sort.Slice(depsets, func(i, j int) bool { |
| 442 | return depsets[i].ContentHash < depsets[j].ContentHash |
| 443 | }) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 444 | }) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 445 | return buildStatements, depsets, nil |
| 446 | } |
| 447 | |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 448 | // depsetContentHash computes and returns a SHA256 checksum of the contents of |
| 449 | // the given depset. This content hash may serve as the depset's identifier. |
| 450 | // Using a content hash for an identifier is superior for determinism. (For example, |
| 451 | // using an integer identifier which depends on the order in which the depsets are |
| 452 | // created would result in nondeterministic depset IDs.) |
| 453 | func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string { |
| 454 | h := sha256.New() |
| 455 | // Use newline as delimiter, as paths cannot contain newline. |
| 456 | h.Write([]byte(strings.Join(directPaths, "\n"))) |
Usta Shrestha | 2ccdb42 | 2022-06-02 10:19:13 -0400 | [diff] [blame] | 457 | h.Write([]byte(strings.Join(transitiveDepsetHashes, ""))) |
| 458 | fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 459 | return fullHash |
| 460 | } |
| 461 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 462 | func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []uint32) ([]string, error) { |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 463 | var hashes []string |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 464 | for _, id := range inputDepsetIds { |
| 465 | dId := depsetId(id) |
| 466 | if aqueryDepset, exists := a.depsetIdToAqueryDepset[dId]; !exists { |
| 467 | if _, empty := a.emptyDepsetIds[dId]; !empty { |
| 468 | return nil, fmt.Errorf("undefined (not even empty) input depsetId %d", dId) |
Usta Shrestha | 13fd5ae | 2023-01-27 10:55:34 -0500 | [diff] [blame] | 469 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 470 | } else { |
| 471 | hashes = append(hashes, aqueryDepset.ContentHash) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 472 | } |
| 473 | } |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 474 | return hashes, nil |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 475 | } |
| 476 | |
Spandan Das | da72486 | 2023-06-16 23:35:55 +0000 | [diff] [blame] | 477 | // escapes the args received from aquery and creates a command string |
| 478 | func commandString(actionEntry *analysis_v2_proto.Action) string { |
| 479 | switch actionEntry.Mnemonic { |
| 480 | case "GoCompilePkg": |
| 481 | argsEscaped := []string{} |
| 482 | for _, arg := range actionEntry.Arguments { |
| 483 | if arg == "" { |
| 484 | // If this is an empty string, add '' |
| 485 | // And not |
| 486 | // 1. (literal empty) |
| 487 | // 2. `''\'''\'''` (escaped version of '') |
| 488 | // |
| 489 | // If we had used (1), then this would appear as a whitespace when we strings.Join |
| 490 | argsEscaped = append(argsEscaped, "''") |
| 491 | } else { |
| 492 | argsEscaped = append(argsEscaped, proptools.ShellEscapeIncludingSpaces(arg)) |
| 493 | } |
| 494 | } |
| 495 | return strings.Join(argsEscaped, " ") |
| 496 | default: |
| 497 | return strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ") |
| 498 | } |
| 499 | } |
| 500 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 501 | func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { |
Spandan Das | da72486 | 2023-06-16 23:35:55 +0000 | [diff] [blame] | 502 | command := commandString(actionEntry) |
Usta Shrestha | c237249 | 2022-05-27 10:45:00 -0400 | [diff] [blame] | 503 | inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 504 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 505 | return nil, err |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 506 | } |
Usta Shrestha | c237249 | 2022-05-27 10:45:00 -0400 | [diff] [blame] | 507 | outputPaths, depfile, err := a.getOutputPaths(actionEntry) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 508 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 509 | return nil, err |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 510 | } |
| 511 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 512 | buildStatement := &BuildStatement{ |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 513 | Command: command, |
| 514 | Depfile: depfile, |
| 515 | OutputPaths: outputPaths, |
| 516 | InputDepsetHashes: inputDepsetHashes, |
| 517 | Env: actionEntry.EnvironmentVariables, |
| 518 | Mnemonic: actionEntry.Mnemonic, |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 519 | } |
| 520 | return buildStatement, nil |
| 521 | } |
| 522 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 523 | func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { |
Usta Shrestha | c237249 | 2022-05-27 10:45:00 -0400 | [diff] [blame] | 524 | outputPaths, depfile, err := a.getOutputPaths(actionEntry) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 525 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 526 | return nil, err |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 527 | } |
| 528 | if len(outputPaths) != 1 { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 529 | return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 530 | } |
| 531 | expandedTemplateContent := expandTemplateContent(actionEntry) |
| 532 | // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, |
| 533 | // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might |
| 534 | // change \n to space and mess up the format of Python programs. |
| 535 | // sed is used to convert \\n back to \n before saving to output file. |
| 536 | // See go/python-binary-host-mixed-build for more details. |
| 537 | command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, |
| 538 | escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) |
Usta Shrestha | c237249 | 2022-05-27 10:45:00 -0400 | [diff] [blame] | 539 | inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 540 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 541 | return nil, err |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 542 | } |
| 543 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 544 | buildStatement := &BuildStatement{ |
Chris Parsons | 0bfb1c0 | 2022-05-12 16:43:01 -0400 | [diff] [blame] | 545 | Command: command, |
| 546 | Depfile: depfile, |
| 547 | OutputPaths: outputPaths, |
| 548 | InputDepsetHashes: inputDepsetHashes, |
| 549 | Env: actionEntry.EnvironmentVariables, |
| 550 | Mnemonic: actionEntry.Mnemonic, |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 551 | } |
| 552 | return buildStatement, nil |
| 553 | } |
| 554 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 555 | func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { |
Sasha Smundak | 1da064c | 2022-06-08 16:36:16 -0700 | [diff] [blame] | 556 | outputPaths, _, err := a.getOutputPaths(actionEntry) |
| 557 | var depsetHashes []string |
| 558 | if err == nil { |
| 559 | depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds) |
| 560 | } |
| 561 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 562 | return nil, err |
Sasha Smundak | 1da064c | 2022-06-08 16:36:16 -0700 | [diff] [blame] | 563 | } |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 564 | return &BuildStatement{ |
Sasha Smundak | 1da064c | 2022-06-08 16:36:16 -0700 | [diff] [blame] | 565 | Depfile: nil, |
| 566 | OutputPaths: outputPaths, |
| 567 | Env: actionEntry.EnvironmentVariables, |
| 568 | Mnemonic: actionEntry.Mnemonic, |
| 569 | InputDepsetHashes: depsetHashes, |
| 570 | FileContents: actionEntry.FileContents, |
| 571 | }, nil |
| 572 | } |
| 573 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 574 | func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 575 | outputPaths, _, err := a.getOutputPaths(actionEntry) |
| 576 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 577 | return nil, err |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 578 | } |
| 579 | inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) |
| 580 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 581 | return nil, err |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 582 | } |
| 583 | if len(inputPaths) != 1 || len(outputPaths) != 1 { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 584 | return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 585 | } |
| 586 | // The actual command is generated in bazelSingleton.GenerateBuildActions |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 587 | return &BuildStatement{ |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 588 | Depfile: nil, |
| 589 | OutputPaths: outputPaths, |
| 590 | Env: actionEntry.EnvironmentVariables, |
| 591 | Mnemonic: actionEntry.Mnemonic, |
| 592 | InputPaths: inputPaths, |
| 593 | }, nil |
| 594 | } |
| 595 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 596 | func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { |
Usta Shrestha | c237249 | 2022-05-27 10:45:00 -0400 | [diff] [blame] | 597 | outputPaths, depfile, err := a.getOutputPaths(actionEntry) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 598 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 599 | return nil, err |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 600 | } |
| 601 | |
Usta Shrestha | c237249 | 2022-05-27 10:45:00 -0400 | [diff] [blame] | 602 | inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 603 | if err != nil { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 604 | return nil, err |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 605 | } |
| 606 | if len(inputPaths) != 1 || len(outputPaths) != 1 { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 607 | return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 608 | } |
| 609 | out := outputPaths[0] |
| 610 | outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) |
| 611 | out = proptools.ShellEscapeIncludingSpaces(out) |
| 612 | in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) |
| 613 | // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). |
| 614 | command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) |
| 615 | symlinkPaths := outputPaths[:] |
| 616 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 617 | buildStatement := &BuildStatement{ |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 618 | Command: command, |
| 619 | Depfile: depfile, |
| 620 | OutputPaths: outputPaths, |
| 621 | InputPaths: inputPaths, |
| 622 | Env: actionEntry.EnvironmentVariables, |
| 623 | Mnemonic: actionEntry.Mnemonic, |
| 624 | SymlinkPaths: symlinkPaths, |
| 625 | } |
| 626 | return buildStatement, nil |
| 627 | } |
| 628 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 629 | func (a *aqueryArtifactHandler) getOutputPaths(actionEntry *analysis_v2_proto.Action) (outputPaths []string, depfile *string, err error) { |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 630 | for _, outputId := range actionEntry.OutputIds { |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 631 | outputPath, exists := a.artifactIdToPath[artifactId(outputId)] |
Chris Parsons | 1a7aca0 | 2022-04-25 22:35:15 -0400 | [diff] [blame] | 632 | if !exists { |
| 633 | err = fmt.Errorf("undefined outputId %d", outputId) |
| 634 | return |
| 635 | } |
| 636 | ext := filepath.Ext(outputPath) |
| 637 | if ext == ".d" { |
| 638 | if depfile != nil { |
| 639 | err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) |
| 640 | return |
| 641 | } else { |
| 642 | depfile = &outputPath |
| 643 | } |
| 644 | } else { |
| 645 | outputPaths = append(outputPaths, outputPath) |
| 646 | } |
| 647 | } |
| 648 | return |
Chris Parsons | dbcb1ff | 2020-12-10 17:19:18 -0500 | [diff] [blame] | 649 | } |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 650 | |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 651 | // expandTemplateContent substitutes the tokens in a template. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 652 | func expandTemplateContent(actionEntry *analysis_v2_proto.Action) string { |
| 653 | replacerString := make([]string, len(actionEntry.Substitutions)*2) |
| 654 | for i, pair := range actionEntry.Substitutions { |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 655 | value := pair.Value |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 656 | if val, ok := templateActionOverriddenTokens[pair.Key]; ok { |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 657 | value = val |
| 658 | } |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 659 | replacerString[i*2] = pair.Key |
| 660 | replacerString[i*2+1] = value |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 661 | } |
| 662 | replacer := strings.NewReplacer(replacerString...) |
| 663 | return replacer.Replace(actionEntry.TemplateContent) |
| 664 | } |
| 665 | |
Liz Kammer | f15a079 | 2023-02-09 14:28:36 -0500 | [diff] [blame] | 666 | // \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"' |
| 667 | var commandLineArgumentReplacer = strings.NewReplacer( |
| 668 | `\`, `\\`, |
| 669 | `$`, `\$`, |
| 670 | "`", "\\`", |
| 671 | `"`, `\"`, |
| 672 | "\n", "\\n", |
| 673 | `'`, `'"'"'`, |
| 674 | ) |
| 675 | |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 676 | func escapeCommandlineArgument(str string) string { |
Liz Kammer | f15a079 | 2023-02-09 14:28:36 -0500 | [diff] [blame] | 677 | return commandLineArgumentReplacer.Replace(str) |
Wei Li | 455ba83 | 2021-11-04 22:58:12 +0000 | [diff] [blame] | 678 | } |
| 679 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 680 | func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { |
| 681 | switch actionEntry.Mnemonic { |
Chris Parsons | c4fb133 | 2021-05-18 12:31:25 -0400 | [diff] [blame] | 682 | // Middleman actions are not handled like other actions; they are handled separately as a |
| 683 | // preparatory step so that their inputs may be relayed to actions depending on middleman |
| 684 | // artifacts. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 685 | case middlemanMnemonic: |
| 686 | return nil, nil |
Sasha Smundak | c180dbd | 2022-07-03 14:55:58 -0700 | [diff] [blame] | 687 | // PythonZipper is bogus action returned by aquery, ignore it (b/236198693) |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 688 | case "PythonZipper": |
| 689 | return nil, nil |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 690 | // Skip "Fail" actions, which are placeholder actions designed to always fail. |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 691 | case "Fail": |
| 692 | return nil, nil |
| 693 | case "BaselineCoverage": |
| 694 | return nil, nil |
| 695 | case "Symlink", "SolibSymlink", "ExecutableSymlink": |
| 696 | return a.symlinkActionBuildStatement(actionEntry) |
| 697 | case "TemplateExpand": |
| 698 | if len(actionEntry.Arguments) < 1 { |
| 699 | return a.templateExpandActionBuildStatement(actionEntry) |
| 700 | } |
Cole Faust | 950689a | 2023-06-21 15:07:21 -0700 | [diff] [blame] | 701 | case "FileWrite", "SourceSymlinkManifest", "RepoMappingManifest": |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 702 | return a.fileWriteActionBuildStatement(actionEntry) |
| 703 | case "SymlinkTree": |
| 704 | return a.symlinkTreeActionBuildStatement(actionEntry) |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 705 | } |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 706 | |
| 707 | if len(actionEntry.Arguments) < 1 { |
| 708 | return nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic) |
Yu Liu | 8d82ac5 | 2022-05-17 15:13:28 -0700 | [diff] [blame] | 709 | } |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 710 | return a.normalActionBuildStatement(actionEntry) |
| 711 | |
Chris Parsons | 8d6e433 | 2021-02-22 16:13:50 -0500 | [diff] [blame] | 712 | } |
| 713 | |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 714 | func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]*analysis_v2_proto.PathFragment) (string, error) { |
Usta Shrestha | 6298cc5 | 2022-05-27 17:40:21 -0400 | [diff] [blame] | 715 | var labels []string |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 716 | currId := id |
| 717 | // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node. |
| 718 | for currId > 0 { |
| 719 | currFragment, ok := pathFragmentsMap[currId] |
| 720 | if !ok { |
Chris Parsons | 4f06989 | 2021-01-15 12:22:41 -0500 | [diff] [blame] | 721 | return "", fmt.Errorf("undefined path fragment id %d", currId) |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 722 | } |
| 723 | labels = append([]string{currFragment.Label}, labels...) |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 724 | parentId := pathFragmentId(currFragment.ParentId) |
| 725 | if currId == parentId { |
Sasha Smundak | fe9a5b8 | 2022-07-27 14:51:45 -0700 | [diff] [blame] | 726 | return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment) |
Liz Kammer | c49e682 | 2021-06-08 15:04:11 -0400 | [diff] [blame] | 727 | } |
Liz Kammer | 00629db | 2023-02-09 14:28:15 -0500 | [diff] [blame] | 728 | currId = parentId |
Chris Parsons | affbb60 | 2020-12-23 12:02:11 -0500 | [diff] [blame] | 729 | } |
| 730 | return filepath.Join(labels...), nil |
| 731 | } |