Emit and parse the product config variables from kati/make
Test: cls && rm -rf out/config/ && m product-config-test product-config && java -jar out/host/linux-x86/testcases/product-config-test/product-config-test.jar && time ( product-config --ckati_bin /source/kati/ckati > ~/Desktop/out.txt )
Change-Id: I52e5c07f9aaf899f9d45680313275c6d9e246ff2
diff --git a/core/dumpconfig.mk b/core/dumpconfig.mk
new file mode 100644
index 0000000..cb7fbcb
--- /dev/null
+++ b/core/dumpconfig.mk
@@ -0,0 +1,121 @@
+# Read and dump the product configuration.
+
+# Called from the product-config tool, not from the main build system.
+
+#
+# Ensure we are being called correctly
+#
+ifndef KATI
+ $(warning Kati must be used to call dumpconfig.mk, not make.)
+ $(error stopping)
+endif
+
+ifdef DEFAULT_GOAL
+ $(warning Calling dumpconfig.mk from inside the make build system is not)
+ $(warning supported. It is only meant to be called via kati by product-confing.)
+ $(error stopping)
+endif
+
+ifndef TARGET_PRODUCT
+ $(warning dumpconfig.mk requires TARGET_PRODUCT to be set)
+ $(error stopping)
+endif
+
+ifndef TARGET_BUILD_VARIANT
+ $(warning dumpconfig.mk requires TARGET_BUILD_VARIANT to be set)
+ $(error stopping)
+endif
+
+ifneq (build/make/core/config.mk,$(wildcard build/make/core/config.mk))
+ $(warning dumpconfig must be called from the root of the source tree)
+ $(error stopping)
+endif
+
+ifeq (,$(DUMPCONFIG_FILE))
+ $(warning dumpconfig requires DUMPCONFIG_FILE to be set)
+ $(error stopping)
+endif
+
+# Before we do anything else output the format version.
+$(file > $(DUMPCONFIG_FILE),dumpconfig_version,1)
+$(file >> $(DUMPCONFIG_FILE),dumpconfig_file,$(DUMPCONFIG_FILE))
+
+# Default goal for dumpconfig
+dumpconfig:
+ $(file >> $(DUMPCONFIG_FILE),***DONE***)
+ @echo ***DONE***
+
+# TODO(Remove): These need to be set externally
+OUT_DIR := out
+TMPDIR = /tmp/build-temp
+BUILD_DATETIME_FILE := $(OUT_DIR)/build_date.txt
+
+# Escape quotation marks for CSV, and wraps in quotation marks.
+define escape-for-csv
+"$(subst ","",$1)"
+endef
+
+# Args:
+# $(1): include stack
+define dump-import-start
+$(eval $(file >> $(DUMPCONFIG_FILE),import,$(strip $(1))))
+endef
+
+# Args:
+# $(1): include stack
+define dump-import-done
+$(eval $(file >> $(DUMPCONFIG_FILE),imported,$(strip $(1))))
+endef
+
+# Args:
+# $(1): Current file
+# $(2): Inherited file
+define dump-inherit
+$(eval $(file >> $(DUMPCONFIG_FILE),inherit,$(strip $(1)),$(strip $(2))))
+endef
+
+# Args:
+# $(1): Config phase (PRODUCT or DEVICE)
+# $(2): Root nodes to import
+# $(3): All variable names
+# $(4): Single-value variables
+define dump-product-var-names
+$(eval $(file >> $(DUMPCONFIG_FILE),phase,$(strip $(1)),$(strip $(2)))) \
+$(foreach var,$(3), \
+ $(eval $(file >> $(DUMPCONFIG_FILE),var,$(if $(filter $(4),$(var)),single,list),$(var))) \
+)
+endef
+
+define dump-debug
+$(eval $(file >> $(DUMPCONFIG_FILE),debug,$(1)))
+endef
+
+# Skip these when dumping. They're not used and they cause a lot of noise in the dump.
+DUMPCONFIG_SKIP_VARS := \
+ .VARIABLES \
+ .KATI_SYMBOLS \
+ 1 \
+ 2 \
+ LOCAL_PATH \
+ MAKEFILE_LIST \
+ PARENT_PRODUCT_FILES \
+ current_mk \
+ inherit_var \
+ np \
+ _node_import_context \
+ _included \
+ _include_stack \
+ _in \
+ _nic.%
+
+# Args:
+# $(1): Makefile that was included
+# $(2): block (before,import,after)
+define dump-config-vals
+$(foreach var,$(filter-out $(DUMPCONFIG_SKIP_VARS),$(.KATI_SYMBOLS)),\
+ $(eval $(file >> $(DUMPCONFIG_FILE),val,$(call escape-for-csv,$(1)),$(2),$(call escape-for-csv,$(var)),$(call escape-for-csv,$($(var))),$(call escape-for-csv,$(KATI_variable_location $(var))))) \
+)
+endef
+
+include build/make/core/config.mk
+
diff --git a/core/node_fns.mk b/core/node_fns.mk
index b81d60c..878a4dd 100644
--- a/core/node_fns.mk
+++ b/core/node_fns.mk
@@ -195,7 +195,11 @@
$(call clear-var-list, $(3))
$(eval LOCAL_PATH := $(patsubst %/,%,$(dir $(2))))
$(eval MAKEFILE_LIST :=)
+ $(call dump-import-start,$(_include_stack))
+ $(call dump-config-vals,$(2),before)
$(eval include $(2))
+ $(call dump-import-done,$(_include_stack))
+ $(call dump-config-vals,$(2),after)
$(eval _included := $(filter-out $(2),$(MAKEFILE_LIST)))
$(eval MAKEFILE_LIST :=)
$(eval LOCAL_PATH :=)
@@ -256,6 +260,7 @@
$(if $(_include_stack),$(eval $(error ASSERTION FAILED: _include_stack \
should be empty here: $(_include_stack))),) \
$(eval _include_stack := ) \
+ $(call dump-product-var-names,$(1),$(2),$(3),$(4)) \
$(call _import-nodes-inner,$(_node_import_context),$(_in),$(3),$(4)) \
$(call move-var-list,$(_node_import_context).$(_in),$(1).$(_in),$(3)) \
$(eval _node_import_context :=) \
diff --git a/core/product.mk b/core/product.mk
index ce9bacf..5890202 100644
--- a/core/product.mk
+++ b/core/product.mk
@@ -460,7 +460,9 @@
$(eval current_mk := $(strip $(word 1,$(_include_stack)))) \
$(eval inherit_var := PRODUCTS.$(current_mk).INHERITS_FROM) \
$(eval $(inherit_var) := $(sort $($(inherit_var)) $(np))) \
- $(eval PARENT_PRODUCT_FILES := $(sort $(PARENT_PRODUCT_FILES) $(current_mk)))
+ $(eval PARENT_PRODUCT_FILES := $(sort $(PARENT_PRODUCT_FILES) $(current_mk))) \
+ $(call dump-inherit,$(strip $(word 1,$(_include_stack))),$(1)) \
+ $(call dump-config-vals,$(current_mk),inherit)
endef
# Specifies a number of path prefixes, relative to PRODUCT_OUT, where the
diff --git a/tools/product_config/src/com/android/build/config/ConfigBase.java b/tools/product_config/src/com/android/build/config/ConfigBase.java
new file mode 100644
index 0000000..5ac1fc2
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/ConfigBase.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Common parts between MakeConfig and the to-be-added GenericConfig, BazelConfig and SoongConfig.
+ */
+public class ConfigBase {
+ protected String mPhase;
+ protected List<String> mRootNodes;
+
+ /**
+ * The variables that are handled specially.
+ */
+ protected final TreeMap<String, VarType> mProductVars = new TreeMap();
+
+ /**
+ * Whether a product config variable is a list or single-value variable.
+ */
+ public enum VarType {
+ LIST,
+ SINGLE,
+ UNKNOWN // For non-product vars
+ }
+
+ public void setPhase(String phase) {
+ mPhase = phase;
+ }
+
+ public String getPhase() {
+ return mPhase;
+ }
+
+ public void setRootNodes(List<String> filenames) {
+ mRootNodes = new ArrayList(filenames);
+ }
+
+ public List<String> getRootNodes() {
+ return mRootNodes;
+ }
+
+ public void addProductVar(String name, VarType type) {
+ mProductVars.put(name, type);
+ }
+
+ public VarType getVarType(String name) {
+ final VarType t = mProductVars.get(name);
+ if (t != null) {
+ return t;
+ } else {
+ return VarType.UNKNOWN;
+ }
+ }
+
+ public boolean isProductVar(String name) {
+ return mProductVars.get(name) != null;
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/DumpConfigParser.java b/tools/product_config/src/com/android/build/config/DumpConfigParser.java
new file mode 100644
index 0000000..b954f32
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/DumpConfigParser.java
@@ -0,0 +1,304 @@
+
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.android.build.config;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * Parses the output of ckati building build/make/core/dumpconfig.mk.
+ *
+ * The format is as follows:
+ * - All processed lines are colon (':') separated fields.
+ * - Lines before the dumpconfig_version line are dropped for forward compatibility
+ * - Lines where the first field is config_var describe variables declared in makefiles
+ * (implemented by the dump-config-vals macro)
+ * Field Description
+ * 0 "config_var" row type
+ * 1 Product makefile being processed
+ * 2 The variable name
+ * 3 The value of the variable
+ * 4 The location of the variable, as best tracked by kati
+ */
+public class DumpConfigParser {
+ private static final boolean DEBUG = true;
+
+ private final Errors mErrors;
+ private final String mFilename;
+ private final Reader mReader;
+
+ private final ArrayList<MakeConfig> mResults = new ArrayList();
+
+ private static final Pattern LIST_SEPARATOR = Pattern.compile("\\s+");
+
+ public class BuildPhase {
+
+ }
+
+ /**
+ * Constructor.
+ */
+ private DumpConfigParser(Errors errors, String filename, Reader reader) {
+ mErrors = errors;
+ mFilename = filename;
+ mReader = reader;
+ }
+
+ /**
+ * Parse the text into a list of MakeConfig objects.
+ */
+ public static ArrayList<MakeConfig> parse(Errors errors, String filename, Reader reader)
+ throws CsvParser.ParseException, IOException {
+ DumpConfigParser parser = new DumpConfigParser(errors, filename, reader);
+ parser.parseImpl();
+ return parser.mResults;
+ }
+
+ /**
+ * Parse the input.
+ */
+ private void parseImpl() throws CsvParser.ParseException, IOException {
+ final List<CsvParser.Line> lines = CsvParser.parse(mReader);
+ final int lineCount = lines.size();
+ int index = 0;
+
+ int dumpconfigVersion = 0;
+
+ // Ignore lines until until we get a dumpconfig_version line for forward compatibility.
+ // In a previous life, this loop parsed from all of kati's stdout, not just the file
+ // that dumpconfig.mk writes, but it's harmless to leave this loop in. It gives us a
+ // little bit of flexibility which we probably won't need anyway, this tool probably
+ // won't diverge from dumpconfig.mk anyway.
+ for (; index < lineCount; index++) {
+ final CsvParser.Line line = lines.get(index);
+ final List<String> fields = line.getFields();
+
+ if (matchLineType(line, "dumpconfig_version", 1)) {
+ try {
+ dumpconfigVersion = Integer.parseInt(fields.get(1));
+ } catch (NumberFormatException ex) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Couldn't parse dumpconfig_version: " + fields.get(1));
+ }
+ break;
+ }
+ }
+
+ // If we never saw dumpconfig_version, there's a problem with the command, so stop.
+ if (dumpconfigVersion == 0) {
+ mErrors.ERROR_DUMPCONFIG.fatal(
+ new Position(mFilename),
+ "Never saw a valid dumpconfig_version line.");
+ }
+
+ // Any lines before the start signal will be dropped. We create garbage objects
+ // here to avoid having to check for null everywhere.
+ MakeConfig makeConfig = new MakeConfig();
+ MakeConfig.ConfigFile configFile = new MakeConfig.ConfigFile("<ignored>");
+ MakeConfig.Block block = new MakeConfig.Block(MakeConfig.BlockType.UNSET);
+
+ // Number of "phases" we've seen so far.
+ for (; index < lineCount; index++) {
+ final CsvParser.Line line = lines.get(index);
+ final List<String> fields = line.getFields();
+ final String lineType = fields.get(0);
+
+ if (matchLineType(line, "phase", 2)) {
+ makeConfig = new MakeConfig();
+ makeConfig.setPhase(fields.get(1));
+ makeConfig.setRootNodes(splitList(fields.get(2)));
+ mResults.add(makeConfig);
+
+ if (DEBUG) {
+ System.out.println("PHASE:");
+ System.out.println(" " + makeConfig.getPhase());
+ System.out.println(" " + makeConfig.getRootNodes());
+ }
+ } else if (matchLineType(line, "var", 2)) {
+ final MakeConfig.VarType type = "list".equals(fields.get(1))
+ ? MakeConfig.VarType.LIST : MakeConfig.VarType.SINGLE;
+ makeConfig.addProductVar(fields.get(2), type);
+
+ if (DEBUG) {
+ System.out.println(" VAR: " + type + " " + fields.get(2));
+ }
+ } else if (matchLineType(line, "import", 1)) {
+ final List<String> importStack = splitList(fields.get(1));
+ if (importStack.size() == 0) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "'import' line with empty include stack.");
+ continue;
+ }
+
+ // The beginning of importing a new file.
+ configFile = new MakeConfig.ConfigFile(importStack.get(0));
+ if (makeConfig.addConfigFile(configFile) != null) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Duplicate file imported in section: " + configFile.getFilename());
+ }
+ // We expect a Variable block next.
+ block = new MakeConfig.Block(MakeConfig.BlockType.BEFORE);
+ configFile.addBlock(block);
+
+ if (DEBUG) {
+ System.out.println(" IMPORT: " + configFile.getFilename());
+ }
+ } else if (matchLineType(line, "inherit", 2)) {
+ final String currentFile = fields.get(1);
+ final String inheritedFile = fields.get(2);
+ if (!configFile.getFilename().equals(currentFile)) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Unexpected current file in 'inherit' line '" + currentFile
+ + "' while processing '" + configFile.getFilename() + "'");
+ continue;
+ }
+
+ // There is already a file in progress, so add another var block to that.
+ block = new MakeConfig.Block(MakeConfig.BlockType.INHERIT);
+ block.setInheritedFile(inheritedFile);
+ configFile.addBlock(block);
+
+ if (DEBUG) {
+ System.out.println(" INHERIT: " + inheritedFile);
+ }
+ } else if (matchLineType(line, "imported", 1)) {
+ final List<String> importStack = splitList(fields.get(1));
+ if (importStack.size() == 0) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "'imported' line with empty include stack.");
+ continue;
+ }
+ final String currentFile = importStack.get(0);
+ if (!configFile.getFilename().equals(currentFile)) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Unexpected current file in 'imported' line '" + currentFile
+ + "' while processing '" + configFile.getFilename() + "'");
+ continue;
+ }
+
+ // There is already a file in progress, so add another var block to that.
+ // This will be the last one, but will check that after parsing.
+ block = new MakeConfig.Block(MakeConfig.BlockType.AFTER);
+ configFile.addBlock(block);
+
+ if (DEBUG) {
+ System.out.println(" AFTER: " + currentFile);
+ }
+ } else if (matchLineType(line, "val", 5)) {
+ final String productMakefile = fields.get(1);
+ final MakeConfig.BlockType blockType = parseBlockType(line, fields.get(2));
+ final String varName = fields.get(3);
+ final String varValue = fields.get(4);
+ final Position pos = Position.parse(fields.get(5));
+
+ if (!productMakefile.equals(configFile.getFilename())) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Mismatched 'val' product makefile."
+ + " Expected: " + configFile.getFilename()
+ + " Saw: " + productMakefile);
+ continue;
+ }
+ if (blockType == null) {
+ continue;
+ }
+ if (blockType != block.getBlockType()) {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Mismatched 'val' block type."
+ + " Expected: " + block.getBlockType()
+ + " Saw: " + blockType);
+ }
+
+ // Add the value to the block in progress
+ block.addValue(varName, new Str(pos, varValue));
+ } else {
+ if (DEBUG) {
+ System.out.print("# ");
+ for (int d = 0; d < fields.size(); d++) {
+ System.out.print(fields.get(d));
+ if (d != fields.size() - 1) {
+ System.out.print(",");
+ }
+ }
+ System.out.println();
+ }
+ }
+ }
+ }
+
+ /**
+ * Return true if the line type matches 'lineType' and there are at least 'fieldCount'
+ * fields (not including the first field which is the line type).
+ */
+ private boolean matchLineType(CsvParser.Line line, String lineType, int fieldCount) {
+ final List<String> fields = line.getFields();
+ if (!lineType.equals(fields.get(0))) {
+ return false;
+ }
+ if (fields.size() < (fieldCount + 1)) {
+ mErrors.WARNING_DUMPCONFIG.add(new Position(mFilename, line.getLine()),
+ fields.get(0) + " line has " + fields.size() + " fields. Expected at least "
+ + (fieldCount + 1) + " fields.");
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Split a string with space separated items (i.e. the make list format) into a List<String>.
+ */
+ private static List<String> splitList(String text) {
+ // Arrays.asList returns a fixed-length List, so we copy it into an ArrayList to not
+ // propagate that surprise detail downstream.
+ return new ArrayList(Arrays.asList(LIST_SEPARATOR.split(text.trim())));
+ }
+
+ /**
+ * Parse a BockType or issue a warning if it can't be parsed.
+ */
+ private MakeConfig.BlockType parseBlockType(CsvParser.Line line, String text) {
+ if ("before".equals(text)) {
+ return MakeConfig.BlockType.BEFORE;
+ } else if ("inherit".equals(text)) {
+ return MakeConfig.BlockType.INHERIT;
+ } else if ("after".equals(text)) {
+ return MakeConfig.BlockType.AFTER;
+ } else {
+ mErrors.WARNING_DUMPCONFIG.add(
+ new Position(mFilename, line.getLine()),
+ "Invalid block type: " + text);
+ return null;
+ }
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/Errors.java b/tools/product_config/src/com/android/build/config/Errors.java
index 5424338..9290b72 100644
--- a/tools/product_config/src/com/android/build/config/Errors.java
+++ b/tools/product_config/src/com/android/build/config/Errors.java
@@ -45,4 +45,11 @@
public final Category ERROR_KATI = new Category(3, false, Level.ERROR,
"Error executing or reading from Kati.");
+
+ public final Category WARNING_DUMPCONFIG = new Category(4, true, Level.WARNING,
+ "Anomaly parsing the output of kati and dumpconfig.mk.");
+
+ public final Category ERROR_DUMPCONFIG = new Category(5, false, Level.ERROR,
+ "Error parsing the output of kati and dumpconfig.mk.");
+
}
diff --git a/tools/product_config/src/com/android/build/config/Kati.java b/tools/product_config/src/com/android/build/config/Kati.java
new file mode 100644
index 0000000..026ddb5
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/Kati.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.util.List;
+
+/**
+ * Wrapper for invoking kati.
+ */
+public interface Kati {
+ public MakeConfig loadProductConfig();
+}
diff --git a/tools/product_config/src/com/android/build/config/KatiImpl.java b/tools/product_config/src/com/android/build/config/KatiImpl.java
new file mode 100644
index 0000000..feb374c
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/KatiImpl.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class KatiImpl implements Kati {
+ // Subdirectory inside out for config stuff.
+ private static final String CONFIG_SUBDIR = "config";
+
+ private final Errors mErrors;
+ private final Options mOptions;
+ private final KatiCommand mCommand;
+
+ // TODO: Do we need to consider the whole or a greater subset of the
+ // environment (or a hash of it?). In theory product-variant is enough, but we know
+ // people use stuff from the environment, even though we're trying to get rid of that.
+ private String getWorkDirPath() {
+ return Paths.get(mOptions.getOutDir(), CONFIG_SUBDIR,
+ mOptions.getProduct() + '-' + mOptions.getVariant()).toString();
+ }
+
+ private String getDumpConfigCsvPath() {
+ return Paths.get(getWorkDirPath(), "dumpconfig.csv").toString();
+ }
+
+ public KatiImpl(Errors errors, Options options) {
+ this(errors, options, new KatiCommandImpl(errors, options));
+ }
+
+ // VisibleForTesting
+ public KatiImpl(Errors errors, Options options, KatiCommand command) {
+ mErrors = errors;
+ mOptions = options;
+ mCommand = command;
+ }
+
+ @Override
+ public MakeConfig loadProductConfig() {
+ final String csvPath = getDumpConfigCsvPath();
+ try {
+ File workDir = new File(getWorkDirPath());
+
+ if (!workDir.mkdirs()) {
+ mErrors.ERROR_KATI.add("Unable to create directory: " + workDir);
+ return null; // TODO: throw exception?
+ }
+
+ System.out.println("running kati");
+ String out = mCommand.run(new String[] {
+ "-f", "build/make/core/dumpconfig.mk",
+ "DUMPCONFIG_FILE=" + csvPath
+ });
+
+ if (!out.contains("***DONE***")) {
+ mErrors.ERROR_KATI.add(
+ "Unknown error with kati, but it didn't print ***DONE*** message");
+ return null; // TODO: throw exception?
+ }
+ // TODO: Check that output was good.
+ } catch (KatiCommand.KatiException ex) {
+ mErrors.ERROR_KATI.add("Error running kati:\n" + ex.getStderr());
+ return null;
+ }
+
+ if (!(new File(csvPath)).canRead()) {
+ mErrors.ERROR_KATI.add("Kati ran but did not create " + csvPath);
+ return null;
+ }
+
+ try (FileReader reader = new FileReader(csvPath)) {
+ System.out.println("csvPath=" + csvPath);
+ List<MakeConfig> makeConfigs = DumpConfigParser.parse(mErrors, csvPath, reader);
+
+ if (makeConfigs.size() == 0) {
+ // TODO: Issue error?
+ return null;
+ }
+
+ // TODO: There are multiple passes. That should be cleaned up in the make
+ // build system, but for now, the first one is the one we want.
+ return makeConfigs.get(0);
+ } catch (CsvParser.ParseException ex) {
+ mErrors.ERROR_KATI.add(new Position(csvPath, ex.getLine()),
+ "Unable to parse output of dumpconfig.mk: " + ex.getMessage());
+ return null; // TODO: throw exception?
+ } catch (IOException ex) {
+ System.out.println(ex);
+ mErrors.ERROR_KATI.add("Unable to read " + csvPath + ": " + ex.getMessage());
+ return null; // TODO: throw exception?
+ }
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/Main.java b/tools/product_config/src/com/android/build/config/Main.java
index e577dfe..81d9e7b 100644
--- a/tools/product_config/src/com/android/build/config/Main.java
+++ b/tools/product_config/src/com/android/build/config/Main.java
@@ -16,6 +16,9 @@
package com.android.build.config;
+import java.util.List;
+import java.util.Map;
+
public class Main {
private final Errors mErrors;
private final Options mOptions;
@@ -31,6 +34,17 @@
// TODO: Check the build environment to make sure we're running in a real
// build environment, e.g. actually inside a source tree, with TARGET_PRODUCT
// and TARGET_BUILD_VARIANT defined, etc.
+ Kati kati = new KatiImpl(mErrors, mOptions);
+ MakeConfig makeConfig = kati.loadProductConfig();
+ if (makeConfig == null || mErrors.hadError()) {
+ return;
+ }
+
+ System.out.println();
+ System.out.println("====================");
+ System.out.println("PRODUCT CONFIG FILES");
+ System.out.println("====================");
+ makeConfig.printToStream(System.out);
// TODO: Run kati and extract the variables and convert all that into starlark files.
@@ -38,8 +52,6 @@
// TODO: Get the variables that were defined in starlark and use that to write
// out the make, soong and bazel input files.
- mErrors.ERROR_COMMAND_LINE.add("asdf");
- throw new RuntimeException("poop");
}
public static void main(String[] args) {
diff --git a/tools/product_config/src/com/android/build/config/MakeConfig.java b/tools/product_config/src/com/android/build/config/MakeConfig.java
new file mode 100644
index 0000000..300b655
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/MakeConfig.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class MakeConfig extends ConfigBase {
+ /**
+ * The config files that were imported in this config pass.
+ */
+ protected final ArrayList<ConfigFile> mConfigFiles = new ArrayList();
+
+ public enum BlockType {
+ UNSET,
+ BEFORE,
+ INHERIT,
+ AFTER
+ }
+
+ public static class ConfigFile {
+ /**
+ * The name of the file, relative to the tree root.
+ */
+ private final String mFilename;
+
+ /**
+ * Sections of variable definitions and import statements. Product config
+ * files will always have at least one block.
+ */
+ private final ArrayList<Block> mBlocks = new ArrayList();
+
+ public ConfigFile(String filename) {
+ mFilename = filename;
+ }
+
+ public String getFilename() {
+ return mFilename;
+ }
+
+ public void addBlock(Block block) {
+ mBlocks.add(block);
+ }
+
+ public ArrayList<Block> getBlocks() {
+ return mBlocks;
+ }
+ }
+
+ /**
+ * A set of variables that were defined.
+ */
+ public static class Block {
+ private final BlockType mBlockType;
+ private final TreeMap<String, Str> mValues = new TreeMap();
+ private String mInheritedFile;
+
+ public Block(BlockType blockType) {
+ mBlockType = blockType;
+ }
+
+ public BlockType getBlockType() {
+ return mBlockType;
+ }
+
+ public void addValue(String varName, Str varValue) {
+ mValues.put(varName, varValue);
+ }
+
+ public TreeMap<String, Str> getValues() {
+ return mValues;
+ }
+
+ public void setInheritedFile(String filename) {
+ mInheritedFile = filename;
+ }
+
+ public String getInheritedFile() {
+ return mInheritedFile;
+ }
+ }
+
+ /**
+ * Adds the given config file. Returns any one previously added, or null.
+ */
+ public ConfigFile addConfigFile(ConfigFile file) {
+ ConfigFile prev = null;
+ for (ConfigFile f: mConfigFiles) {
+ if (f.getFilename().equals(file.getFilename())) {
+ prev = f;
+ break;
+ }
+ }
+ mConfigFiles.add(file);
+ return prev;
+ }
+
+ public List<ConfigFile> getConfigFiles() {
+ return mConfigFiles;
+ }
+
+ public void printToStream(PrintStream out) {
+ out.println("MakeConfig {");
+ out.println(" phase: " + mPhase);
+ out.println(" rootNodes: " + mRootNodes);
+ out.print(" singleVars: [ ");
+ for (Map.Entry<String,VarType> entry: mProductVars.entrySet()) {
+ if (entry.getValue() == VarType.SINGLE) {
+ out.print(entry.getKey());
+ out.print(" ");
+ }
+ }
+ out.println("]");
+ out.print(" listVars: [ ");
+ for (Map.Entry<String,VarType> entry: mProductVars.entrySet()) {
+ if (entry.getValue() == VarType.LIST) {
+ out.print(entry.getKey());
+ out.print(" ");
+ }
+ }
+ out.println("]");
+ out.println(" configFiles: [");
+ for (final ConfigFile configFile: mConfigFiles) {
+ out.println(" ConfigFile {");
+ out.println(" filename: " + configFile.getFilename());
+ out.println(" blocks: [");
+ for (Block block: configFile.getBlocks()) {
+ out.println(" Block {");
+ out.println(" type: " + block.getBlockType());
+ if (block.getBlockType() == BlockType.INHERIT) {
+ out.println(" inherited: " + block.getInheritedFile());
+ }
+ out.println(" values: {");
+ for (Map.Entry<String,Str> var: block.getValues().entrySet()) {
+ if (!var.getKey().equals("PRODUCT_PACKAGES")) {
+ continue;
+ }
+ out.println(" " + var.getKey() + ": " + var.getValue());
+ }
+ out.println(" }");
+ out.println(" }");
+ }
+ out.println(" ]");
+ out.println(" }");
+ }
+ out.println(" ] // configFiles");
+ out.println("} // MakeConfig");
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/Position.java b/tools/product_config/src/com/android/build/config/Position.java
index 7953942..266021d 100644
--- a/tools/product_config/src/com/android/build/config/Position.java
+++ b/tools/product_config/src/com/android/build/config/Position.java
@@ -16,6 +16,9 @@
package com.android.build.config;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/**
* Position in a source file.
*/
@@ -25,6 +28,9 @@
*/
public static final int NO_LINE = -1;
+ private static final Pattern REGEX = Pattern.compile("([^:]*)(?::(\\d)*)?:?\\s*");
+ public static final String UNKNOWN = "<unknown>";
+
private final String mFile;
private final int mLine;
@@ -63,12 +69,39 @@
return mLine;
}
+ /**
+ * Return a Position object from a string containing <filename>:<line>, or the default
+ * Position(null, NO_LINE) if the string can't be parsed.
+ */
+ public static Position parse(String str) {
+ final Matcher m = REGEX.matcher(str);
+ if (!m.matches()) {
+ return new Position();
+ }
+ String filename = m.group(1);
+ if (filename.length() == 0 || UNKNOWN.equals(filename)) {
+ filename = null;
+ }
+ String lineString = m.group(2);
+ int line;
+ if (lineString == null || lineString.length() == 0) {
+ line = NO_LINE;
+ } else {
+ try {
+ line = Integer.parseInt(lineString);
+ } catch (NumberFormatException ex) {
+ line = NO_LINE;
+ }
+ }
+ return new Position(filename, line);
+ }
+
@Override
public String toString() {
if (mFile == null && mLine == NO_LINE) {
return "";
} else if (mFile == null && mLine != NO_LINE) {
- return "<unknown>:" + mLine + ": ";
+ return UNKNOWN + ":" + mLine + ": ";
} else if (mFile != null && mLine == NO_LINE) {
return mFile + ": ";
} else { // if (mFile != null && mLine != NO_LINE)
diff --git a/tools/product_config/src/com/android/build/config/Str.java b/tools/product_config/src/com/android/build/config/Str.java
new file mode 100644
index 0000000..7dbe2e5
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/Str.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+/**
+ * A String and a Position, where it came from in source code.
+ */
+public class Str {
+ private String mValue;
+ private Position mPosition;
+
+ public Str(String s) {
+ mValue = s;
+ mPosition = new Position();
+ }
+
+ public Str(Position pos, String s) {
+ mValue = s;
+ mPosition = pos;
+ }
+
+ @Override
+ public String toString() {
+ return mValue;
+ }
+
+ public Position getPosition() {
+ return mPosition;
+ }
+
+ /**
+ * Str is equal if the string value is equal, regardless of whether the position
+ * is the same.
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (o == null) {
+ return false;
+ } else if (o instanceof String) {
+ return mValue.equals(o);
+ } else if (o instanceof Str) {
+ final Str that = (Str)o;
+ return mValue.equals(that.mValue);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return mValue.hashCode();
+ }
+}
diff --git a/tools/product_config/test/com/android/build/config/PositionTest.java b/tools/product_config/test/com/android/build/config/PositionTest.java
new file mode 100644
index 0000000..82b5dd4
--- /dev/null
+++ b/tools/product_config/test/com/android/build/config/PositionTest.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class PositionTest {
+
+ @Test
+ public void testParseEmpty() {
+ final Position pos = Position.parse("");
+
+ Assert.assertEquals(null, pos.getFile());
+ Assert.assertEquals(Position.NO_LINE, pos.getLine());
+ }
+
+ @Test
+ public void testParseOnlyFile() {
+ final Position pos = Position.parse("asdf");
+
+ Assert.assertEquals("asdf", pos.getFile());
+ Assert.assertEquals(Position.NO_LINE, pos.getLine());
+ }
+
+ @Test
+ public void testParseBoth() {
+ final Position pos = Position.parse("asdf:1");
+
+ Assert.assertEquals("asdf", pos.getFile());
+ Assert.assertEquals(1, pos.getLine());
+ }
+
+ @Test
+ public void testParseEndsWithColon() {
+ final Position pos = Position.parse("asdf:");
+
+ Assert.assertEquals("asdf", pos.getFile());
+ Assert.assertEquals(Position.NO_LINE, pos.getLine());
+ }
+
+ @Test
+ public void testParseEndsWithSpace() {
+ final Position pos = Position.parse("asdf: ");
+
+ Assert.assertEquals("asdf", pos.getFile());
+ Assert.assertEquals(Position.NO_LINE, pos.getLine());
+ }
+
+
+}
+
diff --git a/tools/product_config/test/com/android/build/config/TestRunner.java b/tools/product_config/test/com/android/build/config/TestRunner.java
index 79d87d5..546518f 100644
--- a/tools/product_config/test/com/android/build/config/TestRunner.java
+++ b/tools/product_config/test/com/android/build/config/TestRunner.java
@@ -41,7 +41,8 @@
});
Result result = junit.run(CsvParserTest.class,
ErrorReporterTest.class,
- OptionsTest.class);
+ OptionsTest.class,
+ PositionTest.class);
if (!result.wasSuccessful()) {
System.out.println("\n*** FAILED ***");
}