Merge "Deprecate VNDK-Lite"
diff --git a/core/Makefile b/core/Makefile
index 584a144..8395e45 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -940,7 +940,7 @@
$(if $(filter $(1),xml_vendor_dlkm),-i vendor_dlkm --xml-output, \
$(if $(filter $(1),xml_odm_dlkm),-i odm_dlkm --xml-output, \
--html-output)))))))) $(3) \
- -t $$(PRIVATE_MESSAGE) -s $$(PRIVATE_DIR)/src
+ -t $$(PRIVATE_MESSAGE) $$(foreach dir,$$(sort $$(PRIVATE_DIR)), -s $$(dir)/src)
notice_files: $(2) $(3)
endef
diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk
new file mode 100644
index 0000000..ee12c8c
--- /dev/null
+++ b/core/android_soong_config_vars.mk
@@ -0,0 +1,29 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file defines the Soong Config Variable namespace ANDROID, and also any
+# variables in that namespace.
+
+# The expectation is that no vendor should be using the ANDROID namespace. This
+# check ensures that we don't collide with any existing vendor usage.
+
+ifdef SOONG_CONFIG_ANDROID
+$(error The Soong config namespace ANDROID is reserved.)
+endif
+
+$(call add_soong_config_namespace,ANDROID)
+
+# Add variables to the namespace below:
+
+$(call add_soong_config_var,ANDROID,TARGET_ENABLE_MEDIADRM_64)
diff --git a/core/base_rules.mk b/core/base_rules.mk
index d604480..3f93c2c 100644
--- a/core/base_rules.mk
+++ b/core/base_rules.mk
@@ -592,12 +592,22 @@
ifneq ($(strip $(LOCAL_TEST_DATA)),)
ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE))
+# Soong LOCAL_TEST_DATA is of the form <from_base>:<file>:<relative_install_path>
+# or <from_base>:<file>, to be installed to
+# <install_root>/<relative_install_path>/<file> or <install_root>/<file>,
+# respectively.
ifeq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK))
define copy_test_data_pairs
_src_base := $$(call word-colon,1,$$(td))
_file := $$(call word-colon,2,$$(td))
- my_test_data_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(call append-path,$$(my_module_path),$$(_file))
- my_test_data_file_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(_file)
+ _relative_install_path := $$(call word-colon,3,$$(td))
+ ifeq (,$$(_relative_install_path))
+ _relative_dest_file := $$(_file)
+ else
+ _relative_dest_file := $$(call append-path,$$(_relative_install_path),$$(_file))
+ endif
+ my_test_data_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(call append-path,$$(my_module_path),$$(_relative_dest_file))
+ my_test_data_file_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(_relative_dest_file)
endef
else
define copy_test_data_pairs
diff --git a/core/config.mk b/core/config.mk
index d701d4d..a35b718 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -1228,6 +1228,7 @@
product-graph dump-products
ifeq ($(CALLED_FROM_SETUP),true)
+include $(BUILD_SYSTEM)/android_soong_config_vars.mk
include $(BUILD_SYSTEM)/ninja_config.mk
include $(BUILD_SYSTEM)/soong_config.mk
endif
diff --git a/core/dex_preopt.mk b/core/dex_preopt.mk
index 20b4051..dd31999 100644
--- a/core/dex_preopt.mk
+++ b/core/dex_preopt.mk
@@ -21,32 +21,38 @@
my_boot_image_arch := TARGET_ARCH
my_boot_image_out := $(PRODUCT_OUT)
my_boot_image_syms := $(TARGET_OUT_UNSTRIPPED)
-my_boot_image_root := DEFAULT_DEX_PREOPT_INSTALLED_IMAGE
-DEFAULT_DEX_PREOPT_INSTALLED_IMAGE :=
-$(foreach my_boot_image_name,$(DEXPREOPT_IMAGE_NAMES),$(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk))
+DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE := \
+ $(foreach my_boot_image_name,$(DEXPREOPT_IMAGE_NAMES),$(strip \
+ $(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk) \
+ $(my_boot_image_module)))
ifdef TARGET_2ND_ARCH
my_boot_image_arch := TARGET_2ND_ARCH
- my_boot_image_root := 2ND_DEFAULT_DEX_PREOPT_INSTALLED_IMAGE
- 2ND_DEFAULT_DEX_PREOPT_INSTALLED_IMAGE :=
- $(foreach my_boot_image_name,$(DEXPREOPT_IMAGE_NAMES),$(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk))
+ 2ND_DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE := \
+ $(foreach my_boot_image_name,$(DEXPREOPT_IMAGE_NAMES),$(strip \
+ $(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk) \
+ $(my_boot_image_module)))
endif
# Install boot images for testing on host. We exclude framework image as it is not part of art manifest.
my_boot_image_arch := HOST_ARCH
my_boot_image_out := $(HOST_OUT)
my_boot_image_syms := $(HOST_OUT)/symbols
-my_boot_image_root := HOST_BOOT_IMAGE
-HOST_BOOT_IMAGE :=
-$(foreach my_boot_image_name,art_host,$(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk))
+HOST_BOOT_IMAGE_MODULE := \
+ $(foreach my_boot_image_name,art_host,$(strip \
+ $(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk) \
+ $(my_boot_image_module)))
+HOST_BOOT_IMAGE := $(call module-installed-files,$(HOST_BOOT_IMAGE_MODULE))
ifdef HOST_2ND_ARCH
my_boot_image_arch := HOST_2ND_ARCH
- my_boot_image_root := 2ND_HOST_BOOT_IMAGE
- 2ND_HOST_BOOT_IMAGE :=
- $(foreach my_boot_image_name,art_host,$(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk))
+ 2ND_HOST_BOOT_IMAGE_MODULE := \
+ $(foreach my_boot_image_name,art_host,$(strip \
+ $(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk) \
+ $(my_boot_image_module)))
+ 2ND_HOST_BOOT_IMAGE := $(call module-installed-files,$(2ND_HOST_BOOT_IMAGE_MODULE))
endif
my_boot_image_arch :=
my_boot_image_out :=
my_boot_image_syms :=
-my_boot_image_root :=
+my_boot_image_module :=
# Build the boot.zip which contains the boot jars and their compilation output
# We can do this only if preopt is enabled and if the product uses libart config (which sets the
diff --git a/core/dex_preopt_libart.mk b/core/dex_preopt_libart.mk
index 12b29f4..8f0702b 100644
--- a/core/dex_preopt_libart.mk
+++ b/core/dex_preopt_libart.mk
@@ -5,38 +5,75 @@
# my_boot_image_arch: the architecture to install (e.g. TARGET_ARCH, not expanded)
# my_boot_image_out: the install directory (e.g. $(PRODUCT_OUT))
# my_boot_image_syms: the symbols director (e.g. $(TARGET_OUT_UNSTRIPPED))
-# my_boot_image_root: make variable used to store installed image path
+#
+# Output variables:
+# my_boot_image_module: the created module name. Empty if no module is created.
+#
+# Install the boot images compiled by Soong.
+# Create a module named dexpreopt_bootjar.$(my_boot_image_name)_$($(my_boot_image_arch))
+# that installs all of boot image files.
+# If there is no file to install for $(my_boot_image_name), for example when
+# building an unbundled build, then no module is created.
#
####################################
# Install $(1) to $(2) so that it is shared between architectures.
+# Returns the target path of the shared vdex file and installed symlink.
define copy-vdex-file
-my_vdex_shared := $$(dir $$(patsubst %/,%,$$(dir $(2))))$$(notdir $(2)) # Remove the arch dir.
-ifneq ($(my_boot_image_arch),$(filter $(my_boot_image_arch), TARGET_2ND_ARCH HOST_2ND_ARCH))
-$$(my_vdex_shared): $(1) # Copy $(1) to directory one level up (i.e. with the arch dir removed).
- @echo "Install: $$@"
- $$(copy-file-to-target)
-endif
-$(2): $$(my_vdex_shared) # Create symlink at $(2) which points to the actual physical copy.
- @echo "Symlink: $$@"
- mkdir -p $$(dir $$@)
- ln -sfn ../$$(notdir $$@) $$@
-my_vdex_shared :=
+$(strip \
+ $(eval # Remove the arch dir) \
+ $(eval my_vdex_shared := $(dir $(patsubst %/,%,$(dir $(2))))$(notdir $(2))) \
+ $(if $(filter-out %_2ND_ARCH,$(my_boot_image_arch)), \
+ $(eval # Copy $(1) to directory one level up (i.e. with the arch dir removed).) \
+ $(eval $(call copy-one-file,$(1),$(my_vdex_shared))) \
+ ) \
+ $(eval # Create symlink at $(2) which points to the actual physical copy.) \
+ $(call symlink-file,$(my_vdex_shared),../$(notdir $(2)),$(2)) \
+ $(my_vdex_shared) $(2) \
+)
endef
# Same as 'copy-many-files' but it uses the vdex-specific helper above.
define copy-vdex-files
-$(foreach v,$(1),$(eval $(call copy-vdex-file, $(call word-colon,1,$(v)), $(2)$(call word-colon,2,$(v)))))
-$(foreach v,$(1),$(2)$(call word-colon,2,$(v)))
+$(foreach v,$(1),$(call copy-vdex-file,$(call word-colon,1,$(v)),$(2)$(call word-colon,2,$(v))))
endef
-# Install the boot images compiled by Soong.
-# The first file is saved in $(my_boot_image_root) and the rest are added as it's dependencies.
-my_suffix := BUILT_INSTALLED_$(my_boot_image_name)_$($(my_boot_image_arch))
-my_installed := $(call copy-many-files,$(DEXPREOPT_IMAGE_$(my_suffix)),$(my_boot_image_out))
-my_installed += $(call copy-many-files,$(DEXPREOPT_IMAGE_UNSTRIPPED_$(my_suffix)),$(my_boot_image_syms))
-my_installed += $(call copy-vdex-files,$(DEXPREOPT_IMAGE_VDEX_$(my_suffix)),$(my_boot_image_out))
-$(my_boot_image_root) += $(firstword $(my_installed))
-$(firstword $(my_installed)): $(wordlist 2,9999,$(my_installed))
-my_installed :=
-my_suffix :=
+my_boot_image_module :=
+
+my_suffix := $(my_boot_image_name)_$($(my_boot_image_arch))
+my_copy_pairs := $(strip $(DEXPREOPT_IMAGE_BUILT_INSTALLED_$(my_suffix)))
+
+# Generate the boot image module only if there is any file to install.
+ifneq (,$(my_copy_pairs))
+ my_first_pair := $(firstword $(my_copy_pairs))
+ my_rest_pairs := $(wordlist 2,$(words $(my_copy_pairs)),$(my_copy_pairs))
+
+ my_first_src := $(call word-colon,1,$(my_first_pair))
+ my_first_dest := $(my_boot_image_out)$(call word-colon,2,$(my_first_pair))
+
+ my_installed := $(call copy-many-files,$(my_rest_pairs),$(my_boot_image_out))
+ my_installed += $(call copy-vdex-files,$(DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_$(my_suffix)),$(my_boot_image_out))
+ my_unstripped_installed := $(call copy-many-files,$(DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_$(my_suffix)),$(my_boot_image_syms))
+
+ # We don't have a LOCAL_PATH for the auto-generated modules, so let it be the $(BUILD_SYSTEM).
+ LOCAL_PATH := $(BUILD_SYSTEM)
+
+ include $(CLEAR_VARS)
+ LOCAL_MODULE := dexpreopt_bootjar.$(my_suffix)
+ LOCAL_PREBUILT_MODULE_FILE := $(my_first_src)
+ LOCAL_MODULE_PATH := $(dir $(my_first_dest))
+ LOCAL_MODULE_STEM := $(notdir $(my_first_dest))
+ ifneq (,$(strip $(filter HOST_%,$(my_boot_image_arch))))
+ LOCAL_IS_HOST_MODULE := true
+ endif
+ LOCAL_MODULE_CLASS := ETC
+ include $(BUILD_PREBUILT)
+ $(LOCAL_BUILT_MODULE): $(my_unstripped_installed)
+ # Installing boot.art causes all boot image bits to be installed.
+ # Keep this old behavior in case anyone still needs it.
+ $(LOCAL_INSTALLED_MODULE): $(my_installed)
+ ALL_MODULES.$(my_register_name).INSTALLED += $(my_installed)
+ $(my_all_targets): $(my_installed)
+
+ my_boot_image_module := $(LOCAL_MODULE)
+endif # my_copy_pairs != empty
diff --git a/core/main.mk b/core/main.mk
index 0bd2c53..90c4cc4 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -150,9 +150,6 @@
# Bring in standard build system definitions.
include $(BUILD_SYSTEM)/definitions.mk
-# Bring in dex_preopt.mk
-include $(BUILD_SYSTEM)/dex_preopt.mk
-
ifneq ($(filter user userdebug eng,$(MAKECMDGOALS)),)
$(info ***************************************************************)
$(info ***************************************************************)
@@ -467,6 +464,12 @@
# Typical build; include any Android.mk files we can find.
#
+# Bring in dex_preopt.mk
+# This creates some modules so it needs to be included after
+# should-install-to-system is defined (in order for base_rules.mk to function
+# properly), but before readonly-final-product-vars is called.
+include $(BUILD_SYSTEM)/dex_preopt.mk
+
# Strip and readonly a few more variables so they won't be modified.
$(readonly-final-product-vars)
ADDITIONAL_SYSTEM_PROPERTIES := $(strip $(ADDITIONAL_SYSTEM_PROPERTIES))
diff --git a/core/robolectric_test_config_template.xml b/core/robolectric_test_config_template.xml
new file mode 100644
index 0000000..78863b7
--- /dev/null
+++ b/core/robolectric_test_config_template.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2020 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This test config file is auto-generated. -->
+<configuration description="Runs {MODULE}">
+ <option name="test-suite-tag" value="apct" />
+ <option name="test-suite-tag" value="apct-junit" />
+
+ {EXTRA_CONFIGS}
+
+ <test class="com.android.tradefed.testtype.HostTest" >
+ <option name="jar" value="{MODULE}.jar" />
+ </test>
+</configuration>
diff --git a/core/soong_java_prebuilt.mk b/core/soong_java_prebuilt.mk
index e4c84e0..05f700d 100644
--- a/core/soong_java_prebuilt.mk
+++ b/core/soong_java_prebuilt.mk
@@ -94,16 +94,18 @@
boot_jars := $(foreach pair,$(PRODUCT_BOOT_JARS), $(call word-colon,2,$(pair)))
ifneq ($(filter $(LOCAL_MODULE),$(boot_jars)),) # is_boot_jar
ifeq (true,$(WITH_DEXPREOPT))
- # For libart, the boot jars' odex files are replaced by $(DEFAULT_DEX_PREOPT_INSTALLED_IMAGE).
- # We use this installed_odex trick to get boot.art installed.
- installed_odex := $(DEFAULT_DEX_PREOPT_INSTALLED_IMAGE)
- # Append the odex for the 2nd arch if we have one.
- installed_odex += $($(TARGET_2ND_ARCH_VAR_PREFIX)DEFAULT_DEX_PREOPT_INSTALLED_IMAGE)
- ALL_MODULES.$(my_register_name).INSTALLED += $(installed_odex)
- # Make sure to install the .odex and .vdex when you run "make <module_name>"
- $(my_all_targets): $(installed_odex)
- # Copy $(LOCAL_BUILT_MODULE) and its dependencies when installing boot.art
- $(DEFAULT_DEX_PREOPT_INSTALLED_IMAGE): $(LOCAL_BUILT_MODULE)
+ # $(DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE) contains modules that installs
+ # all of bootjars' dexpreopt files (.art, .oat, .vdex, ...)
+ # Add them to the required list so they are installed alongside this module.
+ ALL_MODULES.$(my_register_name).REQUIRED_FROM_TARGET += \
+ $(DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE) \
+ $(2ND_DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE)
+ # Copy $(LOCAL_BUILT_MODULE) and its dependencies when installing boot.art
+ # so that dependencies of $(LOCAL_BUILT_MODULE) (which may include
+ # jacoco-report-classes.jar) are copied for every build.
+ $(foreach m,$(DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE) $(2ND_DEFAULT_DEX_PREOPT_INSTALLED_IMAGE_MODULE), \
+ $(eval $(call add-dependency,$(firstword $(call module-installed-files,$(m))),$(LOCAL_BUILT_MODULE))) \
+ )
endif
endif # is_boot_jar
diff --git a/target/board/generic_arm64/BoardConfig.mk b/target/board/generic_arm64/BoardConfig.mk
index 2963ee4..22108fa 100644
--- a/target/board/generic_arm64/BoardConfig.mk
+++ b/target/board/generic_arm64/BoardConfig.mk
@@ -61,12 +61,17 @@
BOARD_KERNEL-5.4_BOOTIMAGE_PARTITION_SIZE := 67108864
BOARD_KERNEL-5.4-GZ_BOOTIMAGE_PARTITION_SIZE := 47185920
BOARD_KERNEL-5.4-LZ4_BOOTIMAGE_PARTITION_SIZE := 53477376
+BOARD_KERNEL-MAINLINE_BOOTIMAGE_PARTITION_SIZE := 67108864
+BOARD_KERNEL-MAINLINE-GZ_BOOTIMAGE_PARTITION_SIZE := 47185920
+BOARD_KERNEL-MAINLINE-LZ4_BOOTIMAGE_PARTITION_SIZE := 53477376
+
BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800
BOARD_BOOT_HEADER_VERSION := 3
BOARD_MKBOOTIMG_ARGS += --header_version $(BOARD_BOOT_HEADER_VERSION)
-BOARD_KERNEL_BINARIES := kernel-5.4 kernel-5.4-gz kernel-5.4-lz4
+BOARD_KERNEL_BINARIES := kernel-5.4 kernel-5.4-gz kernel-5.4-lz4 \
+ kernel-mainline kernel-mainline-gz kernel-mainline-lz4
# Some vendors still haven't cleaned up all device specific directories under
# root!
diff --git a/target/board/generic_arm64/device.mk b/target/board/generic_arm64/device.mk
index b34004f..d8d06cd 100644
--- a/target/board/generic_arm64/device.mk
+++ b/target/board/generic_arm64/device.mk
@@ -17,4 +17,7 @@
PRODUCT_COPY_FILES += \
device/google/cuttlefish_kernel/5.4-arm64/kernel-5.4:kernel-5.4 \
device/google/cuttlefish_kernel/5.4-arm64/kernel-5.4-gz:kernel-5.4-gz \
- device/google/cuttlefish_kernel/5.4-arm64/kernel-5.4-lz4:kernel-5.4-lz4
+ device/google/cuttlefish_kernel/5.4-arm64/kernel-5.4-lz4:kernel-5.4-lz4 \
+ kernel/prebuilts/mainline/arm64/kernel-mainline:kernel-mainline \
+ kernel/prebuilts/mainline/arm64/kernel-mainline-gz:kernel-mainline-gz \
+ kernel/prebuilts/mainline/arm64/kernel-mainline-lz4:kernel-mainline-lz4
diff --git a/target/board/gsi_system_ext.prop b/target/board/gsi_system_ext.prop
index dd3227e..780aadc 100644
--- a/target/board/gsi_system_ext.prop
+++ b/target/board/gsi_system_ext.prop
@@ -12,8 +12,3 @@
# TODO(b/78105955): disable privapp_permissions checking before the bug solved
ro.control_privapp_permissions=disable
-
-# TODO(b/136212765): the default for LMK
-ro.lmk.kill_heaviest_task=true
-ro.lmk.kill_timeout_ms=100
-ro.lmk.use_minfree_levels=true
diff --git a/target/board/gsi_system_ext_user.prop b/target/board/gsi_system_ext_user.prop
index db6d880..217bd01 100644
--- a/target/board/gsi_system_ext_user.prop
+++ b/target/board/gsi_system_ext_user.prop
@@ -9,8 +9,3 @@
# TODO(b/78105955): disable privapp_permissions checking before the bug solved
ro.control_privapp_permissions=disable
-
-# TODO(b/136212765): the default for LMK
-ro.lmk.kill_heaviest_task=true
-ro.lmk.kill_timeout_ms=100
-ro.lmk.use_minfree_levels=true
diff --git a/target/product/gsi/current.txt b/target/product/gsi/current.txt
index d66136e..345faa4 100644
--- a/target/product/gsi/current.txt
+++ b/target/product/gsi/current.txt
@@ -18,6 +18,8 @@
LLNDK: libsync.so
LLNDK: libvndksupport.so
LLNDK: libvulkan.so
+VNDK-SP: android.hardware.common-V1-ndk_platform.so
+VNDK-SP: android.hardware.graphics.common-V1-ndk_platform.so
VNDK-SP: android.hardware.graphics.common@1.0.so
VNDK-SP: android.hardware.graphics.common@1.1.so
VNDK-SP: android.hardware.graphics.common@1.2.so
diff --git a/tools/generate-notice-files.py b/tools/generate-notice-files.py
index 49011b2..18f2166 100755
--- a/tools/generate-notice-files.py
+++ b/tools/generate-notice-files.py
@@ -73,10 +73,10 @@
</style>
"""
-def combine_notice_files_html(file_hash, input_dir, output_filename):
+def combine_notice_files_html(file_hash, input_dirs, output_filename):
"""Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME."""
- SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
+ SRC_DIR_STRIP_RE = re.compile("(?:" + "|".join(input_dirs) + ")(/.*).txt")
# Set up a filename to row id table (anchors inside tables don't work in
# most browsers, but href's to table row ids do)
@@ -131,10 +131,10 @@
print >> output_file, "</body></html>"
output_file.close()
-def combine_notice_files_text(file_hash, input_dir, output_filename, file_title):
+def combine_notice_files_text(file_hash, input_dirs, output_filename, file_title):
"""Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME."""
- SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
+ SRC_DIR_STRIP_RE = re.compile("(?:" + "|".join(input_dirs) + ")(/.*).txt")
output_file = open(output_filename, "wb")
print >> output_file, file_title
for value in file_hash:
@@ -146,10 +146,10 @@
print >> output_file, open(value[0]).read()
output_file.close()
-def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename):
+def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename):
"""Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME."""
- SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
+ SRC_DIR_STRIP_RE = re.compile("(?:" + "|".join(input_dirs) + ")(/.*).txt")
# Set up a filename to row id table (anchors inside tables don't work in
# most browsers, but href's to table row ids do)
@@ -205,7 +205,7 @@
'-t', '--title', required=True,
help='The file title.')
parser.add_argument(
- '-s', '--source-dir', required=True,
+ '-s', '--source-dir', required=True, action='append',
help='The directory containing notices.')
parser.add_argument(
'-i', '--included-subdirs', action='append',
@@ -229,39 +229,40 @@
if args.excluded_subdirs is not None:
excluded_subdirs = args.excluded_subdirs
+ input_dirs = [os.path.normpath(source_dir) for source_dir in args.source_dir]
# Find all the notice files and md5 them
- input_dir = os.path.normpath(args.source_dir)
- files_with_same_hash = defaultdict(list)
- for root, dir, files in os.walk(input_dir):
- for file in files:
- matched = True
- if len(included_subdirs) > 0:
- matched = False
- for subdir in included_subdirs:
- if (root == (input_dir + '/' + subdir) or
- root.startswith(input_dir + '/' + subdir + '/')):
- matched = True
- break
- elif len(excluded_subdirs) > 0:
- for subdir in excluded_subdirs:
- if (root == (input_dir + '/' + subdir) or
- root.startswith(input_dir + '/' + subdir + '/')):
- matched = False
- break
- if matched and file.endswith(".txt"):
- filename = os.path.join(root, file)
- file_md5sum = md5sum(filename)
- files_with_same_hash[file_md5sum].append(filename)
+ for input_dir in input_dirs:
+ files_with_same_hash = defaultdict(list)
+ for root, dir, files in os.walk(input_dir):
+ for file in files:
+ matched = True
+ if len(included_subdirs) > 0:
+ matched = False
+ for subdir in included_subdirs:
+ if (root == (input_dir + '/' + subdir) or
+ root.startswith(input_dir + '/' + subdir + '/')):
+ matched = True
+ break
+ elif len(excluded_subdirs) > 0:
+ for subdir in excluded_subdirs:
+ if (root == (input_dir + '/' + subdir) or
+ root.startswith(input_dir + '/' + subdir + '/')):
+ matched = False
+ break
+ if matched and file.endswith(".txt"):
+ filename = os.path.join(root, file)
+ file_md5sum = md5sum(filename)
+ files_with_same_hash[file_md5sum].append(filename)
- filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(files_with_same_hash.keys())]
+ filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(files_with_same_hash.keys())]
- combine_notice_files_text(filesets, input_dir, txt_output_file, file_title)
+ combine_notice_files_text(filesets, input_dirs, txt_output_file, file_title)
if html_output_file is not None:
- combine_notice_files_html(filesets, input_dir, html_output_file)
+ combine_notice_files_html(filesets, input_dirs, html_output_file)
if xml_output_file is not None:
- combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file)
+ combine_notice_files_xml(files_with_same_hash, input_dirs, xml_output_file)
if __name__ == "__main__":
main(sys.argv)