Support java compilation sharding for target side.
Add "LOCAL_JAVAC_SHARD_SIZE" to represent the number of java
source path entries in each shard.
Sharding is not allowed when "LOCAL_JAVAC_SHARD_SIZE" and
"LOCAL_JAR_PROCESSORS" are both enabled.
Limitation:
1. 0 <= LOCAL_JAVAC_SHARD_SIZE <= 8192
1. 0 < NUM(java_sources) <= 8192
2. 0 <= NUM(shards) <= 100
Performance Compare:
<Unsharded Build------------------------------------------------>
1. Build from clean state
rm -r -f out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/
&& time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
real 1m2.720s user 5m26.604s sys 0m39.552s
2. Incremental build
m out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar &&
touch frameworks/base/core/java/android/net/http/HttpResponseCache.java && time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
real 0m37.586s user 5m47.804s sys 0m50.388s
<Sharded Build-------------------------------------------------->
1. Build from clean state
rm -r -f out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/
&& time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
Javac Shard Size: 50----real 1m10.163s user 25m59.008s sys 1m58.460s
Javac Shard Size: 100---real 1m2.115s user 21m3.600s sys 1m15.964s
Javac Shard Size: 150---real 0m59.520s user 18m10.544s sys 1m12.628s
Javac Shard Size: 200---real 0m56.894s user 15m39.244s sys 1m11.608s
Javac Shard Size: 250---real 0m55.991s user 14m38.716s sys 1m2.292s
Javac Shard Size: 300---real 0m55.114s user 13m6.568s sys 1m8.200s
Javac Shard Size: 350---real 0m53.144s user 12m7.740s sys 1m3.836s
Javac Shard Size: 400---real 0m54.929s user 12m9.324s sys 1m4.340s
Javac Shard Size: 450---real 1m30.194s user 25m31.468s sys 1m52.416s
Javac Shard Size: 500---real 0m53.976s user 10m35.500s sys 0m55.160s
2. Incremental build
m out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar &&
touch frameworks/base/core/java/android/net/http/HttpResponseCache.java && time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
Javac Shard Size: 50-----real 0m16.322s user 1m8.648s sys 0m31.700s
Javac Shard Size: 100----real 0m16.163s user 1m22.932s sys 0m29.440s
Javac Shard Size: 150----real 0m16.611s user 1m37.828s sys 0m21.168s
Javac Shard Size: 200----real 0m16.936s user 1m49.248s sys 0m28.636s
Javac Shard Size: 250----real 0m17.509s user 1m54.944s sys 0m32.768s
Javac Shard Size: 300----real 0m18.868s user 1m54.088s sys 0m28.824s
Javac Shard Size: 350----real 0m17.629s user 1m54.108s sys 0m31.056s
Javac Shard Size: 400----real 0m18.658s user 2m7.712s sys 0m30.636s
Javac Shard Size: 450----real 0m18.874s user 2m8.808s sys 0m33.540s
Javac Shard Size: 500----real 0m19.432s user 2m24.400s sys 0m30.368s
time m nothing:
real 0m5.799s user 0m7.236s sys 0m3.068s
Test: m clean && m -j checkbuild
Bug: b/67424047
Change-Id: Id0766d2b7de7c4546d29bbc7f8a0dd0e4b9ad45b
diff --git a/core/java.mk b/core/java.mk
index 5eddb0e..cf49994 100644
--- a/core/java.mk
+++ b/core/java.mk
@@ -352,7 +352,36 @@
##########################################
java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \
$(filter %.java,$(LOCAL_GENERATED_SOURCES))
-all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
+java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
+all_java_sources := $(java_sources) $(java_intermediate_sources)
+
+enable_sharding :=
+ifneq ($(TURBINE_ENABLED),false)
+ifneq ($(LOCAL_JAVAC_SHARD_SIZE),)
+ifneq ($(LOCAL_JAR_PROCESSOR),)
+$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!)
+endif # LOCAL_JAR_PROCESSOR is not empty
+enable_sharding := true
+
+num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE))
+ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards)))
+# increment number of shards by 1.
+num_shards := $(call int_plus,$(num_shards),1)
+endif
+
+shard_idx_list := $(call int_range_list,1,$(num_shards))
+sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x))
+sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x))
+
+# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
+# increment number of shards by 1.
+num_shards := $(call int_plus,$(num_shards),1)
+sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards)
+sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards)
+LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files)
+LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list)
+endif # LOCAL_JAVAC_SHARD_SIZE is not empty
+endif # TURBINE_ENABLED != false
include $(BUILD_SYSTEM)/java_common.mk
@@ -419,23 +448,47 @@
$(java_source_list_file): $(java_sources_deps)
$(write-java-source-list)
-$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags)
-$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES)
-$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES)
-$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES)
-$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF)
-$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file)
-$(full_classes_compiled_jar): \
- $(java_source_list_file) \
- $(java_sources_deps) \
- $(full_java_header_libs) \
- $(full_java_bootclasspath_libs) \
- $(layers_file) \
- $(annotation_processor_deps) \
- $(NORMALIZE_PATH) \
- $(JAR_ARGS) \
- | $(SOONG_JAVAC_WRAPPER)
- $(transform-java-to-classes.jar)
+ifdef enable_sharding
+$(foreach x,$(shard_idx_list),\
+ $(eval $(call save-sharded-java-source-list,$(x),\
+ $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
+ $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
+
+# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
+$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources)
+$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \
+ $(RenderScript_file_stamp) \
+ $(proto_java_sources_file_stamp) \
+ $(LOCAL_ADDITIONAL_DEPENDENCIES) \
+ $(NORMALIZE_PATH)
+ $(hide) rm -f $@
+ $(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp)
+ $(call fetch-additional-java-source,$@.tmp)
+ $(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
+
+# Javac sharding with header libs including its own header jar as one of dependency.
+$(foreach x,$(shard_idx_list),\
+ $(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\
+ $(java_source_list_file).shard.$(x),\
+ $(full_java_header_libs) $(full_classes_header_jar),$(x),\
+ $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
+ $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
+
+# Javac sharding for last shard with additional Java dependencies.
+$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\
+ $(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \
+ $(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \
+ $$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES)))
+
+$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list)
+$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS)
+ $(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST)
+else
+# we can't use single $ for java_sources_deps since it may contain hash '#' sign.
+$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\
+ $(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps)))
+
+endif # ifdef enable_sharding
ifneq ($(TURBINE_ENABLED),false)