Add the library absence check as a kati stage check.

Primarily for use with build_test during presubmit.

Moved the check to main.mk to avoid nonobvious dependency on
product_target_FILES from main.mk to Makefile.

Test: m systemimage
Test: build/soong/build_test.bash
Test: m out/target/product/taimen/system/lib/libdexfile.so; m systemimage;
  check that "find"-based check fails
Test: Add a system library to APEX_MODULE_LIBS; m systemimage;
  check that "product_target_FILES"-based check fails;
  env DISABLE_APEX_LIBS_ABSENCE_CHECK=true m systemimage;
  check that it works
Bug: 124293228
Change-Id: I7a93cc0ac4480404a3d56c75eb43ae499e8a3a05
diff --git a/core/main.mk b/core/main.mk
index 7e1bdd5..6f92ff1 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -1200,6 +1200,117 @@
 )
 endef
 
+# Check that libraries that should only be in APEXes don't end up in the system
+# image. For the Runtime APEX this complements the checks in
+# art/build/apex/art_apex_test.py.
+# TODO(b/128708192): Implement this restriction in Soong instead.
+
+# Runtime APEX libraries
+APEX_MODULE_LIBS := \
+  libadbconnection.so \
+  libandroidicu.so \
+  libandroidio.so \
+  libart-compiler.so \
+  libart-dexlayout.so \
+  libart.so \
+  libartbase.so \
+  libartpalette.so \
+  libdexfile.so \
+  libdexfile_external.so \
+  libdexfiled_external.so \
+  libdt_fd_forward.so \
+  libdt_socket.so \
+  libicui18n.so \
+  libicuuc.so \
+  libjavacore.so \
+  libjdwp.so \
+  libnativebridge.so \
+  libnativehelper.so \
+  libnativeloader.so \
+  libnpt.so \
+  libopenjdk.so \
+  libopenjdkjvm.so \
+  libopenjdkjvmti.so \
+  libpac.so \
+  libprofile.so \
+  libsigchain.so \
+
+# Conscrypt APEX libraries
+APEX_MODULE_LIBS += \
+  libjavacrypto.so \
+
+# An option to disable the check below, for local use since some build targets
+# still may create these libraries in /system (b/129006418).
+DISABLE_APEX_LIBS_ABSENCE_CHECK ?=
+
+# Exclude lib/arm and lib/arm64 which contain the native bridge proxy libs. They
+# are compiled for the guest architecture and used with an entirely different
+# linker config. The native libs are then linked to as usual via exported
+# interfaces, so the proxy libs do not violate the interface boundaries on the
+# native architecture.
+# TODO(b/130630776): Introduce a make variable for the appropriate directory
+# when native bridge is active.
+APEX_LIBS_ABSENCE_CHECK_EXCLUDE := lib/arm lib/arm64
+
+# Exclude vndk-* subdirectories which contain prebuilts from older releases.
+APEX_LIBS_ABSENCE_CHECK_EXCLUDE += lib/vndk-% lib64/vndk-%
+
+ifdef DISABLE_APEX_LIBS_ABSENCE_CHECK
+  check-apex-libs-absence :=
+  check-apex-libs-absence-on-disk :=
+else
+  # If the check below fails, some library has ended up in system/lib or
+  # system/lib64 that is intended to only go into some APEX package. The likely
+  # cause is that a library or binary in /system has grown a dependency that
+  # directly or indirectly pulls in the prohibited library.
+  #
+  # To resolve this, look for the APEX package that the library belong to -
+  # search for it in 'native_shared_lib' properties in 'apex' build modules (see
+  # art/build/apex/Android.bp for an example). Then check if there is an
+  # exported library in that APEX package that should be used instead, i.e. one
+  # listed in its 'native_shared_lib' property for which the corresponding
+  # 'cc_library' module has a 'stubs' clause (like libdexfile_external in
+  # art/libdexfile/Android.bp).
+  #
+  # If you cannot find an APEX exported library that fits your needs, or you
+  # think that the library you want to depend on should be allowed in /system,
+  # then please contact the owners of the APEX package containing the library.
+  #
+  # If you get this error for a library that is exported in an APEX, then the
+  # APEX might be misconfigured or something is wrong in the build system.
+  # Please reach out to the APEX package owners and/or soong-team@, or
+  # android-building@googlegroups.com externally.
+  define check-apex-libs-absence
+    $(call maybe-print-list-and-error, \
+      $(filter $(foreach lib,$(APEX_MODULE_LIBS),%/$(lib)), \
+        $(filter-out $(foreach dir,$(APEX_LIBS_ABSENCE_CHECK_EXCLUDE), \
+                       $(TARGET_OUT)/$(if $(findstring %,$(dir)),$(dir),$(dir)/%)), \
+          $(filter $(TARGET_OUT)/lib/% $(TARGET_OUT)/lib64/%,$(1)))), \
+      APEX libraries found in system image (see comment for check-apex-libs-absence in \
+      build/make/core/main.mk for details))
+  endef
+
+  # TODO(b/129006418): The check above catches libraries through product
+  # dependencies visible to make, but as long as they have install rules in
+  # /system they may still be created there through other make targets. To catch
+  # that we also do a check on disk just before the system image is built.
+  define check-apex-libs-absence-on-disk
+    $(hide) ( \
+      cd $(TARGET_OUT) && \
+      findres=$$(find lib* \
+        $(foreach dir,$(APEX_LIBS_ABSENCE_CHECK_EXCLUDE),-path "$(subst %,*,$(dir))" -prune -o) \
+        -type f \( -false $(foreach lib,$(APEX_MODULE_LIBS),-o -name $(lib)) \) \
+        -print) && \
+      if [ -n "$$findres" ]; then \
+        echo "APEX libraries found in system image (see comment for check-apex-libs-absence" 1>&2; \
+        echo "in build/make/core/main.mk for details):" 1>&2; \
+        echo "$$findres" | sort 1>&2; \
+        false; \
+      fi; \
+    )
+  endef
+endif
+
 ifdef FULL_BUILD
   ifneq (true,$(ALLOW_MISSING_DEPENDENCIES))
     # Check to ensure that all modules in PRODUCT_PACKAGES exist (opt in per product)
@@ -1312,6 +1423,8 @@
 	rm -f $@
 	$(foreach f,$(sort $(all_offending_files)),echo $(f) >> $@;)
   endif
+
+  $(call check-apex-libs-absence,$(product_target_FILES))
 else
   # We're not doing a full build, and are probably only including
   # a subset of the module makefiles.  Don't try to build any modules