Merge "Move CompOS to /system_ext"
diff --git a/compos/Android.bp b/compos/Android.bp
index b1e5f89..b9fcfff 100644
--- a/compos/Android.bp
+++ b/compos/Android.bp
@@ -6,7 +6,7 @@
     name: "pvm_exec",
     srcs: ["src/pvm_exec.rs"],
     rustlibs: [
-        "android.system.composd-rust",
+        "android.system.composd.internal-rust",
         "compos_aidl_interface-rust",
         "libandroid_logger",
         "libanyhow",
diff --git a/compos/apex/composd.rc b/compos/apex/composd.rc
index 3e2efb1..bf34335 100644
--- a/compos/apex/composd.rc
+++ b/compos/apex/composd.rc
@@ -17,5 +17,6 @@
     user root
     group system
     interface aidl android.system.composd
+    interface aidl android.system.composd.internal
     disabled
     oneshot
diff --git a/compos/composd/Android.bp b/compos/composd/Android.bp
index 8391ed6..a76d96c 100644
--- a/compos/composd/Android.bp
+++ b/compos/composd/Android.bp
@@ -9,6 +9,7 @@
     prefer_rlib: true,
     rustlibs: [
         "android.system.composd-rust",
+        "android.system.composd.internal-rust",
         "android.system.virtualizationservice-rust",
         "compos_aidl_interface-rust",
         "libandroid_logger",
diff --git a/compos/composd/aidl/Android.bp b/compos/composd/aidl/Android.bp
index 8116632..62c1b40 100644
--- a/compos/composd/aidl/Android.bp
+++ b/compos/composd/aidl/Android.bp
@@ -5,7 +5,6 @@
 aidl_interface {
     name: "android.system.composd",
     srcs: ["android/system/composd/*.aidl"],
-    imports: ["compos_aidl_interface"],
     // TODO: Make this stable when the APEX becomes updatable.
     unstable: true,
     backend: {
@@ -18,7 +17,17 @@
                 "com.android.compos",
             ],
         },
-        ndk: {
+    },
+}
+
+aidl_interface {
+    name: "android.system.composd.internal",
+    srcs: ["android/system/composd/internal/*.aidl"],
+    imports: ["compos_aidl_interface"],
+    // TODO: Make this stable when the APEX becomes updatable.
+    unstable: true,
+    backend: {
+        rust: {
             enabled: true,
             apex_available: [
                 "com.android.compos",
diff --git a/compos/composd/aidl/android/system/composd/IIsolatedCompilationService.aidl b/compos/composd/aidl/android/system/composd/IIsolatedCompilationService.aidl
index 3d28894..6b3a6bb 100644
--- a/compos/composd/aidl/android/system/composd/IIsolatedCompilationService.aidl
+++ b/compos/composd/aidl/android/system/composd/IIsolatedCompilationService.aidl
@@ -17,8 +17,6 @@
 
 import android.system.composd.ICompilationTask;
 import android.system.composd.ICompilationTaskCallback;
-import com.android.compos.CompilationResult;
-import com.android.compos.FdAnnotation;
 
 interface IIsolatedCompilationService {
     /**
@@ -31,22 +29,4 @@
      * a reference to the ICompilationTask until compilation completes or is cancelled.
      */
     ICompilationTask startTestCompile(ICompilationTaskCallback callback);
-
-    /**
-     * Run dex2oat in the currently running instance of the CompOS VM. This is a simple proxy
-     * to ICompOsService#compile_cmd.
-     *
-     * This method can only be called from odrefresh. If there is no currently running instance
-     * an error is returned.
-     */
-    CompilationResult compile_cmd(in String[] args, in FdAnnotation fd_annotation);
-
-    /**
-     * Run dex2oat in the currently running instance of the CompOS VM. This is a simple proxy
-     * to ICompOsService#compile.
-     *
-     * This method can only be called from libcompos_client. If there is no currently running
-     * instance an error is returned.
-     */
-    byte compile(in byte[] marshaledArguments, in FdAnnotation fd_annotation);
 }
diff --git a/compos/composd/aidl/android/system/composd/internal/ICompilationInternal.aidl b/compos/composd/aidl/android/system/composd/internal/ICompilationInternal.aidl
new file mode 100644
index 0000000..182094b
--- /dev/null
+++ b/compos/composd/aidl/android/system/composd/internal/ICompilationInternal.aidl
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.system.composd.internal;
+
+import com.android.compos.CompilationResult;
+import com.android.compos.FdAnnotation;
+
+interface ICompilationInternal {
+    /**
+     * Run dex2oat in the currently running instance of the CompOS VM. This is a simple proxy
+     * to ICompOsService#compile_cmd.
+     *
+     * This method can only be called from odrefresh. If there is no currently running instance
+     * an error is returned.
+     */
+    CompilationResult compile_cmd(in String[] args, in FdAnnotation fd_annotation);
+
+    /**
+     * Run dex2oat in the currently running instance of the CompOS VM. This is a simple proxy
+     * to ICompOsService#compile.
+     *
+     * This method can only be called from libcompos_client. If there is no currently running
+     * instance an error is returned.
+     */
+    byte compile(in byte[] marshaledArguments, in FdAnnotation fd_annotation);
+}
diff --git a/compos/composd/src/composd_main.rs b/compos/composd/src/composd_main.rs
index 671ed16..f9751c3 100644
--- a/compos/composd/src/composd_main.rs
+++ b/compos/composd/src/composd_main.rs
@@ -21,6 +21,7 @@
 mod compilation_task;
 mod instance_manager;
 mod instance_starter;
+mod internal_service;
 mod odrefresh;
 mod service;
 mod util;
@@ -30,6 +31,7 @@
 use anyhow::{Context, Result};
 use compos_common::compos_client::VmInstance;
 use log::{error, info};
+use std::sync::Arc;
 
 fn try_main() -> Result<()> {
     android_logger::init_once(
@@ -39,12 +41,16 @@
     ProcessState::start_thread_pool();
 
     let virtualization_service = VmInstance::connect_to_virtualization_service()?;
-    let instance_manager = InstanceManager::new(virtualization_service);
-    let composd_service = service::new_binder(instance_manager);
+    let instance_manager = Arc::new(InstanceManager::new(virtualization_service));
+    let composd_service = service::new_binder(instance_manager.clone());
     register_lazy_service("android.system.composd", composd_service.as_binder())
-        .context("Registering service")?;
+        .context("Registering composd service")?;
 
-    info!("Registered service, joining threadpool");
+    let internal_service = internal_service::new_binder(instance_manager);
+    register_lazy_service("android.system.composd.internal", internal_service.as_binder())
+        .context("Registering internal service")?;
+
+    info!("Registered services, joining threadpool");
     ProcessState::join_thread_pool();
 
     info!("Exiting");
diff --git a/compos/composd/src/internal_service.rs b/compos/composd/src/internal_service.rs
new file mode 100644
index 0000000..ebebaae
--- /dev/null
+++ b/compos/composd/src/internal_service.rs
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//! Implementation of ICompilationInternal, called from odrefresh during compilation.
+
+use crate::instance_manager::InstanceManager;
+use crate::util::to_binder_result;
+use android_system_composd_internal::aidl::android::system::composd::internal::ICompilationInternal::{
+    BnCompilationInternal, ICompilationInternal,
+};
+use android_system_composd::binder::{
+    self, BinderFeatures, ExceptionCode, Interface, Status, Strong, ThreadState,
+};
+use anyhow::{Context, Result};
+use binder_common::new_binder_service_specific_error;
+use compos_aidl_interface::aidl::com::android::compos::{
+    CompilationResult::CompilationResult, FdAnnotation::FdAnnotation,
+};
+use rustutils::users::AID_ROOT;
+use std::sync::Arc;
+
+pub struct CompilationInternalService {
+    instance_manager: Arc<InstanceManager>,
+}
+
+pub fn new_binder(instance_manager: Arc<InstanceManager>) -> Strong<dyn ICompilationInternal> {
+    let service = CompilationInternalService { instance_manager };
+    BnCompilationInternal::new_binder(service, BinderFeatures::default())
+}
+
+impl Interface for CompilationInternalService {}
+
+impl ICompilationInternal for CompilationInternalService {
+    fn compile_cmd(
+        &self,
+        args: &[String],
+        fd_annotation: &FdAnnotation,
+    ) -> binder::Result<CompilationResult> {
+        let calling_uid = ThreadState::get_calling_uid();
+        // This should only be called by odrefresh, which runs as root
+        if calling_uid != AID_ROOT {
+            return Err(Status::new_exception(ExceptionCode::SECURITY, None));
+        }
+        to_binder_result(self.do_compile_cmd(args, fd_annotation))
+    }
+
+    fn compile(&self, _marshaled: &[u8], _fd_annotation: &FdAnnotation) -> binder::Result<i8> {
+        Err(new_binder_service_specific_error(-1, "Not yet implemented"))
+    }
+}
+
+impl CompilationInternalService {
+    fn do_compile_cmd(
+        &self,
+        args: &[String],
+        fd_annotation: &FdAnnotation,
+    ) -> Result<CompilationResult> {
+        let compos = self.instance_manager.get_running_service()?;
+        compos.compile_cmd(args, fd_annotation).context("Compiling")
+    }
+}
diff --git a/compos/composd/src/service.rs b/compos/composd/src/service.rs
index 4d9dc58..d9963d1 100644
--- a/compos/composd/src/service.rs
+++ b/compos/composd/src/service.rs
@@ -29,17 +29,16 @@
     self, BinderFeatures, ExceptionCode, Interface, Status, Strong, ThreadState,
 };
 use anyhow::{Context, Result};
-use binder_common::new_binder_service_specific_error;
-use compos_aidl_interface::aidl::com::android::compos::{
-    CompilationResult::CompilationResult, FdAnnotation::FdAnnotation,
-};
 use rustutils::users::{AID_ROOT, AID_SYSTEM};
+use std::sync::Arc;
 
 pub struct IsolatedCompilationService {
-    instance_manager: InstanceManager,
+    instance_manager: Arc<InstanceManager>,
 }
 
-pub fn new_binder(instance_manager: InstanceManager) -> Strong<dyn IIsolatedCompilationService> {
+pub fn new_binder(
+    instance_manager: Arc<InstanceManager>,
+) -> Strong<dyn IIsolatedCompilationService> {
     let service = IsolatedCompilationService { instance_manager };
     BnIsolatedCompilationService::new_binder(service, BinderFeatures::default())
 }
@@ -58,23 +57,6 @@
         }
         to_binder_result(self.do_start_test_compile(callback))
     }
-
-    fn compile_cmd(
-        &self,
-        args: &[String],
-        fd_annotation: &FdAnnotation,
-    ) -> binder::Result<CompilationResult> {
-        let calling_uid = ThreadState::get_calling_uid();
-        // This should only be called by odrefresh, which runs as root
-        if calling_uid != AID_ROOT {
-            return Err(Status::new_exception(ExceptionCode::SECURITY, None));
-        }
-        to_binder_result(self.do_compile_cmd(args, fd_annotation))
-    }
-
-    fn compile(&self, _marshaled: &[u8], _fd_annotation: &FdAnnotation) -> binder::Result<i8> {
-        Err(new_binder_service_specific_error(-1, "Not yet implemented"))
-    }
 }
 
 impl IsolatedCompilationService {
@@ -88,13 +70,4 @@
 
         Ok(BnCompilationTask::new_binder(task, BinderFeatures::default()))
     }
-
-    fn do_compile_cmd(
-        &self,
-        args: &[String],
-        fd_annotation: &FdAnnotation,
-    ) -> Result<CompilationResult> {
-        let compos = self.instance_manager.get_running_service()?;
-        compos.compile_cmd(args, fd_annotation).context("Compiling")
-    }
 }
diff --git a/compos/libcompos_client/Android.bp b/compos/libcompos_client/Android.bp
index 5528ea1..db6294e 100644
--- a/compos/libcompos_client/Android.bp
+++ b/compos/libcompos_client/Android.bp
@@ -31,7 +31,7 @@
     srcs: ["libcompos_client.rs"],
     include_dirs: ["include"],
     rustlibs: [
-        "android.system.composd-rust",
+        "android.system.composd.internal-rust",
         "compos_aidl_interface-rust",
         "libandroid_logger",
         "libanyhow",
diff --git a/compos/libcompos_client/libcompos_client.rs b/compos/libcompos_client/libcompos_client.rs
index 55d70a4..701e515 100644
--- a/compos/libcompos_client/libcompos_client.rs
+++ b/compos/libcompos_client/libcompos_client.rs
@@ -29,8 +29,8 @@
 use std::path::Path;
 use std::slice::from_raw_parts;
 
-use android_system_composd::{
-    aidl::android::system::composd::IIsolatedCompilationService::IIsolatedCompilationService,
+use android_system_composd_internal::{
+    aidl::android::system::composd::internal::ICompilationInternal::ICompilationInternal,
     binder::wait_for_interface,
 };
 use compos_aidl_interface::aidl::com::android::compos::{
@@ -41,9 +41,9 @@
 
 const FD_SERVER_BIN: &str = "/apex/com.android.virt/bin/fd_server";
 
-fn get_composd() -> Result<Strong<dyn IIsolatedCompilationService>> {
-    wait_for_interface::<dyn IIsolatedCompilationService>("android.system.composd")
-        .context("Failed to find IIsolatedCompilationService")
+fn get_composd() -> Result<Strong<dyn ICompilationInternal>> {
+    wait_for_interface::<dyn ICompilationInternal>("android.system.composd.internal")
+        .context("Failed to find ICompilationInternal")
 }
 
 fn spawn_fd_server(fd_annotation: &FdAnnotation, ready_file: File) -> Result<Minijail> {
diff --git a/compos/src/pvm_exec.rs b/compos/src/pvm_exec.rs
index cae0702..3068c66 100644
--- a/compos/src/pvm_exec.rs
+++ b/compos/src/pvm_exec.rs
@@ -40,8 +40,8 @@
 use std::path::Path;
 use std::process::exit;
 
-use android_system_composd::{
-    aidl::android::system::composd::IIsolatedCompilationService::IIsolatedCompilationService,
+use android_system_composd_internal::{
+    aidl::android::system::composd::internal::ICompilationInternal::ICompilationInternal,
     binder::wait_for_interface,
 };
 use compos_aidl_interface::aidl::com::android::compos::{
@@ -52,9 +52,9 @@
 
 const FD_SERVER_BIN: &str = "/apex/com.android.virt/bin/fd_server";
 
-fn get_composd() -> Result<Strong<dyn IIsolatedCompilationService>> {
-    wait_for_interface::<dyn IIsolatedCompilationService>("android.system.composd")
-        .context("Failed to find IIsolatedCompilationService")
+fn get_composd() -> Result<Strong<dyn ICompilationInternal>> {
+    wait_for_interface::<dyn ICompilationInternal>("android.system.composd.internal")
+        .context("Failed to find ICompilationInternal")
 }
 
 fn get_rpc_binder(cid: u32) -> Result<Strong<dyn ICompOsService>> {
diff --git a/demo/README.md b/demo/README.md
index 7ae2e0e..37198ad 100644
--- a/demo/README.md
+++ b/demo/README.md
@@ -10,6 +10,7 @@
 
 ```
 adb install out/dist/MicrodroidDemoApp.apk
+adb shell pm grant com.android.microdroid.demo android.permission.MANAGE_VIRTUAL_MACHINE
 ```
 
 ## Running
diff --git a/docs/getting_started/goldfish.md b/docs/getting_started/goldfish.md
deleted file mode 100644
index 0705982..0000000
--- a/docs/getting_started/goldfish.md
+++ /dev/null
@@ -1,46 +0,0 @@
-# Android Emulator (goldfish)
-
-The built-in local emulator is the quickest way how to get started with KVM and Android.
-
-## x86_64
-
-KVM on x86_64 does not provide the same guest protection as arm64 but you will be able to spawn
-virtual machines and use the same APIs to communicate with the guest. The main reason for choosing
-the x86_64 emulator over its arm64 counterpart is performance. With native virtualization it is
-easily 10x faster than arm64 emulation.
-
-For optimal performance make sure to
-[enable nested virtualization](https://www.linux-kvm.org/page/Nested_Guests) on your machine.
-Don't forget to add your user account into the `kvm` group, then re-login for it to take effect.
-``` shell
-$ sudo gpasswd -a $USER kvm
-```
-
-Build Android for the emulator:
-``` shell
-$ . build/envsetup.sh
-$ lunch sdk_phone_x86_64-eng
-$ m -j$(nproc)
-```
-
-Once you have an Android image, invoke `emulator`. The script will automatically find the image you
-just built and run it in QEMU.
-``` shell
-$ emulator -no-window -show-kernel -writable-system -qemu -cpu host
-```
-Explanation of the arguments:
-  * `-no-window`: run headless
-  * `-show-kernel`: print kernel UART logs to the console (useful for debugging),
-  * `-writable-system`: support remounting `system/` as writable, needed for `adb sync`,
-  * `-qemu -cpu host`: needed to enable nested virtualization, instructs QEMU to allow Android
-    access CPU features of the host kernel
-
-If you get an error saying “x86_64 emulation currently requires hardware acceleration!”, your
-user account is not in the `kvm` group (see above).
-
-You should now see the virtual device when you run:
-``` shell
-$ adb devices
-List of devices attached
-emulator-5554   device
-```
diff --git a/docs/getting_started/index.md b/docs/getting_started/index.md
index 148b8d8..f82f982 100644
--- a/docs/getting_started/index.md
+++ b/docs/getting_started/index.md
@@ -2,88 +2,95 @@
 
 ## Prepare a device
 
-First you will need a device that is capable of running virtual machines. On arm64, this means
-a device which boots the kernel in EL2 and the kernel was built with KVM enabled.
+First you will need a device that is capable of running virtual machines. On arm64, this means a
+device which boots the kernel in EL2 and the kernel was built with KVM enabled. Unfortunately at the
+moment, we don't have an arm64 device in AOSP which does that. Instead, use cuttlefish which
+provides the same functionalities except that the virtual machines are not protected from the host
+(i.e. Android). This however should be enough for functional testing.
 
-Here are instructions for select devices:
+We support the following device:
 
- * [yukawa: Khadas VIM3L](yukawa.md) (arm64)
- * [goldfish: Android Emulator](goldfish.md) (x86_64)
+* aosp_cf_x86_64_phone (Cuttlefish a.k.a. Cloud Android)
+
+Building Cuttlefish
+
+```shell
+source build/envsetup.sh
+lunch aosp_cf_x86_64_phone-userdebug
+m
+```
+
+Run Cuttlefish locally by
+
+```shell
+acloud create --local-instance --local-image
+```
+
+## Running demo app
+
+The instruction is [here](../../demo/README.md).
 
 ## Running tests
 
-Virtualization source code and relevant tests are located in
-[packages/modules/Virtualization](https://android.googlesource.com/platform/packages/modules/Virtualization)
-of the AOSP repository.
-
-### Device-side tests
-
-The tests spawn guest VMs and test different aspects of the architecture.
-
-You can build and run them with:
+There are various tests that spawn guest VMs and check different aspects of the architecture. They
+all can run via `atest`.
 
 ```shell
-atest VirtualizationTestCases
+atest VirtualizationTestCasea
+atest MicrodroidHostTestHostCases
+atest MicrodroidDemoTestApp
 ```
 
 If you run into problems, inspect the logs produced by `atest`. Their location is printed at the
 end. The `host_log_*.zip` file should contain the output of individual commands as well as VM logs.
 
-## CrosVM
+## Spawning your own VMs with custom kernel
 
-[CrosVM](https://android.googlesource.com/platform/external/crosvm/) is a Rust-based Virtual Machine
-Monitor (VMM) originally built for ChromeOS and ported to Android.
-
-It is not installed in regular Android builds (yet!), but it's installed in the VIM3L (yukawa)
-build, as part of the `com.android.virt` APEX.
-
-### Spawning your own VMs
-
-You can spawn your own VMs by passing a JSON config file to the VirtualizationService via the `vm` tool on a
-rooted KVM-enabled device. If your device is attached over ADB, you can run:
+You can spawn your own VMs by passing a JSON config file to the VirtualizationService via the `vm`
+tool on a rooted KVM-enabled device. If your device is attached over ADB, you can run:
 
 ```shell
-$ cat > vm_config.json
+cat > vm_config.json
 {
   "kernel": "/data/local/tmp/kernel",
   "initrd": "/data/local/tmp/ramdisk",
   "params": "rdinit=/bin/init"
 }
-$ adb root
-$ adb push <kernel> /data/local/tmp/kernel
-$ adb push <ramdisk> /data/local/tmp/ramdisk
-$ adb push vm_config.json /data/local/tmp/vm_config.json
-$ adb shell "start virtualizationservice"
-$ adb shell "/apex/com.android.virt/bin/vm run /data/local/tmp/vm_config.json"
+adb root
+adb push <kernel> /data/local/tmp/kernel
+adb push <ramdisk> /data/local/tmp/ramdisk
+adb push vm_config.json /data/local/tmp/vm_config.json
+adb shell "start virtualizationservice"
+adb shell "/apex/com.android.virt/bin/vm run /data/local/tmp/vm_config.json"
 ```
 
 The `vm` command also has other subcommands for debugging; run `/apex/com.android.virt/bin/vm help`
 for details.
 
-### Building and updating CrosVM and VirtualizationService
+## Spawning your own VMs with Microdroid
 
-You can update CrosVM and the VirtualizationService by updating the `com.android.virt` APEX. If your
-device already has `com.android.virt` (e.g. VIM3L):
+[Microdroid](../../microdroid/README.md) is a lightweight version of Android that is intended to run
+on pVM. You can manually run the demo app on top of Microdroid as follows:
 
 ```shell
-$ TARGET_BUILD_APPS="com.android.virt" m
-$ adb install $ANDROID_PRODUCT_OUT/system/apex/com.android.virt.apex
-$ adb reboot
+TARGET_BUILD_APPS=MicrodroidDemoApp m apps_only dist
+adb shell mkdir -p /data/local/tmp/virt
+adb push out/dist/MicrodroidDemoApp.apk /data/local/tmp/virt/
+adb shell /apex/com.android.virt/bin/vm run-app \
+  --debug full \
+  /data/local/tmp/virt/MicrodroidDemoApp.apk \
+  /data/local/tmp/virt/MicrodroidDemoApp.apk.idsig \
+  /data/local/tmp/virt/instance.img assets/vm_config.json
 ```
 
-If it doesn't have the APEX yet, you first need to place it manually to the
-system partition.
+## Building and updating CrosVM and VirtualizationService
+
+You can update CrosVM and the VirtualizationService by updating the `com.android.virt` APEX instead
+of rebuilding the entire image.
 
 ```shell
-$ adb root
-$ adb disable-verity
-$ adb reboot
-$ adb wait-for-device root
-$ adb remount
-$ m com.android.virt
-$ adb sync
-$ adb reboot
+banchan com.android.virt aosp_arm64   // or aosp_x86_64 if the device is cuttlefish
+m apps_only dist
+adb install out/dist/com.android.virt.apex
+adb reboot
 ```
-
-Once the APEX is in `/system/apex`, you can use `adb install` to update it
-further.
diff --git a/docs/getting_started/yukawa.md b/docs/getting_started/yukawa.md
deleted file mode 100644
index 8ff569a..0000000
--- a/docs/getting_started/yukawa.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# Khadas VIM3L (yukawa)
-
-The [Khadas VIM3L](https://www.khadas.com/vim3l) is an extremely hackable development board with an
-Amlogic Armv8.2 SoC and complete upstream support in U-boot, Linux and even
-[AOSP](https://android.googlesource.com/device/amlogic/yukawa/+/refs/heads/master).
-That makes it a compelling target for testing virtualization.
-
-The [prebuilt kernel](https://android.googlesource.com/device/amlogic/yukawa-kernel/+/refs/heads/master)
-in AOSP is currently not GKI, but it is close and kept up to date.
-
-Note that the `yukawa` target has SELinux policy set to `permissive`.
-
-Resources:
-  * [AOSP instructions](https://android.googlesource.com/device/amlogic/yukawa/+/refs/heads/master/sei610/README)
-    for flashing a bootloader with `fastboot` support
-  * [Manufaturer's wiki](https://docs.khadas.com/vim3/index.html) for things like setting up UART
-    and entering recovery mode
-  * [go/vim3l](https://goto.google.com/vim3l) is a more detailed document but only accessible to
-    Google employees
-
-Build Android for the board:
-``` shell
-$ . build/envsetup.sh
-$ lunch yukawa-userdebug
-$ export TARGET_VIM3L=true
-$ export TARGET_KERNEL_USE=5.10
-$ m
-```
-
-Flash your device and reboot.
diff --git a/javalib/src/android/system/virtualmachine/VirtualMachine.java b/javalib/src/android/system/virtualmachine/VirtualMachine.java
index 6556b87..d04da0e 100644
--- a/javalib/src/android/system/virtualmachine/VirtualMachine.java
+++ b/javalib/src/android/system/virtualmachine/VirtualMachine.java
@@ -33,6 +33,7 @@
 import android.system.virtualizationservice.PartitionType;
 import android.system.virtualizationservice.VirtualMachineAppConfig;
 import android.system.virtualizationservice.VirtualMachineState;
+import android.util.JsonReader;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -40,13 +41,17 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.zip.ZipFile;
 
 /**
  * A handle to the virtual machine. The virtual machine is local to the app which created the
@@ -67,6 +72,9 @@
     /** Name of the idsig file for a VM */
     private static final String IDSIG_FILE = "idsig";
 
+    /** Name of the idsig files for extra APKs. */
+    private static final String EXTRA_IDSIG_FILE_PREFIX = "extra_idsig_";
+
     /** Name of the virtualization service. */
     private static final String SERVICE_NAME = "android.system.virtualizationservice";
 
@@ -100,6 +108,22 @@
     /** Path to the idsig file for this VM. */
     private final @NonNull File mIdsigFilePath;
 
+    private static class ExtraApkSpec {
+        public final File apk;
+        public final File idsig;
+
+        ExtraApkSpec(File apk, File idsig) {
+            this.apk = apk;
+            this.idsig = idsig;
+        }
+    }
+
+    /**
+     * List of extra apks. Apks are specified by the vm config, and corresponding idsigs are to be
+     * generated.
+     */
+    private final @NonNull List<ExtraApkSpec> mExtraApks;
+
     /** Size of the instance image. 10 MB. */
     private static final long INSTANCE_FILE_SIZE = 10 * 1024 * 1024;
 
@@ -128,16 +152,18 @@
     }
 
     private VirtualMachine(
-            @NonNull Context context, @NonNull String name, @NonNull VirtualMachineConfig config) {
+            @NonNull Context context, @NonNull String name, @NonNull VirtualMachineConfig config)
+            throws VirtualMachineException {
         mPackageName = context.getPackageName();
         mName = name;
         mConfig = config;
+        mConfigFilePath = getConfigFilePath(context, name);
 
         final File vmRoot = new File(context.getFilesDir(), VM_DIR);
         final File thisVmDir = new File(vmRoot, mName);
-        mConfigFilePath = new File(thisVmDir, CONFIG_FILE);
         mInstanceFilePath = new File(thisVmDir, INSTANCE_IMAGE_FILE);
         mIdsigFilePath = new File(thisVmDir, IDSIG_FILE);
+        mExtraApks = setupExtraApks(context, config, thisVmDir);
     }
 
     /**
@@ -198,11 +224,10 @@
     /** Loads a virtual machine that is already created before. */
     /* package */ static @NonNull VirtualMachine load(
             @NonNull Context context, @NonNull String name) throws VirtualMachineException {
-        VirtualMachine vm = new VirtualMachine(context, name, /* config */ null);
-
-        try (FileInputStream input = new FileInputStream(vm.mConfigFilePath)) {
-            VirtualMachineConfig config = VirtualMachineConfig.from(input);
-            vm.mConfig = config;
+        File configFilePath = getConfigFilePath(context, name);
+        VirtualMachineConfig config;
+        try (FileInputStream input = new FileInputStream(configFilePath)) {
+            config = VirtualMachineConfig.from(input);
         } catch (FileNotFoundException e) {
             // The VM doesn't exist.
             return null;
@@ -210,6 +235,8 @@
             throw new VirtualMachineException(e);
         }
 
+        VirtualMachine vm = new VirtualMachine(context, name, config);
+
         // If config file exists, but the instance image file doesn't, it means that the VM is
         // corrupted. That's different from the case that the VM doesn't exist. Throw an exception
         // instead of returning null.
@@ -292,6 +319,9 @@
 
         try {
             mIdsigFilePath.createNewFile();
+            for (ExtraApkSpec extraApk : mExtraApks) {
+                extraApk.idsig.createNewFile();
+            }
         } catch (IOException e) {
             // If the file already exists, exception is not thrown.
             throw new VirtualMachineException("failed to create idsig file", e);
@@ -320,9 +350,20 @@
             service.createOrUpdateIdsigFile(
                     appConfig.apk, ParcelFileDescriptor.open(mIdsigFilePath, MODE_READ_WRITE));
 
+            for (ExtraApkSpec extraApk : mExtraApks) {
+                service.createOrUpdateIdsigFile(
+                        ParcelFileDescriptor.open(extraApk.apk, MODE_READ_ONLY),
+                        ParcelFileDescriptor.open(extraApk.idsig, MODE_READ_WRITE));
+            }
+
             // Re-open idsig file in read-only mode
             appConfig.idsig = ParcelFileDescriptor.open(mIdsigFilePath, MODE_READ_ONLY);
             appConfig.instanceImage = ParcelFileDescriptor.open(mInstanceFilePath, MODE_READ_WRITE);
+            List<ParcelFileDescriptor> extraIdsigs = new ArrayList<>();
+            for (ExtraApkSpec extraApk : mExtraApks) {
+                extraIdsigs.add(ParcelFileDescriptor.open(extraApk.idsig, MODE_READ_ONLY));
+            }
+            appConfig.extraIdsigs = extraIdsigs;
 
             android.system.virtualizationservice.VirtualMachineConfig vmConfigParcel =
                     android.system.virtualizationservice.VirtualMachineConfig.appConfig(appConfig);
@@ -426,6 +467,9 @@
             throw new VirtualMachineException("Virtual machine is not stopped");
         }
         final File vmRootDir = mConfigFilePath.getParentFile();
+        for (ExtraApkSpec extraApks : mExtraApks) {
+            extraApks.idsig.delete();
+        }
         mConfigFilePath.delete();
         mInstanceFilePath.delete();
         mIdsigFilePath.delete();
@@ -507,4 +551,76 @@
         sb.append(")");
         return sb.toString();
     }
+
+    private static List<String> parseExtraApkListFromPayloadConfig(JsonReader reader)
+            throws VirtualMachineException {
+        /**
+         * JSON schema from packages/modules/Virtualization/microdroid/payload/config/src/lib.rs:
+         *
+         * <p>{ "extra_apks": [ { "path": "/system/app/foo.apk", }, ... ], ... }
+         */
+        try {
+            List<String> apks = new ArrayList<>();
+
+            reader.beginObject();
+            while (reader.hasNext()) {
+                if (reader.nextName().equals("extra_apks")) {
+                    reader.beginArray();
+                    while (reader.hasNext()) {
+                        reader.beginObject();
+                        String name = reader.nextName();
+                        if (name.equals("path")) {
+                            apks.add(reader.nextString());
+                        } else {
+                            reader.skipValue();
+                        }
+                        reader.endObject();
+                    }
+                    reader.endArray();
+                } else {
+                    reader.skipValue();
+                }
+            }
+            reader.endObject();
+            return apks;
+        } catch (IOException e) {
+            throw new VirtualMachineException(e);
+        }
+    }
+
+    /**
+     * Reads the payload config inside the application, parses extra APK information, and then
+     * creates corresponding idsig file paths.
+     */
+    private static List<ExtraApkSpec> setupExtraApks(
+            @NonNull Context context, @NonNull VirtualMachineConfig config, @NonNull File vmDir)
+            throws VirtualMachineException {
+        try {
+            ZipFile zipFile = new ZipFile(context.getPackageCodePath());
+            String payloadPath = config.getPayloadConfigPath();
+            InputStream inputStream =
+                    zipFile.getInputStream(zipFile.getEntry(config.getPayloadConfigPath()));
+            List<String> apkList =
+                    parseExtraApkListFromPayloadConfig(
+                            new JsonReader(new InputStreamReader(inputStream)));
+
+            List<ExtraApkSpec> extraApks = new ArrayList<>();
+            for (int i = 0; i < apkList.size(); ++i) {
+                extraApks.add(
+                        new ExtraApkSpec(
+                                new File(apkList.get(i)),
+                                new File(vmDir, EXTRA_IDSIG_FILE_PREFIX + i)));
+            }
+
+            return extraApks;
+        } catch (IOException e) {
+            throw new VirtualMachineException("Couldn't parse extra apks from the vm config", e);
+        }
+    }
+
+    private static File getConfigFilePath(@NonNull Context context, @NonNull String name) {
+        final File vmRoot = new File(context.getFilesDir(), VM_DIR);
+        final File thisVmDir = new File(vmRoot, name);
+        return new File(thisVmDir, CONFIG_FILE);
+    }
 }
diff --git a/microdroid/README.md b/microdroid/README.md
index 196c543..b7c70d5 100644
--- a/microdroid/README.md
+++ b/microdroid/README.md
@@ -21,11 +21,8 @@
 Build the target after adding the line, and flash it. This step needs to be done
 only once for the target.
 
-If you are using `yukawa` (VIM3L) or `aosp_cf_x86_64_phone` (Cuttlefish), adding
-above line is not necessary as it's already done.
-
-Instructions for building and flashing Android for `yukawa` can be found
-[here](../docs/getting_started/yukawa.md).
+If you are using `aosp_oriole` (Pixel 6) or `aosp_cf_x86_64_phone` (Cuttlefish),
+adding above line is not necessary as it's already done.
 
 ## Building and installing microdroid
 
diff --git a/microdroid/payload/config/src/lib.rs b/microdroid/payload/config/src/lib.rs
index 2547f3d..67e8feb 100644
--- a/microdroid/payload/config/src/lib.rs
+++ b/microdroid/payload/config/src/lib.rs
@@ -31,6 +31,10 @@
     #[serde(default)]
     pub apexes: Vec<ApexConfig>,
 
+    /// Extra APKs to be passed to a VM
+    #[serde(default)]
+    pub extra_apks: Vec<ApkConfig>,
+
     /// Tells VirtualizationService to use staged APEXes if possible
     #[serde(default)]
     pub prefer_staged: bool,
@@ -91,3 +95,10 @@
     /// The name of APEX
     pub name: String,
 }
+
+/// APK config
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct ApkConfig {
+    /// The path of APK
+    pub path: String,
+}
diff --git a/virtualizationservice/aidl/android/system/virtualizationservice/VirtualMachineAppConfig.aidl b/virtualizationservice/aidl/android/system/virtualizationservice/VirtualMachineAppConfig.aidl
index 073c088..0cb187c 100644
--- a/virtualizationservice/aidl/android/system/virtualizationservice/VirtualMachineAppConfig.aidl
+++ b/virtualizationservice/aidl/android/system/virtualizationservice/VirtualMachineAppConfig.aidl
@@ -23,6 +23,9 @@
     /** idsig for an APK */
     ParcelFileDescriptor idsig;
 
+    /** Idsigs for the extra APKs. Must match with the extra_apks in the payload config. */
+    List<ParcelFileDescriptor> extraIdsigs;
+
     /** instance.img that has per-instance data */
     ParcelFileDescriptor instanceImage;
 
diff --git a/virtualizationservice/src/payload.rs b/virtualizationservice/src/payload.rs
index 55eb19b..8f7a69a 100644
--- a/virtualizationservice/src/payload.rs
+++ b/virtualizationservice/src/payload.rs
@@ -20,7 +20,7 @@
     VirtualMachineRawConfig::VirtualMachineRawConfig,
 };
 use android_system_virtualizationservice::binder::ParcelFileDescriptor;
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use binder::wait_for_interface;
 use log::{error, info};
 use microdroid_metadata::{ApexPayload, ApkPayload, Metadata};
@@ -194,22 +194,34 @@
 ///   ..
 ///   microdroid-apk: apk
 ///   microdroid-apk-idsig: idsig
+///   extra-apk-0:   additional apk 0
+///   extra-idsig-0: additional idsig 0
+///   extra-apk-1:   additional apk 1
+///   extra-idsig-1: additional idsig 1
+///   ..
 fn make_payload_disk(
+    app_config: &VirtualMachineAppConfig,
     apk_file: File,
     idsig_file: File,
-    config_path: &str,
     vm_payload_config: &VmPayloadConfig,
     temporary_directory: &Path,
-    debug_level: DebugLevel,
 ) -> Result<DiskImage> {
+    if vm_payload_config.extra_apks.len() != app_config.extraIdsigs.len() {
+        bail!(
+            "payload config has {} apks, but app config has {} idsigs",
+            vm_payload_config.extra_apks.len(),
+            app_config.extraIdsigs.len()
+        );
+    }
+
     let pm = PackageManager::new()?;
     let apex_list = pm.get_apex_list(vm_payload_config.prefer_staged)?;
 
     // collect APEX names from config
-    let apexes = collect_apex_names(&apex_list, &vm_payload_config.apexes, debug_level);
+    let apexes = collect_apex_names(&apex_list, &vm_payload_config.apexes, app_config.debugLevel);
     info!("Microdroid payload APEXes: {:?}", apexes);
 
-    let metadata_file = make_metadata_file(config_path, &apexes, temporary_directory)?;
+    let metadata_file = make_metadata_file(&app_config.configPath, &apexes, temporary_directory)?;
     // put metadata at the first partition
     let mut partitions = vec![Partition {
         label: "payload-metadata".to_owned(),
@@ -237,6 +249,23 @@
         writable: false,
     });
 
+    // we've already checked that extra_apks and extraIdsigs are in the same size.
+    let extra_apks = &vm_payload_config.extra_apks;
+    let extra_idsigs = &app_config.extraIdsigs;
+    for (i, (extra_apk, extra_idsig)) in extra_apks.iter().zip(extra_idsigs.iter()).enumerate() {
+        partitions.push(Partition {
+            label: format!("extra-apk-{}", i),
+            image: Some(ParcelFileDescriptor::new(File::open(PathBuf::from(&extra_apk.path))?)),
+            writable: false,
+        });
+
+        partitions.push(Partition {
+            label: format!("extra-idsig-{}", i),
+            image: Some(ParcelFileDescriptor::new(extra_idsig.as_ref().try_clone()?)),
+            writable: false,
+        });
+    }
+
     Ok(DiskImage { image: None, partitions, writable: false })
 }
 
@@ -298,12 +327,11 @@
     vm_config: &mut VirtualMachineRawConfig,
 ) -> Result<()> {
     vm_config.disks.push(make_payload_disk(
+        config,
         apk_file,
         idsig_file,
-        &config.configPath,
         vm_payload_config,
         temporary_directory,
-        config.debugLevel,
     )?);
 
     vm_config.disks[1].partitions.push(Partition {
diff --git a/vm/Android.bp b/vm/Android.bp
index 9010674..2d22562 100644
--- a/vm/Android.bp
+++ b/vm/Android.bp
@@ -14,10 +14,12 @@
         "libenv_logger",
         "liblibc",
         "liblog_rust",
+        "libmicrodroid_payload_config",
         "libserde_json",
         "libserde",
         "libstructopt",
         "libvmconfig",
+        "libzip",
     ],
     apex_available: [
         "com.android.virt",
diff --git a/vm/src/main.rs b/vm/src/main.rs
index 87bcda7..d53305b 100644
--- a/vm/src/main.rs
+++ b/vm/src/main.rs
@@ -33,6 +33,9 @@
 const VIRTUALIZATION_SERVICE_BINDER_SERVICE_IDENTIFIER: &str =
     "android.system.virtualizationservice";
 
+#[derive(Debug)]
+struct Idsigs(Vec<PathBuf>);
+
 #[derive(StructOpt)]
 #[structopt(no_version, global_settings = &[AppSettings::DisableVersion])]
 enum Opt {
@@ -73,6 +76,10 @@
         /// in the VM config file.
         #[structopt(short, long)]
         mem: Option<u32>,
+
+        /// Paths to extra idsig files.
+        #[structopt(long)]
+        extra_idsigs: Vec<PathBuf>,
     },
     /// Run a virtual machine
     Run {
@@ -138,20 +145,30 @@
         .context("Failed to find VirtualizationService")?;
 
     match opt {
-        Opt::RunApp { apk, idsig, instance, config_path, daemonize, console, log, debug, mem } => {
-            command_run_app(
-                service,
-                &apk,
-                &idsig,
-                &instance,
-                &config_path,
-                daemonize,
-                console.as_deref(),
-                log.as_deref(),
-                debug,
-                mem,
-            )
-        }
+        Opt::RunApp {
+            apk,
+            idsig,
+            instance,
+            config_path,
+            daemonize,
+            console,
+            log,
+            debug,
+            mem,
+            extra_idsigs,
+        } => command_run_app(
+            service,
+            &apk,
+            &idsig,
+            &instance,
+            &config_path,
+            daemonize,
+            console.as_deref(),
+            log.as_deref(),
+            debug,
+            mem,
+            &extra_idsigs,
+        ),
         Opt::Run { config, daemonize, console } => {
             command_run(service, &config, daemonize, console.as_deref(), /* mem */ None)
         }
diff --git a/vm/src/run.rs b/vm/src/run.rs
index 15775cb..1cd51a1 100644
--- a/vm/src/run.rs
+++ b/vm/src/run.rs
@@ -28,12 +28,14 @@
     BinderFeatures, DeathRecipient, IBinder, ParcelFileDescriptor, Strong,
 };
 use android_system_virtualizationservice::binder::{Interface, Result as BinderResult};
-use anyhow::{Context, Error};
+use anyhow::{bail, Context, Error};
+use microdroid_payload_config::VmPayloadConfig;
 use std::fs::File;
 use std::io::{self, BufRead, BufReader};
 use std::os::unix::io::{AsRawFd, FromRawFd};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use vmconfig::{open_parcel_file, VmConfig};
+use zip::ZipArchive;
 
 /// Run a VM from the given APK, idsig, and config.
 #[allow(clippy::too_many_arguments)]
@@ -48,7 +50,23 @@
     log_path: Option<&Path>,
     debug_level: DebugLevel,
     mem: Option<u32>,
+    extra_idsigs: &[PathBuf],
 ) -> Result<(), Error> {
+    let extra_apks = parse_extra_apk_list(apk, config_path)?;
+    if extra_apks.len() != extra_idsigs.len() {
+        bail!(
+            "Found {} extra apks, but there are {} extra idsigs",
+            extra_apks.len(),
+            extra_idsigs.len()
+        )
+    }
+
+    for i in 0..extra_apks.len() {
+        let extra_apk_fd = ParcelFileDescriptor::new(File::open(&extra_apks[i])?);
+        let extra_idsig_fd = ParcelFileDescriptor::new(File::create(&extra_idsigs[i])?);
+        service.createOrUpdateIdsigFile(&extra_apk_fd, &extra_idsig_fd)?;
+    }
+
     let apk_file = File::open(apk).context("Failed to open APK file")?;
     let idsig_file = File::create(idsig).context("Failed to create idsig file")?;
 
@@ -69,9 +87,13 @@
         )?;
     }
 
+    let extra_idsig_files: Result<Vec<File>, _> = extra_idsigs.iter().map(File::open).collect();
+    let extra_idsig_fds = extra_idsig_files?.into_iter().map(ParcelFileDescriptor::new).collect();
+
     let config = VirtualMachineConfig::AppConfig(VirtualMachineAppConfig {
         apk: apk_fd.into(),
         idsig: idsig_fd.into(),
+        extraIdsigs: extra_idsig_fds,
         instanceImage: open_parcel_file(instance, true /* writable */)?.into(),
         configPath: config_path.to_owned(),
         debugLevel: debug_level,
@@ -204,6 +226,13 @@
     Ok(death_recipient)
 }
 
+fn parse_extra_apk_list(apk: &Path, config_path: &str) -> Result<Vec<String>, Error> {
+    let mut archive = ZipArchive::new(File::open(apk)?)?;
+    let config_file = archive.by_name(config_path)?;
+    let config: VmPayloadConfig = serde_json::from_reader(config_file)?;
+    Ok(config.extra_apks.into_iter().map(|x| x.path).collect())
+}
+
 #[derive(Debug)]
 struct VirtualMachineCallback {
     dead: AtomicFlag,