Merge "libaudiohal: Enforce serialization of calls into IModule" into main
diff --git a/media/codec2/components/apv/C2SoftApvEnc.cpp b/media/codec2/components/apv/C2SoftApvEnc.cpp
index 9d84bc7..9036df1 100644
--- a/media/codec2/components/apv/C2SoftApvEnc.cpp
+++ b/media/codec2/components/apv/C2SoftApvEnc.cpp
@@ -1056,10 +1056,20 @@
                                          input->width(), input->width(), input->width(),
                                          input->width(), input->width(), input->height(),
                                          CONV_FORMAT_I420);
-            } else if (IsYUV420(*input)) {
-                return C2_BAD_VALUE;
             } else if (IsI420(*input)) {
-                return C2_BAD_VALUE;
+                uint8_t  *srcY  = (uint8_t*)input->data()[0];
+                uint8_t  *srcU  = (uint8_t*)input->data()[1];
+                uint8_t  *srcV  = (uint8_t*)input->data()[2];
+                uint16_t *dstY  = (uint16_t*)inputFrames->frm[0].imgb->a[0];
+                uint16_t *dstUV = (uint16_t*)inputFrames->frm[0].imgb->a[1];
+                convertPlanar8ToP210(dstY, dstUV, srcY, srcU, srcV,
+                                        layout.planes[C2PlanarLayout::PLANE_Y].rowInc,
+                                        layout.planes[C2PlanarLayout::PLANE_U].rowInc,
+                                        layout.planes[C2PlanarLayout::PLANE_V].rowInc,
+                                        input->width(), input->width(),
+                                        input->width(), input->height(),
+                                        CONV_FORMAT_I420);
+
             } else {
                 ALOGE("Not supported color format. %d", mColorFormat);
                 return C2_BAD_VALUE;
@@ -1317,10 +1327,6 @@
         return;
     }
 
-    if (work->input.buffers.empty()) {
-        return;
-    }
-
     std::shared_ptr<C2GraphicView> view;
     std::shared_ptr<C2Buffer> inputBuffer = nullptr;
     if (!work->input.buffers.empty()) {
@@ -1332,7 +1338,19 @@
             work->workletsProcessed = 1u;
             return;
         }
+    } else {
+        ALOGV("Empty input Buffer");
+        uint32_t flags = 0;
+        if (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) {
+            flags |= C2FrameData::FLAG_END_OF_STREAM;
+        }
+        work->worklets.front()->output.flags = (C2FrameData::flags_t)flags;
+        work->worklets.front()->output.buffers.clear();
+        work->worklets.front()->output.ordinal = work->input.ordinal;
+        work->workletsProcessed = 1u;
+        return;
     }
+
     if (!inputBuffer) {
         fillEmptyWork(work);
         return;
@@ -1361,6 +1379,7 @@
 
     error = setEncodeArgs(&mInputFrames, view.get(), workIndex);
     if (error != C2_OK) {
+        ALOGE("setEncodeArgs has failed. err = %d", error);
         mSignalledError = true;
         work->result = error;
         work->workletsProcessed = 1u;
@@ -1382,6 +1401,7 @@
         int32_t status =
                 oapve_encode(mEncoderId, &mInputFrames, mMetaId, bits.get(), &stat, &mReconFrames);
         if (status != C2_OK) {
+            ALOGE("oapve_encode has failed. err = %d", status);
             mSignalledError = true;
             work->result = C2_CORRUPTED;
             work->workletsProcessed = 1u;
diff --git a/media/codec2/components/base/SimpleC2Component.cpp b/media/codec2/components/base/SimpleC2Component.cpp
index a03f24f..ea67bf4 100644
--- a/media/codec2/components/base/SimpleC2Component.cpp
+++ b/media/codec2/components/base/SimpleC2Component.cpp
@@ -713,6 +713,39 @@
   }
 }
 
+void convertPlanar8ToP210(uint16_t *dstY, uint16_t *dstUV,
+                              const uint8_t *srcY, const uint8_t *srcU, const uint8_t *srcV,
+                              size_t srcYStride, size_t srcUStride, size_t srcVStride,
+                              size_t dstYStride, size_t dstUVStride,
+                              uint32_t width, uint32_t height,
+                              CONV_FORMAT_T format) {
+  if (format != CONV_FORMAT_I420) {
+    ALOGE("No support for planar8 to P210. format is %d", format);
+    return;
+  }
+
+  for (int32_t y = 0; y < height; ++y) {
+    for (int32_t x = 0; x < width; ++x) {
+      dstY[x] = ((uint16_t)((double)srcY[x] * 1023 / 255 + 0.5) << 6) & 0xFFC0;
+    }
+    dstY += dstYStride;
+    srcY += srcYStride;
+  }
+
+  for (int32_t y = 0; y < height / 2; ++y) {
+    for (int32_t x = 0; x < width / 2; ++x) {
+      dstUV[x<<1] = dstUV[(x<<1) + dstUVStride] =
+                ((uint16_t)((double)srcU[x] * 1023 / 255 + 0.5) << 6) & 0xFFC0;
+      dstUV[(x<<1) + 1] = dstUV[(x<<1) + dstUVStride + 1] =
+                ((uint16_t)((double)srcV[x] * 1023 / 255 + 0.5) << 6) & 0xFFC0;
+    }
+    dstUV += dstUVStride << 1;
+    srcU += srcUStride;
+    srcV += srcVStride;
+  }
+}
+
+
 std::unique_ptr<C2Work> SimpleC2Component::WorkQueue::pop_front() {
     std::unique_ptr<C2Work> work = std::move(mQueue.front().work);
     mQueue.pop_front();
diff --git a/media/codec2/components/base/include/SimpleC2Component.h b/media/codec2/components/base/include/SimpleC2Component.h
index 4306e55..5d2e8cd 100644
--- a/media/codec2/components/base/include/SimpleC2Component.h
+++ b/media/codec2/components/base/include/SimpleC2Component.h
@@ -111,6 +111,12 @@
                               size_t dstYStride, size_t dstUVStride,
                               uint32_t width, uint32_t height,
                               CONV_FORMAT_T format);
+void convertPlanar8ToP210(uint16_t *dstY, uint16_t *dstUV,
+                              const uint8_t *srcY, const uint8_t *srcU, const uint8_t *srcV,
+                              size_t srcYStride, size_t srcUStride, size_t srcVStride,
+                              size_t dstYStride, size_t dstUVStride,
+                              uint32_t width, uint32_t height,
+                              CONV_FORMAT_T format);
 
 class SimpleC2Component
         : public C2Component, public std::enable_shared_from_this<SimpleC2Component> {
diff --git a/media/codec2/core/include/C2Config.h b/media/codec2/core/include/C2Config.h
index fa5ce77..fbd1b36 100644
--- a/media/codec2/core/include/C2Config.h
+++ b/media/codec2/core/include/C2Config.h
@@ -269,7 +269,7 @@
     kParamIndexSuspendAt, // input-surface, struct
     kParamIndexResumeAt, // input-surface, struct
     kParamIndexStopAt, // input-surface, struct
-    kParamIndexTimeOffset, // input-surface, struct
+    kParamIndexTimeOffset, // input-surface, int64_t
     kParamIndexMinFrameRate, // input-surface, float
     kParamIndexTimestampGapAdjustment, // input-surface, struct
 
@@ -299,6 +299,10 @@
 
     // allow tunnel peek behavior to be unspecified for app compatibility
     kParamIndexTunnelPeekMode, // tunnel mode, enum
+
+    // input surface
+    kParamIndexCaptureFrameRate, // input-surface, float
+    kParamIndexStopTimeOffset, // input-surface, int64_t
 };
 
 }
@@ -2651,6 +2655,14 @@
 constexpr char C2_PARAMKEY_INPUT_SURFACE_MIN_FRAME_RATE[] = "input-surface.min-frame-rate";
 
 /**
+ * Maximum fps for input surface.
+ *
+ * Drop frame to meet this.
+ */
+typedef C2PortParam<C2Tuning, C2FloatValue, kParamIndexMaxFrameRate> C2PortMaxFrameRateTuning;
+constexpr char C2_PARAMKEY_INPUT_SURFACE_MAX_FRAME_RATE[] = "input-surface.max-frame-rate";
+
+/**
  * Timestamp adjustment (override) for input surface buffers. These control the input timestamp
  * fed to the codec, but do not impact the output timestamp.
  */
@@ -2680,9 +2692,26 @@
 inline C2TimestampGapAdjustmentStruct::C2TimestampGapAdjustmentStruct()
     : mode(C2TimestampGapAdjustmentStruct::NONE), value(0) { }
 
-typedef C2PortParam<C2Tuning, C2TimestampGapAdjustmentStruct> C2PortTimestampGapTuning;
+typedef C2PortParam<C2Tuning, C2TimestampGapAdjustmentStruct, kParamIndexTimestampGapAdjustment>
+        C2PortTimestampGapTuning;
 constexpr char C2_PARAMKEY_INPUT_SURFACE_TIMESTAMP_ADJUSTMENT[] = "input-surface.timestamp-adjustment";
 
+/**
+ * Capture frame rate for input surface. During timelapse or slowmo encoding,
+ * this represents the frame rate of input surface.
+ */
+typedef C2PortParam<C2Tuning, C2FloatValue, kParamIndexCaptureFrameRate>
+        C2PortCaptureFrameRateTuning;
+constexpr char C2_PARAMKEY_INPUT_SURFACE_CAPTURE_FRAME_RATE[] = "input-surface.capture-frame-rate";
+
+/**
+ * Stop time offset for input surface. Stop time offset is the elapsed time
+ * offset to the last frame time from the stop time. This could be returned from
+ * IInputSurface when it is queried.
+ */
+typedef C2PortParam<C2Tuning, C2Int64Value, kParamIndexStopTimeOffset> C2PortStopTimeOffset;
+constexpr char C2_PARAMKEY_INPUT_SURFACE_STOP_TIME_OFFSET[] = "input-surface.stop-time-offset";
+
 /* ===================================== TUNNELED CODEC ==================================== */
 
 /**
diff --git a/media/codec2/hal/aidl/Android.bp b/media/codec2/hal/aidl/Android.bp
index e16e2b1..eaabc33 100644
--- a/media/codec2/hal/aidl/Android.bp
+++ b/media/codec2/hal/aidl/Android.bp
@@ -78,6 +78,8 @@
         "Configurable.cpp",
         "InputBufferManager.cpp",
         "ParamTypes.cpp",
+        "inputsurface/InputSurface.cpp",
+        "inputsurface/InputSurfaceConnection.cpp",
     ],
 
     header_libs: [
@@ -98,6 +100,7 @@
         "libhidlbase",
         "liblog",
         "libnativewindow",
+        "libmediandk",
         "libstagefright_aidl_bufferpool2",
         "libstagefright_bufferpool@2.0.1",
         "libui",
diff --git a/media/codec2/hal/aidl/include/codec2/aidl/inputsurface/InputSurface.h b/media/codec2/hal/aidl/include/codec2/aidl/inputsurface/InputSurface.h
new file mode 100644
index 0000000..5c2cc2e
--- /dev/null
+++ b/media/codec2/hal/aidl/include/codec2/aidl/inputsurface/InputSurface.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <aidl/android/hardware/media/c2/BnInputSurface.h>
+
+#include <codec2/aidl/Configurable.h>
+#include <util/C2InterfaceHelper.h>
+
+#include <C2.h>
+
+#include <memory>
+
+namespace aidl::android::hardware::media::c2::utils {
+
+struct InputSurface : public BnInputSurface {
+    InputSurface();
+    c2_status_t status() const;
+
+    // Methods from IInputSurface follow.
+    ::ndk::ScopedAStatus getSurface(
+            ::aidl::android::view::Surface* surface) override;
+    ::ndk::ScopedAStatus getConfigurable(
+            std::shared_ptr<IConfigurable>* configurable) override;
+    ::ndk::ScopedAStatus connect(
+            const std::shared_ptr<IInputSink>& sink,
+            std::shared_ptr<IInputSurfaceConnection>* connection) override;
+
+protected:
+    class Interface;
+    class ConfigurableIntf;
+
+    c2_status_t mInit;
+    std::shared_ptr<Interface> mIntf;
+    std::shared_ptr<CachedConfigurable> mConfigurable;
+
+    virtual ~InputSurface() override;
+
+
+    ::ndk::ScopedAIBinder_DeathRecipient mDeathRecipient;
+    static void OnBinderDied(void *cookie);
+    static void OnBinderUnlinked(void *cookie);
+    struct DeathContext;
+    DeathContext *mDeathContext;
+};
+
+}  // namespace aidl::android::hardware::media::c2::utils
diff --git a/media/codec2/hal/aidl/include/codec2/aidl/inputsurface/InputSurfaceConnection.h b/media/codec2/hal/aidl/include/codec2/aidl/inputsurface/InputSurfaceConnection.h
new file mode 100644
index 0000000..59361e1
--- /dev/null
+++ b/media/codec2/hal/aidl/include/codec2/aidl/inputsurface/InputSurfaceConnection.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <aidl/android/hardware/media/c2/BnInputSurfaceConnection.h>
+#include <media/NdkImage.h>
+
+#include <C2.h>
+
+#include <memory>
+
+namespace aidl::android::hardware::media::c2::utils {
+
+struct InputSurfaceConnection : public BnInputSurfaceConnection {
+    InputSurfaceConnection();
+    c2_status_t status() const;
+
+    // Methods from IInputSurfaceConnection follow.
+    ::ndk::ScopedAStatus disconnect() override;
+    ::ndk::ScopedAStatus signalEndOfStream() override;
+
+    // implementation specific interface.
+
+    // Submit a buffer to the connected component.
+    c2_status_t submitBuffer(
+            int32_t bufferId,
+            const AImage *buffer = nullptr,
+            int64_t timestamp = 0,
+            int fenceFd = -1);
+
+    // Submit eos to the connected component.
+    c2_status_t submitEos(int32_t bufferId);
+
+    // notify dataspace being changed to the component.
+    void dispatchDataSpaceChanged(
+            int32_t dataSpace, int32_t aspects, int32_t pixelFormat);
+
+protected:
+    virtual ~InputSurfaceConnection() override;
+};
+
+}  // namespace aidl::android::hardware::media::c2::utils
diff --git a/media/codec2/hal/aidl/inputsurface/InputSurface.cpp b/media/codec2/hal/aidl/inputsurface/InputSurface.cpp
new file mode 100644
index 0000000..5f6d176
--- /dev/null
+++ b/media/codec2/hal/aidl/inputsurface/InputSurface.cpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "Codec2-InputSurface"
+#include <android-base/logging.h>
+
+#include <codec2/aidl/inputsurface/InputSurface.h>
+
+namespace aidl::android::hardware::media::c2::utils {
+
+// Derived class of C2InterfaceHelper
+class InputSurface::Interface : public C2InterfaceHelper {
+public:
+    explicit Interface(
+            const std::shared_ptr<C2ReflectorHelper> &helper)
+        : C2InterfaceHelper(helper) {
+
+        setDerivedInstance(this);
+
+    }
+
+private:
+};
+
+class InputSurface::ConfigurableIntf : public ConfigurableC2Intf {
+public:
+};
+
+struct InputSurface::DeathContext {
+    // TODO;
+};
+
+void InputSurface::OnBinderDied(void *cookie) {
+    (void) cookie;
+}
+
+void InputSurface::OnBinderUnlinked(void *cookie) {
+    (void) cookie;
+}
+
+InputSurface::InputSurface() : mDeathContext(nullptr) {
+    mInit = C2_OK;
+}
+
+InputSurface::~InputSurface() {
+}
+
+::ndk::ScopedAStatus InputSurface::getSurface(::aidl::android::view::Surface* surface) {
+    (void) surface;
+    return ::ndk::ScopedAStatus::ok();
+}
+
+::ndk::ScopedAStatus InputSurface::getConfigurable(
+        std::shared_ptr<IConfigurable>* configurable) {
+    *configurable = mConfigurable;
+    return ::ndk::ScopedAStatus::ok();
+}
+
+::ndk::ScopedAStatus InputSurface::connect(
+        const std::shared_ptr<IInputSink>& sink,
+        std::shared_ptr<IInputSurfaceConnection>* connection) {
+    (void) sink;
+    (void) connection;
+    return ::ndk::ScopedAStatus::ok();
+}
+
+}  // namespace aidl::android::hardware::media::c2::utils
diff --git a/media/codec2/hal/aidl/inputsurface/InputSurfaceConnection.cpp b/media/codec2/hal/aidl/inputsurface/InputSurfaceConnection.cpp
new file mode 100644
index 0000000..44ca924
--- /dev/null
+++ b/media/codec2/hal/aidl/inputsurface/InputSurfaceConnection.cpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "Codec2-InputSurface"
+#include <android-base/logging.h>
+
+#include <codec2/aidl/inputsurface/InputSurfaceConnection.h>
+
+namespace aidl::android::hardware::media::c2::utils {
+
+InputSurfaceConnection::InputSurfaceConnection() {
+}
+
+InputSurfaceConnection::~InputSurfaceConnection() {
+}
+
+::ndk::ScopedAStatus InputSurfaceConnection::disconnect() {
+    return ::ndk::ScopedAStatus::ok();
+}
+
+::ndk::ScopedAStatus InputSurfaceConnection::signalEndOfStream() {
+    return ::ndk::ScopedAStatus::ok();
+}
+
+c2_status_t InputSurfaceConnection::submitBuffer(
+        int32_t bufferId, const AImage *buffer, int64_t timestamp, int fenceFd) {
+    (void)bufferId;
+    (void)buffer;
+    (void)timestamp;
+    (void)fenceFd;
+    return C2_OK;
+}
+
+c2_status_t InputSurfaceConnection::submitEos(int32_t bufferId) {
+    (void)bufferId;
+    return C2_OK;
+}
+
+void InputSurfaceConnection::dispatchDataSpaceChanged(
+            int32_t dataSpace, int32_t aspects, int32_t pixelFormat) {
+    (void)dataSpace;
+    (void)aspects;
+    (void)pixelFormat;
+}
+
+}  // namespace aidl::android::hardware::media::c2::utils
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index 1cb9ea4..58f2dcf 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -8457,6 +8457,10 @@
 
             timestampCorrectionEnabled = isTimestampCorrectionEnabled_l();
             lockEffectChains_l(effectChains);
+            // We're exiting locked scope with non empty activeTracks, make sure
+            // that we're not in standby mode which we could have entered if some
+            // tracks were muted/unmuted.
+            mStandby = false;
         }
 
         // thread mutex is now unlocked, mActiveTracks unknown, activeTracks.size() > 0
diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h b/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h
index f066c09..b29033e 100644
--- a/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h
+++ b/services/audiopolicy/common/managerdefinitions/include/AudioPolicyMix.h
@@ -147,6 +147,7 @@
 private:
     bool mixMatch(const AudioMix* mix, size_t mixIndex,
                             const audio_attributes_t& attributes,
+                            const audio_output_flags_t outputFlags,
                             const audio_config_base_t& config,
                             uid_t uid,
                             audio_session_t session);
diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp
index 3430f4b..ea78a5d 100644
--- a/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp
+++ b/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp
@@ -361,7 +361,7 @@
             continue; // Primary output already found
         }
 
-        if(!mixMatch(policyMix.get(), i, attributes, config, uid, session)) {
+        if(!mixMatch(policyMix.get(), i, attributes, flags, config, uid, session)) {
             ALOGV("%s: Mix %zu: does not match", __func__, i);
             continue; // skip the mix
         }
@@ -422,8 +422,8 @@
 }
 
 bool AudioPolicyMixCollection::mixMatch(const AudioMix* mix, size_t mixIndex,
-    const audio_attributes_t& attributes, const audio_config_base_t& config,
-    uid_t uid, audio_session_t session) {
+    const audio_attributes_t& attributes, const audio_output_flags_t outputFlags,
+    const audio_config_base_t& config, uid_t uid, audio_session_t session) {
 
     if (mix->mMixType == MIX_TYPE_PLAYERS) {
         // Loopback render mixes are created from a public API and thus restricted
@@ -451,12 +451,17 @@
         }
 
         // Permit match only if requested format and mix format are PCM and can be format
-        // adapted by the mixer, or are the same (compressed) format.
+        // adapted by the mixer, or are the same format on direct output.
         if (!is_mix_loopback(mix->mRouteFlags) &&
-            !((audio_is_linear_pcm(config.format) && audio_is_linear_pcm(mix->mFormat.format)) ||
-              (config.format == mix->mFormat.format)) &&
-              config.format != AUDIO_CONFIG_BASE_INITIALIZER.format) {
-            return false;
+                config.format != AUDIO_CONFIG_BASE_INITIALIZER.format) {
+            if (!audio_output_is_mixed_output_flags(outputFlags)) {
+                // Direct output must match format exactly.
+                if (config.format != mix->mFormat.format) return false;
+            } else {
+                // If mixable, both requested and mix format must be linear pcm.
+                if (!audio_is_linear_pcm(config.format) ||
+                          !audio_is_linear_pcm(mix->mFormat.format)) return false;
+            }
         }
 
         // if there is an address match, prioritize that match