media: support MediaCodec::detachOutputSurface

Bug: 325550522
Test: atest CodecDecoderSurfaceTest.java
Change-Id: Ifcb678c3843fe9d6f60486282ae54587ba28bff1
diff --git a/media/java/android/media/MediaCodec.java b/media/java/android/media/MediaCodec.java
index 2018a39..17c4a77 100644
--- a/media/java/android/media/MediaCodec.java
+++ b/media/java/android/media/MediaCodec.java
@@ -2364,12 +2364,16 @@
         }
 
         // at the moment no codecs support detachable surface
+        boolean canDetach = GetFlag(() -> android.media.codec.Flags.nullOutputSurfaceSupport());
         if (GetFlag(() -> android.media.codec.Flags.nullOutputSurface())) {
             // Detached surface flag is only meaningful if surface is null. Otherwise, it is
             // ignored.
-            if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0) {
+            if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0 && !canDetach) {
                 throw new IllegalArgumentException("Codec does not support detached surface");
             }
+        } else {
+            // don't allow detaching if API is disabled
+            canDetach = false;
         }
 
         String[] keys = null;
@@ -2411,6 +2415,14 @@
         }
 
         native_configure(keys, values, surface, crypto, descramblerBinder, flags);
+
+        if (canDetach) {
+            // If we were able to configure native codec with a detached surface
+            // we now know that we have a surface.
+            if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0) {
+                mHasSurface = true;
+            }
+        }
     }
 
     /**
@@ -2455,12 +2467,19 @@
         if (!mHasSurface) {
             throw new IllegalStateException("codec was not configured for an output surface");
         }
+
         // note: we still have a surface in detached mode, so keep mHasSurface
         // we also technically allow calling detachOutputSurface multiple times in a row
-        throw new IllegalStateException("codec does not support detaching output surface");
-        // native_detachSurface();
+
+        if (GetFlag(() -> android.media.codec.Flags.nullOutputSurfaceSupport())) {
+            native_detachOutputSurface();
+        } else {
+            throw new IllegalStateException("codec does not support detaching output surface");
+        }
     }
 
+    private native void native_detachOutputSurface();
+
     /**
      * Create a persistent input surface that can be used with codecs that normally have an input
      * surface, such as video encoders. A persistent input can be reused by subsequent
diff --git a/media/jni/android_media_MediaCodec.cpp b/media/jni/android_media_MediaCodec.cpp
index 0fc80dd..82561f9 100644
--- a/media/jni/android_media_MediaCodec.cpp
+++ b/media/jni/android_media_MediaCodec.cpp
@@ -389,6 +389,14 @@
     return err;
 }
 
+status_t JMediaCodec::detachOutputSurface() {
+    status_t err = mCodec->detachOutputSurface();
+    if (err == OK) {
+        mSurfaceTextureClient.clear();
+    }
+    return err;
+}
+
 status_t JMediaCodec::createInputSurface(
         sp<IGraphicBufferProducer>* bufferProducer) {
     return mCodec->createInputSurface(bufferProducer);
@@ -1798,6 +1806,20 @@
     throwExceptionAsNecessary(env, err, codec);
 }
 
+static void android_media_MediaCodec_native_detachOutputSurface(
+        JNIEnv *env,
+        jobject thiz) {
+    sp<JMediaCodec> codec = getMediaCodec(env, thiz);
+
+    if (codec == NULL || codec->initCheck() != OK) {
+        throwExceptionAsNecessary(env, INVALID_OPERATION, codec);
+        return;
+    }
+
+    status_t err = codec->detachOutputSurface();
+    throwExceptionAsNecessary(env, err, codec);
+}
+
 sp<PersistentSurface> android_media_MediaCodec_getPersistentInputSurface(
         JNIEnv* env, jobject object) {
     sp<PersistentSurface> persistentSurface;
@@ -4107,6 +4129,10 @@
       "(Landroid/view/Surface;)V",
       (void *)android_media_MediaCodec_native_setSurface },
 
+    { "native_detachOutputSurface",
+      "()V",
+      (void *)android_media_MediaCodec_native_detachOutputSurface },
+
     { "createInputSurface", "()Landroid/view/Surface;",
       (void *)android_media_MediaCodec_createInputSurface },
 
diff --git a/media/jni/android_media_MediaCodec.h b/media/jni/android_media_MediaCodec.h
index abb23f5..c9b6b7f6 100644
--- a/media/jni/android_media_MediaCodec.h
+++ b/media/jni/android_media_MediaCodec.h
@@ -80,6 +80,8 @@
     status_t setSurface(
             const sp<IGraphicBufferProducer> &surface);
 
+    status_t detachOutputSurface();
+
     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
     status_t setInputSurface(const sp<PersistentSurface> &surface);