Support RGBA input buffers.
This change adds separate EGL shader for RGBA->YUV conversion
and modifies the JPEG compression to render input texture
into temporary framebuffer (doing the compression if necessary).
Bug: 301023410
Test: atest virtual_camera_tests
Test: atest VirtualCameraTest
Change-Id: Id3bd19d4c364691e2b1554fcf78d5f9940754314
diff --git a/services/camera/virtualcamera/VirtualCameraDevice.cc b/services/camera/virtualcamera/VirtualCameraDevice.cc
index 0657d81..84f721b 100644
--- a/services/camera/virtualcamera/VirtualCameraDevice.cc
+++ b/services/camera/virtualcamera/VirtualCameraDevice.cc
@@ -119,9 +119,10 @@
const std::vector<SupportedStreamConfiguration>& supportedInputConfig) {
if (!std::all_of(supportedInputConfig.begin(), supportedInputConfig.end(),
[](const SupportedStreamConfiguration& config) {
- return config.pixelFormat == Format::YUV_420_888;
+ return isFormatSupportedForInput(
+ config.width, config.height, config.pixelFormat);
})) {
- ALOGE("%s: input configuration contains unsupported pixel format", __func__);
+ ALOGE("%s: input configuration contains unsupported format", __func__);
return std::nullopt;
}
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.cc b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
index 1f61e34..79c91ef 100644
--- a/services/camera/virtualcamera/VirtualCameraRenderThread.cc
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
@@ -19,6 +19,7 @@
#include <chrono>
#include <cstddef>
+#include <cstdint>
#include <future>
#include <memory>
#include <mutex>
@@ -38,6 +39,7 @@
#include "android-base/thread_annotations.h"
#include "android/binder_auto_utils.h"
#include "android/hardware_buffer.h"
+#include "ui/GraphicBuffer.h"
#include "util/EglFramebuffer.h"
#include "util/JpegUtil.h"
#include "util/MetadataBuilder.h"
@@ -109,6 +111,45 @@
return msg;
}
+std::shared_ptr<EglFrameBuffer> allocateTemporaryFramebuffer(
+ EGLDisplay eglDisplay, const uint width, const int height) {
+ const AHardwareBuffer_Desc desc{
+ .width = static_cast<uint32_t>(width),
+ .height = static_cast<uint32_t>(height),
+ .layers = 1,
+ .format = AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420,
+ .usage = AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER |
+ AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN,
+ .rfu0 = 0,
+ .rfu1 = 0};
+
+ AHardwareBuffer* hwBufferPtr;
+ int status = AHardwareBuffer_allocate(&desc, &hwBufferPtr);
+ if (status != NO_ERROR) {
+ ALOGE(
+ "%s: Failed to allocate hardware buffer for temporary framebuffer: %d",
+ __func__, status);
+ return nullptr;
+ }
+
+ return std::make_shared<EglFrameBuffer>(
+ eglDisplay,
+ std::shared_ptr<AHardwareBuffer>(hwBufferPtr, AHardwareBuffer_release));
+}
+
+bool isYuvFormat(const PixelFormat pixelFormat) {
+ switch (static_cast<android_pixel_format_t>(pixelFormat)) {
+ case HAL_PIXEL_FORMAT_YCBCR_422_I:
+ case HAL_PIXEL_FORMAT_YCBCR_422_SP:
+ case HAL_PIXEL_FORMAT_Y16:
+ case HAL_PIXEL_FORMAT_YV12:
+ case HAL_PIXEL_FORMAT_YCBCR_420_888:
+ return true;
+ default:
+ return false;
+ }
+}
+
} // namespace
CaptureRequestBuffer::CaptureRequestBuffer(int streamId, int bufferId,
@@ -218,7 +259,10 @@
ALOGV("Render thread starting");
mEglDisplayContext = std::make_unique<EglDisplayContext>();
- mEglTextureProgram = std::make_unique<EglTextureProgram>();
+ mEglTextureYuvProgram =
+ std::make_unique<EglTextureProgram>(EglTextureProgram::TextureFormat::YUV);
+ mEglTextureRgbProgram = std::make_unique<EglTextureProgram>(
+ EglTextureProgram::TextureFormat::RGBA);
mEglSurfaceTexture = std::make_unique<EglSurfaceTexture>(mInputSurfaceWidth,
mInputSurfaceHeight);
mInputSurfacePromise.set_value(mEglSurfaceTexture->getSurface());
@@ -371,6 +415,22 @@
return cameraStatus(Status::INTERNAL_ERROR);
}
+ // Let's create YUV framebuffer and render the surface into this.
+ // This will take care about rescaling as well as potential format conversion.
+ std::shared_ptr<EglFrameBuffer> framebuffer = allocateTemporaryFramebuffer(
+ mEglDisplayContext->getEglDisplay(), stream->width, stream->height);
+ if (framebuffer == nullptr) {
+ ALOGE("Failed to allocate temporary framebuffer for JPEG compression");
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+
+ // Render into temporary framebuffer.
+ ndk::ScopedAStatus status = renderIntoEglFramebuffer(*framebuffer);
+ if (!status.isOk()) {
+ ALOGE("Failed to render input texture into temporary framebuffer");
+ return status;
+ }
+
AHardwareBuffer_Planes planes_info;
int32_t rawFence = fence != nullptr ? fence->get() : -1;
@@ -382,11 +442,15 @@
return cameraStatus(Status::INTERNAL_ERROR);
}
- sp<GraphicBuffer> gBuffer = mEglSurfaceTexture->getCurrentBuffer();
+ std::shared_ptr<AHardwareBuffer> inHwBuffer = framebuffer->getHardwareBuffer();
+ GraphicBuffer* gBuffer = GraphicBuffer::fromAHardwareBuffer(inHwBuffer.get());
+
bool compressionSuccess = true;
if (gBuffer != nullptr) {
android_ycbcr ycbcr;
if (gBuffer->getPixelFormat() != HAL_PIXEL_FORMAT_YCbCr_420_888) {
+ // This should never happen since we're allocating the temporary buffer
+ // with YUV420 layout above.
ALOGE("%s: Cannot compress non-YUV buffer (pixelFormat %d)", __func__,
gBuffer->getPixelFormat());
AHardwareBuffer_unlock(hwBuffer.get(), nullptr);
@@ -441,31 +505,7 @@
return cameraStatus(Status::ILLEGAL_ARGUMENT);
}
- // Wait for fence to clear.
- if (fence != nullptr && fence->isValid()) {
- status_t ret = fence->wait(kAcquireFenceTimeout.count());
- if (ret != 0) {
- ALOGE(
- "Timeout while waiting for the acquire fence for buffer %d"
- " for streamId %d",
- bufferId, streamId);
- return cameraStatus(Status::INTERNAL_ERROR);
- }
- }
-
- mEglDisplayContext->makeCurrent();
- framebuffer->beforeDraw();
-
- if (mEglSurfaceTexture->getCurrentBuffer() == nullptr) {
- // If there's no current buffer, nothing was written to the surface and
- // texture is not initialized yet. Let's render the framebuffer black
- // instead of rendering the texture.
- glClearColor(0.0f, 0.5f, 0.5f, 0.0f);
- glClear(GL_COLOR_BUFFER_BIT);
- } else {
- mEglTextureProgram->draw(mEglSurfaceTexture->updateTexture());
- }
- framebuffer->afterDraw();
+ ndk::ScopedAStatus status = renderIntoEglFramebuffer(*framebuffer, fence);
const std::chrono::nanoseconds after =
std::chrono::duration_cast<std::chrono::nanoseconds>(
@@ -477,6 +517,43 @@
return ndk::ScopedAStatus::ok();
}
+ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer(
+ EglFrameBuffer& framebuffer, sp<Fence> fence) {
+ ALOGV("%s", __func__);
+ // Wait for fence to clear.
+ if (fence != nullptr && fence->isValid()) {
+ status_t ret = fence->wait(kAcquireFenceTimeout.count());
+ if (ret != 0) {
+ ALOGE("Timeout while waiting for the acquire fence for buffer");
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+ }
+
+ mEglDisplayContext->makeCurrent();
+ framebuffer.beforeDraw();
+
+ sp<GraphicBuffer> textureBuffer = mEglSurfaceTexture->getCurrentBuffer();
+ if (textureBuffer == nullptr) {
+ // If there's no current buffer, nothing was written to the surface and
+ // texture is not initialized yet. Let's render the framebuffer black
+ // instead of rendering the texture.
+ glClearColor(0.0f, 0.5f, 0.5f, 0.0f);
+ glClear(GL_COLOR_BUFFER_BIT);
+ } else {
+ const bool renderSuccess =
+ isYuvFormat(static_cast<PixelFormat>(textureBuffer->getPixelFormat()))
+ ? mEglTextureYuvProgram->draw(mEglSurfaceTexture->updateTexture())
+ : mEglTextureRgbProgram->draw(mEglSurfaceTexture->updateTexture());
+ if (!renderSuccess) {
+ ALOGE("%s: Failed to render texture", __func__);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+ }
+ framebuffer.afterDraw();
+
+ return ndk::ScopedAStatus::ok();
+}
+
} // namespace virtualcamera
} // namespace companion
} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.h b/services/camera/virtualcamera/VirtualCameraRenderThread.h
index a4374e6..b3aaed8 100644
--- a/services/camera/virtualcamera/VirtualCameraRenderThread.h
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.h
@@ -24,7 +24,9 @@
#include "VirtualCameraSessionContext.h"
#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h"
+#include "android/binder_auto_utils.h"
#include "util/EglDisplayContext.h"
+#include "util/EglFramebuffer.h"
#include "util/EglProgram.h"
#include "util/EglSurfaceTexture.h"
@@ -135,6 +137,13 @@
ndk::ScopedAStatus renderIntoImageStreamBuffer(int streamId, int bufferId,
sp<Fence> fence = nullptr);
+ // Render current image into provided EglFramebuffer.
+ // If fence is specified, this function will block until the fence is cleared
+ // before writing to the buffer.
+ // Always called on the render thread.
+ ndk::ScopedAStatus renderIntoEglFramebuffer(EglFrameBuffer& framebuffer,
+ sp<Fence> fence = nullptr);
+
// Camera callback
const std::shared_ptr<
::aidl::android::hardware::camera::device::ICameraDeviceCallback>
@@ -156,7 +165,8 @@
// EGL helpers - constructed and accessed only from rendering thread.
std::unique_ptr<EglDisplayContext> mEglDisplayContext;
- std::unique_ptr<EglTextureProgram> mEglTextureProgram;
+ std::unique_ptr<EglTextureProgram> mEglTextureYuvProgram;
+ std::unique_ptr<EglTextureProgram> mEglTextureRgbProgram;
std::unique_ptr<EglSurfaceTexture> mEglSurfaceTexture;
std::promise<sp<Surface>> mInputSurfacePromise;
diff --git a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl
index d9b90a6..b155147 100644
--- a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl
+++ b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl
@@ -24,5 +24,6 @@
@Backing(type="int")
enum Format {
UNKNOWN = 0,
+ RGBA_8888 = 1,
YUV_420_888 = 0x23,
}
diff --git a/services/camera/virtualcamera/tests/EglUtilTest.cc b/services/camera/virtualcamera/tests/EglUtilTest.cc
index d0b7218..589e312 100644
--- a/services/camera/virtualcamera/tests/EglUtilTest.cc
+++ b/services/camera/virtualcamera/tests/EglUtilTest.cc
@@ -61,12 +61,24 @@
EXPECT_TRUE(eglTestPatternProgram.isInitialized());
}
-TEST_F(EglTest, EglTextureProgramSuccessfulInit) {
+TEST_F(EglTest, EglTextureProgramYuvSuccessfulInit) {
if (!isGlExtensionSupported(kGlExtYuvTarget)) {
GTEST_SKIP() << "Skipping test because of missing required GL extension " << kGlExtYuvTarget;
}
- EglTextureProgram eglTextureProgram;
+ EglTextureProgram eglTextureProgram(EglTextureProgram::TextureFormat::YUV);
+
+ // Verify the shaders compiled and linked successfully.
+ EXPECT_TRUE(eglTextureProgram.isInitialized());
+}
+
+TEST_F(EglTest, EglTextureProgramRgbaSuccessfulInit) {
+ if (!isGlExtensionSupported(kGlExtYuvTarget)) {
+ GTEST_SKIP() << "Skipping test because of missing required GL extension "
+ << kGlExtYuvTarget;
+ }
+
+ EglTextureProgram eglTextureProgram(EglTextureProgram::TextureFormat::RGBA);
// Verify the shaders compiled and linked successfully.
EXPECT_TRUE(eglTextureProgram.isInitialized());
diff --git a/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc b/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc
index f4d5042..38261fb 100644
--- a/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc
+++ b/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc
@@ -153,7 +153,7 @@
createConfiguration(kVgaWidth, kVgaHeight, Format::YUV_420_888);
};
-TEST_F(VirtualCameraServiceTest, RegisterCameraSucceeds) {
+TEST_F(VirtualCameraServiceTest, RegisterCameraWithYuvInputSucceeds) {
sp<BBinder> token = sp<BBinder>::make();
ndk::SpAIBinder ndkToken(AIBinder_fromPlatformBinder(token));
bool aidlRet;
@@ -167,6 +167,20 @@
EXPECT_THAT(getCameraIds(), SizeIs(1));
}
+TEST_F(VirtualCameraServiceTest, RegisterCameraWithRgbaInputSucceeds) {
+ sp<BBinder> token = sp<BBinder>::make();
+ ndk::SpAIBinder ndkToken(AIBinder_fromPlatformBinder(token));
+ bool aidlRet;
+
+ VirtualCameraConfiguration config =
+ createConfiguration(kVgaWidth, kVgaHeight, Format::RGBA_8888);
+
+ ASSERT_TRUE(mCameraService->registerCamera(ndkToken, config, &aidlRet).isOk());
+
+ EXPECT_TRUE(aidlRet);
+ EXPECT_THAT(getCameraIds(), SizeIs(1));
+}
+
TEST_F(VirtualCameraServiceTest, RegisterCameraTwiceSecondReturnsFalse) {
createCamera();
bool aidlRet;
diff --git a/services/camera/virtualcamera/util/EglFramebuffer.cc b/services/camera/virtualcamera/util/EglFramebuffer.cc
index acf0122..57b94d3 100644
--- a/services/camera/virtualcamera/util/EglFramebuffer.cc
+++ b/services/camera/virtualcamera/util/EglFramebuffer.cc
@@ -112,6 +112,10 @@
return mHeight;
}
+std::shared_ptr<AHardwareBuffer> EglFrameBuffer::getHardwareBuffer() {
+ return mHardwareBuffer;
+}
+
} // namespace virtualcamera
} // namespace companion
} // namespace android
diff --git a/services/camera/virtualcamera/util/EglFramebuffer.h b/services/camera/virtualcamera/util/EglFramebuffer.h
index 35f85e2..b39df6d 100644
--- a/services/camera/virtualcamera/util/EglFramebuffer.h
+++ b/services/camera/virtualcamera/util/EglFramebuffer.h
@@ -50,6 +50,9 @@
// Return height of framebuffer (in pixels).
int getHeight() const;
+ // Return underlying hardware buffer.
+ std::shared_ptr<AHardwareBuffer> getHardwareBuffer();
+
private:
// Keeping shared_ptr to hardware buffer instance here prevents it from being
// freed while tied to EGL framebufer / EGL texture.
diff --git a/services/camera/virtualcamera/util/EglProgram.cc b/services/camera/virtualcamera/util/EglProgram.cc
index c468d39..510fd33 100644
--- a/services/camera/virtualcamera/util/EglProgram.cc
+++ b/services/camera/virtualcamera/util/EglProgram.cc
@@ -76,17 +76,29 @@
vTextureCoord = aTextureCoord;
})";
-constexpr char kExternalTextureFragmentShader[] = R"(#version 300 es
+constexpr char kExternalYuvTextureFragmentShader[] = R"(#version 300 es
#extension GL_OES_EGL_image_external_essl3 : require
#extension GL_EXT_YUV_target : require
precision mediump float;
in vec2 vTextureCoord;
- out vec4 fragColor;
+ layout (yuv) out vec4 fragColor;
uniform __samplerExternal2DY2YEXT uTexture;
void main() {
fragColor = texture(uTexture, vTextureCoord);
})";
+constexpr char kExternalRgbaTextureFragmentShader[] = R"(#version 300 es
+ #extension GL_OES_EGL_image_external : require
+ #extension GL_EXT_YUV_target : require
+ precision mediump float;
+ in vec2 vTextureCoord;
+ layout (yuv) out vec4 fragColor;
+ uniform samplerExternalOES uTexture;
+ void main() {
+ vec4 rgbaColor = texture(uTexture, vTextureCoord);
+ fragColor = vec4(rgb_2_yuv(rgbaColor.xyz, itu_601_full_range), 0.0);
+ })";
+
constexpr int kCoordsPerVertex = 3;
constexpr std::array<float, 12> kSquareCoords{-1.f, 1.0f, 0.0f, // top left
-1.f, -1.f, 0.0f, // bottom left
@@ -237,7 +249,7 @@
return true;
}
-EglTextureProgram::EglTextureProgram() {
+EglTextureProgram::EglTextureProgram(const TextureFormat textureFormat) {
if (!isGlExtensionSupported(kGlExtYuvTarget)) {
ALOGE(
"Cannot initialize external texture program due to missing "
@@ -245,7 +257,10 @@
return;
}
- if (initialize(kExternalTextureVertexShader, kExternalTextureFragmentShader)) {
+ const char* fragmentShaderSrc = textureFormat == TextureFormat::YUV
+ ? kExternalYuvTextureFragmentShader
+ : kExternalRgbaTextureFragmentShader;
+ if (initialize(kExternalTextureVertexShader, fragmentShaderSrc)) {
ALOGV("Successfully initialized EGL shaders for external texture program.");
} else {
ALOGE("External texture EGL shader program initialization failed.");
diff --git a/services/camera/virtualcamera/util/EglProgram.h b/services/camera/virtualcamera/util/EglProgram.h
index 8e394e7..1b5f2cd 100644
--- a/services/camera/virtualcamera/util/EglProgram.h
+++ b/services/camera/virtualcamera/util/EglProgram.h
@@ -55,7 +55,9 @@
// TODO(b/301023410) Add support for translation / cropping.
class EglTextureProgram : public EglProgram {
public:
- EglTextureProgram();
+ enum class TextureFormat { RGBA, YUV };
+
+ EglTextureProgram(TextureFormat textureFormat = TextureFormat::YUV);
bool draw(GLuint textureId);
};
diff --git a/services/camera/virtualcamera/util/Util.cc b/services/camera/virtualcamera/util/Util.cc
index 11dc3a6..df771b1 100644
--- a/services/camera/virtualcamera/util/Util.cc
+++ b/services/camera/virtualcamera/util/Util.cc
@@ -18,20 +18,26 @@
#include <unistd.h>
+#include <algorithm>
+#include <array>
+
#include "jpeglib.h"
namespace android {
namespace companion {
namespace virtualcamera {
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::hardware::common::NativeHandle;
+
// Lower bound for maximal supported texture size is at least 2048x2048
// but on most platforms will be more.
// TODO(b/301023410) - Query actual max texture size.
constexpr int kMaxTextureSize = 2048;
constexpr int kLibJpegDctSize = DCTSIZE;
-using ::aidl::android::companion::virtualcamera::Format;
-using ::aidl::android::hardware::common::NativeHandle;
+constexpr std::array<Format, 2> kSupportedFormats{Format::YUV_420_888,
+ Format::RGBA_8888};
sp<Fence> importFence(const NativeHandle& aidlHandle) {
if (aidlHandle.fds.size() != 1) {
@@ -41,11 +47,15 @@
return sp<Fence>::make(::dup(aidlHandle.fds[0].get()));
}
+bool isPixelFormatSupportedForInput(const Format format) {
+ return std::find(kSupportedFormats.begin(), kSupportedFormats.end(),
+ format) != kSupportedFormats.end();
+}
+
// Returns true if specified format is supported for virtual camera input.
bool isFormatSupportedForInput(const int width, const int height,
const Format format) {
- if (format != Format::YUV_420_888) {
- // For now only YUV_420_888 is supported for input.
+ if (!isPixelFormatSupportedForInput(format)) {
return false;
}
diff --git a/services/camera/virtualcamera/util/Util.h b/services/camera/virtualcamera/util/Util.h
index b778321..a73c99b 100644
--- a/services/camera/virtualcamera/util/Util.h
+++ b/services/camera/virtualcamera/util/Util.h
@@ -43,6 +43,10 @@
sp<Fence> importFence(
const ::aidl::android::hardware::common::NativeHandle& handle);
+// Returns true if specified pixel format is supported for virtual camera input.
+bool isPixelFormatSupportedForInput(
+ ::aidl::android::companion::virtualcamera::Format format);
+
// Returns true if specified format is supported for virtual camera input.
bool isFormatSupportedForInput(
int width, int height,