Use GL to render preview.
To speed up the preview, we direct the decoder output to a
SurfaceTexture, then draw the texture to a surface. The media
rendering parameters (crop, black-border) are implemented
using different vertex coordinates. The color effects are
implemented using fragment shaders. Currently only three color
effects are implemented, but that's all the appplication uses.
Change-Id: If84439fee572ed37ea077749ef9f2bd4f78703e1
diff --git a/libvideoeditor/lvpp/NativeWindowRenderer.cpp b/libvideoeditor/lvpp/NativeWindowRenderer.cpp
new file mode 100755
index 0000000..cde8b89
--- /dev/null
+++ b/libvideoeditor/lvpp/NativeWindowRenderer.cpp
@@ -0,0 +1,621 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "NativeWindowRenderer"
+#include "NativeWindowRenderer.h"
+
+#include <GLES2/gl2.h>
+#include <GLES2/gl2ext.h>
+#include <cutils/log.h>
+#include <gui/SurfaceTexture.h>
+#include <gui/SurfaceTextureClient.h>
+#include <stagefright/MediaBuffer.h>
+#include <stagefright/MediaDebug.h>
+#include <stagefright/MetaData.h>
+#include <surfaceflinger/Surface.h>
+#include "VideoEditorTools.h"
+
+#define CHECK_EGL_ERROR CHECK(EGL_SUCCESS == eglGetError())
+#define CHECK_GL_ERROR CHECK(GLenum(GL_NO_ERROR) == glGetError())
+
+//
+// Vertex and fragment programs
+//
+
+// The matrix is derived from
+// frameworks/base/media/libstagefright/colorconversion/ColorConverter.cpp
+//
+// R * 255 = 1.164 * (Y - 16) + 1.596 * (V - 128)
+// G * 255 = 1.164 * (Y - 16) - 0.813 * (V - 128) - 0.391 * (U - 128)
+// B * 255 = 1.164 * (Y - 16) + 2.018 * (U - 128)
+//
+// Here we assume YUV are in the range of [0,255], RGB are in the range of
+// [0, 1]
+#define RGB2YUV_MATRIX \
+"const mat4 rgb2yuv = mat4("\
+" 65.52255, -37.79398, 111.98732, 0.00000,"\
+" 128.62729, -74.19334, -93.81088, 0.00000,"\
+" 24.92233, 111.98732, -18.17644, 0.00000,"\
+" 16.00000, 128.00000, 128.00000, 1.00000);\n"
+
+#define YUV2RGB_MATRIX \
+"const mat4 yuv2rgb = mat4("\
+" 0.00456, 0.00456, 0.00456, 0.00000,"\
+" 0.00000, -0.00153, 0.00791, 0.00000,"\
+" 0.00626, -0.00319, 0.00000, 0.00000,"\
+" -0.87416, 0.53133, -1.08599, 1.00000);\n"
+
+static const char vSrcNormal[] =
+ "attribute vec4 vPosition;\n"
+ "attribute vec2 vTexPos;\n"
+ "uniform mat4 texMatrix;\n"
+ "varying vec2 texCoords;\n"
+ "varying float topDown;\n"
+ "void main() {\n"
+ " gl_Position = vPosition;\n"
+ " texCoords = (texMatrix * vec4(vTexPos, 0.0, 1.0)).xy;\n"
+ " topDown = vTexPos.y;\n"
+ "}\n";
+
+static const char fSrcNormal[] =
+ "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "uniform samplerExternalOES texSampler;\n"
+ "varying vec2 texCoords;\n"
+ "void main() {\n"
+ " gl_FragColor = texture2D(texSampler, texCoords);\n"
+ "}\n";
+
+static const char fSrcSepia[] =
+ "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "uniform samplerExternalOES texSampler;\n"
+ "varying vec2 texCoords;\n"
+ RGB2YUV_MATRIX
+ YUV2RGB_MATRIX
+ "void main() {\n"
+ " vec4 rgb = texture2D(texSampler, texCoords);\n"
+ " vec4 yuv = rgb2yuv * rgb;\n"
+ " yuv = vec4(yuv.x, 117.0, 139.0, 1.0);\n"
+ " gl_FragColor = yuv2rgb * yuv;\n"
+ "}\n";
+
+static const char fSrcNegative[] =
+ "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "uniform samplerExternalOES texSampler;\n"
+ "varying vec2 texCoords;\n"
+ RGB2YUV_MATRIX
+ YUV2RGB_MATRIX
+ "void main() {\n"
+ " vec4 rgb = texture2D(texSampler, texCoords);\n"
+ " vec4 yuv = rgb2yuv * rgb;\n"
+ " yuv = vec4(255.0 - yuv.x, yuv.y, yuv.z, 1.0);\n"
+ " gl_FragColor = yuv2rgb * yuv;\n"
+ "}\n";
+
+static const char fSrcGradient[] =
+ "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "uniform samplerExternalOES texSampler;\n"
+ "varying vec2 texCoords;\n"
+ "varying float topDown;\n"
+ RGB2YUV_MATRIX
+ YUV2RGB_MATRIX
+ "void main() {\n"
+ " vec4 rgb = texture2D(texSampler, texCoords);\n"
+ " vec4 yuv = rgb2yuv * rgb;\n"
+ " vec4 mixin = vec4(15.0/31.0, 59.0/63.0, 31.0/31.0, 1.0);\n"
+ " vec4 yuv2 = rgb2yuv * vec4((mixin.xyz * topDown), 1);\n"
+ " yuv = vec4(yuv.x, yuv2.y, yuv2.z, 1);\n"
+ " gl_FragColor = yuv2rgb * yuv;\n"
+ "}\n";
+
+namespace android {
+
+NativeWindowRenderer::NativeWindowRenderer(sp<ANativeWindow> nativeWindow,
+ int width, int height)
+ : mNativeWindow(nativeWindow)
+ , mDstWidth(width)
+ , mDstHeight(height)
+ , mLastVideoEffect(-1)
+ , mNextTextureId(100)
+ , mActiveInputs(0)
+ , mThreadCmd(CMD_IDLE) {
+ createThread(threadStart, this);
+}
+
+// The functions below run in the GL thread.
+//
+// All GL-related work is done in this thread, and other threads send
+// requests to this thread using a command code. We expect most of the
+// time there will only be one thread sending in requests, so we let
+// other threads wait until the request is finished by GL thread.
+
+int NativeWindowRenderer::threadStart(void* self) {
+ LOGD("create thread");
+ ((NativeWindowRenderer*)self)->glThread();
+ return 0;
+}
+
+void NativeWindowRenderer::glThread() {
+ initializeEGL();
+ createPrograms();
+
+ Mutex::Autolock autoLock(mLock);
+ bool quit = false;
+ while (!quit) {
+ switch (mThreadCmd) {
+ case CMD_IDLE:
+ mCond.wait(mLock);
+ continue;
+ case CMD_RENDER_INPUT:
+ render(mThreadRenderInput);
+ break;
+ case CMD_RESERVE_TEXTURE:
+ glBindTexture(GL_TEXTURE_EXTERNAL_OES, mThreadTextureId);
+ CHECK_GL_ERROR;
+ break;
+ case CMD_DELETE_TEXTURE:
+ glDeleteTextures(1, &mThreadTextureId);
+ break;
+ case CMD_QUIT:
+ terminateEGL();
+ quit = true;
+ break;
+ }
+ // Tell the requester that the command is finished.
+ mThreadCmd = CMD_IDLE;
+ mCond.broadcast();
+ }
+ LOGD("quit");
+}
+
+void NativeWindowRenderer::initializeEGL() {
+ mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+ CHECK_EGL_ERROR;
+
+ EGLint majorVersion;
+ EGLint minorVersion;
+ eglInitialize(mEglDisplay, &majorVersion, &minorVersion);
+ CHECK_EGL_ERROR;
+
+ EGLConfig config;
+ EGLint numConfigs = -1;
+ EGLint configAttribs[] = {
+ EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
+ EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
+ EGL_RED_SIZE, 8,
+ EGL_GREEN_SIZE, 8,
+ EGL_BLUE_SIZE, 8,
+ EGL_NONE
+ };
+ eglChooseConfig(mEglDisplay, configAttribs, &config, 1, &numConfigs);
+ CHECK_EGL_ERROR;
+
+ mEglSurface = eglCreateWindowSurface(mEglDisplay, config,
+ mNativeWindow.get(), NULL);
+ CHECK_EGL_ERROR;
+
+ EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
+ mEglContext = eglCreateContext(mEglDisplay, config, EGL_NO_CONTEXT,
+ contextAttribs);
+ CHECK_EGL_ERROR;
+
+ eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext);
+ CHECK_EGL_ERROR;
+}
+
+void NativeWindowRenderer::terminateEGL() {
+ eglDestroyContext(mEglDisplay, mEglContext);
+ eglDestroySurface(mEglDisplay, mEglSurface);
+ eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+ eglTerminate(mEglDisplay);
+}
+
+void NativeWindowRenderer::createPrograms() {
+ GLuint vShader;
+ loadShader(GL_VERTEX_SHADER, vSrcNormal, &vShader);
+
+ const char* fSrc[NUMBER_OF_EFFECTS] = {
+ fSrcNormal, fSrcSepia, fSrcNegative, fSrcGradient
+ };
+
+ for (int i = 0; i < NUMBER_OF_EFFECTS; i++) {
+ GLuint fShader;
+ loadShader(GL_FRAGMENT_SHADER, fSrc[i], &fShader);
+ createProgram(vShader, fShader, &mProgram[i]);
+ glDeleteShader(fShader);
+ CHECK_GL_ERROR;
+ }
+
+ glDeleteShader(vShader);
+ CHECK_GL_ERROR;
+}
+
+void NativeWindowRenderer::createProgram(
+ GLuint vertexShader, GLuint fragmentShader, GLuint* outPgm) {
+
+ GLuint program = glCreateProgram();
+ CHECK_GL_ERROR;
+
+ glAttachShader(program, vertexShader);
+ CHECK_GL_ERROR;
+
+ glAttachShader(program, fragmentShader);
+ CHECK_GL_ERROR;
+
+ glLinkProgram(program);
+ CHECK_GL_ERROR;
+
+ GLint linkStatus = GL_FALSE;
+ glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
+ if (linkStatus != GL_TRUE) {
+ GLint infoLen = 0;
+ glGetProgramiv(program, GL_INFO_LOG_LENGTH, &infoLen);
+ if (infoLen) {
+ char* buf = (char*) malloc(infoLen);
+ if (buf) {
+ glGetProgramInfoLog(program, infoLen, NULL, buf);
+ LOGE("Program link log:\n%s\n", buf);
+ free(buf);
+ }
+ }
+ glDeleteProgram(program);
+ program = 0;
+ }
+
+ *outPgm = program;
+}
+
+void NativeWindowRenderer::loadShader(GLenum shaderType, const char* pSource,
+ GLuint* outShader) {
+ GLuint shader = glCreateShader(shaderType);
+ CHECK_GL_ERROR;
+
+ glShaderSource(shader, 1, &pSource, NULL);
+ CHECK_GL_ERROR;
+
+ glCompileShader(shader);
+ CHECK_GL_ERROR;
+
+ GLint compiled = 0;
+ glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
+ if (!compiled) {
+ GLint infoLen = 0;
+ glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);
+ char* buf = (char*) malloc(infoLen);
+ if (buf) {
+ glGetShaderInfoLog(shader, infoLen, NULL, buf);
+ LOGE("Shader compile log:\n%s\n", buf);
+ free(buf);
+ }
+ glDeleteShader(shader);
+ shader = 0;
+ }
+ *outShader = shader;
+}
+
+NativeWindowRenderer::~NativeWindowRenderer() {
+ CHECK(mActiveInputs == 0);
+ startRequest(CMD_QUIT);
+ sendRequest();
+}
+
+void NativeWindowRenderer::render(RenderInput* input) {
+ sp<SurfaceTexture> ST = input->mST;
+ sp<SurfaceTextureClient> STC = input->mSTC;
+
+ if (input->mIsExternalBuffer) {
+ queueExternalBuffer(STC.get(), input->mBuffer,
+ input->mWidth, input->mHeight);
+ } else {
+ queueInternalBuffer(STC.get(), input->mBuffer);
+ }
+
+ ST->updateTexImage();
+ glClearColor(0, 0, 0, 0);
+ glClear(GL_COLOR_BUFFER_BIT);
+
+ calculatePositionCoordinates(input->mRenderingMode,
+ input->mWidth, input->mHeight);
+
+ const GLfloat textureCoordinates[] = {
+ 0.0f, 1.0f,
+ 0.0f, 0.0f,
+ 1.0f, 0.0f,
+ 1.0f, 1.0f,
+ };
+
+ updateProgramAndHandle(input->mVideoEffect);
+
+ glVertexAttribPointer(mPositionHandle, 2, GL_FLOAT, GL_FALSE, 0,
+ mPositionCoordinates);
+ CHECK_GL_ERROR;
+
+ glEnableVertexAttribArray(mPositionHandle);
+ CHECK_GL_ERROR;
+
+ glVertexAttribPointer(mTexPosHandle, 2, GL_FLOAT, GL_FALSE, 0,
+ textureCoordinates);
+ CHECK_GL_ERROR;
+
+ glEnableVertexAttribArray(mTexPosHandle);
+ CHECK_GL_ERROR;
+
+ GLfloat texMatrix[16];
+ ST->getTransformMatrix(texMatrix);
+ glUniformMatrix4fv(mTexMatrixHandle, 1, GL_FALSE, texMatrix);
+ CHECK_GL_ERROR;
+
+ glBindTexture(GL_TEXTURE_EXTERNAL_OES, input->mTextureId);
+ CHECK_GL_ERROR;
+
+ glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+ glTexParameteri(
+ GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glTexParameteri(
+ GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ CHECK_GL_ERROR;
+
+ glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+ CHECK_GL_ERROR;
+
+ eglSwapBuffers(mEglDisplay, mEglSurface);
+}
+
+void NativeWindowRenderer::queueInternalBuffer(ANativeWindow *anw,
+ MediaBuffer* buffer) {
+ int64_t timeUs;
+ CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
+ native_window_set_buffers_timestamp(anw, timeUs * 1000);
+ status_t err = anw->queueBuffer(anw, buffer->graphicBuffer().get());
+ if (err != 0) {
+ LOGE("queueBuffer failed with error %s (%d)", strerror(-err), -err);
+ return;
+ }
+
+ sp<MetaData> metaData = buffer->meta_data();
+ metaData->setInt32(kKeyRendered, 1);
+}
+
+void NativeWindowRenderer::queueExternalBuffer(ANativeWindow* anw,
+ MediaBuffer* buffer, int width, int height) {
+ native_window_set_buffers_geometry(anw, width, height,
+ HAL_PIXEL_FORMAT_YV12);
+ native_window_set_usage(anw, GRALLOC_USAGE_SW_WRITE_OFTEN);
+
+ ANativeWindowBuffer* anb;
+ anw->dequeueBuffer(anw, &anb);
+ CHECK(anb != NULL);
+
+ sp<GraphicBuffer> buf(new GraphicBuffer(anb, false));
+ CHECK(NO_ERROR == anw->lockBuffer(anw, buf->getNativeBuffer()));
+
+ // Copy the buffer
+ uint8_t* img = NULL;
+ buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img));
+ copyYV12Buffer(buffer, img, width, height, buf->getStride());
+ buf->unlock();
+ CHECK(NO_ERROR == anw->queueBuffer(anw, buf->getNativeBuffer()));
+}
+
+void NativeWindowRenderer::copyYV12Buffer(MediaBuffer* src, uint8_t* dst,
+ int srcWidth, int srcHeight, int stride) {
+ int strideUV = (stride / 2 + 0xf) & ~0xf;
+ uint8_t* p = (uint8_t*)src->data() + src->range_offset();
+ // Y
+ for (int i = srcHeight; i > 0; i--) {
+ memcpy(dst, p, srcWidth);
+ dst += stride;
+ p += srcWidth;
+ }
+ // The src is I420, the dst is YV12.
+ // U
+ p += srcWidth * srcHeight / 4;
+ for (int i = srcHeight / 2; i > 0; i--) {
+ memcpy(dst, p, srcWidth / 2);
+ dst += strideUV;
+ p += srcWidth / 2;
+ }
+ // V
+ p -= srcWidth * srcHeight / 2;
+ for (int i = srcHeight / 2; i > 0; i--) {
+ memcpy(dst, p, srcWidth / 2);
+ dst += strideUV;
+ p += srcWidth / 2;
+ }
+}
+
+void NativeWindowRenderer::updateProgramAndHandle(uint32_t videoEffect) {
+ if (mLastVideoEffect == videoEffect) {
+ return;
+ }
+
+ mLastVideoEffect = videoEffect;
+ int i;
+ switch (mLastVideoEffect) {
+ case VIDEO_EFFECT_NONE:
+ i = 0;
+ break;
+ case VIDEO_EFFECT_SEPIA:
+ i = 1;
+ break;
+ case VIDEO_EFFECT_NEGATIVE:
+ i = 2;
+ break;
+ case VIDEO_EFFECT_GRADIENT:
+ i = 3;
+ break;
+ default:
+ i = 0;
+ break;
+ }
+ glUseProgram(mProgram[i]);
+ CHECK_GL_ERROR;
+
+ mPositionHandle = glGetAttribLocation(mProgram[i], "vPosition");
+ mTexPosHandle = glGetAttribLocation(mProgram[i], "vTexPos");
+ mTexMatrixHandle = glGetUniformLocation(mProgram[i], "texMatrix");
+ CHECK_GL_ERROR;
+}
+
+void NativeWindowRenderer::calculatePositionCoordinates(
+ M4xVSS_MediaRendering renderingMode, int srcWidth, int srcHeight) {
+ float x, y;
+ switch (renderingMode) {
+ case M4xVSS_kResizing:
+ default:
+ x = 1;
+ y = 1;
+ break;
+ case M4xVSS_kCropping:
+ x = float(srcWidth) / mDstWidth;
+ y = float(srcHeight) / mDstHeight;
+ // Make the smaller side 1
+ if (x > y) {
+ x /= y;
+ y = 1;
+ } else {
+ y /= x;
+ x = 1;
+ }
+ break;
+ case M4xVSS_kBlackBorders:
+ x = float(srcWidth) / mDstWidth;
+ y = float(srcHeight) / mDstHeight;
+ // Make the larger side 1
+ if (x > y) {
+ y /= x;
+ x = 1;
+ } else {
+ x /= y;
+ y = 1;
+ }
+ break;
+ }
+
+ mPositionCoordinates[0] = -x;
+ mPositionCoordinates[1] = y;
+ mPositionCoordinates[2] = -x;
+ mPositionCoordinates[3] = -y;
+ mPositionCoordinates[4] = x;
+ mPositionCoordinates[5] = -y;
+ mPositionCoordinates[6] = x;
+ mPositionCoordinates[7] = y;
+}
+
+//
+// The functions below run in other threads.
+//
+
+void NativeWindowRenderer::startRequest(int cmd) {
+ mLock.lock();
+ while (mThreadCmd != CMD_IDLE) {
+ mCond.wait(mLock);
+ }
+ mThreadCmd = cmd;
+}
+
+void NativeWindowRenderer::sendRequest() {
+ mCond.broadcast();
+ while (mThreadCmd != CMD_IDLE) {
+ mCond.wait(mLock);
+ }
+ mLock.unlock();
+}
+
+RenderInput* NativeWindowRenderer::createRenderInput() {
+ LOGD("new render input %d", mNextTextureId);
+ RenderInput* input = new RenderInput(this, mNextTextureId);
+
+ startRequest(CMD_RESERVE_TEXTURE);
+ mThreadTextureId = mNextTextureId;
+ sendRequest();
+
+ mNextTextureId++;
+ mActiveInputs++;
+ return input;
+}
+
+void NativeWindowRenderer::destroyRenderInput(RenderInput* input) {
+ LOGD("destroy render input %d", input->mTextureId);
+ GLuint textureId = input->mTextureId;
+ delete input;
+
+ startRequest(CMD_DELETE_TEXTURE);
+ mThreadTextureId = textureId;
+ sendRequest();
+
+ mActiveInputs--;
+}
+
+//
+// RenderInput
+//
+
+RenderInput::RenderInput(NativeWindowRenderer* renderer, GLuint textureId)
+ : mRenderer(renderer)
+ , mTextureId(textureId) {
+ mST = new SurfaceTexture(mTextureId);
+ mSTC = new SurfaceTextureClient(mST);
+}
+
+RenderInput::~RenderInput() {
+}
+
+ANativeWindow* RenderInput::getTargetWindow() {
+ return mSTC.get();
+}
+
+void RenderInput::updateVideoSize(sp<MetaData> meta) {
+ CHECK(meta->findInt32(kKeyWidth, &mWidth));
+ CHECK(meta->findInt32(kKeyHeight, &mHeight));
+
+ int left, top, right, bottom;
+ if (meta->findRect(kKeyCropRect, &left, &top, &right, &bottom)) {
+ mWidth = right - left + 1;
+ mHeight = bottom - top + 1;
+ }
+
+ // If rotation degrees is 90 or 270, swap width and height
+ // (mWidth and mHeight are the _rotated_ source rectangle).
+ int32_t rotationDegrees;
+ if (!meta->findInt32(kKeyRotation, &rotationDegrees)) {
+ rotationDegrees = 0;
+ }
+
+ if (rotationDegrees == 90 || rotationDegrees == 270) {
+ int tmp = mWidth;
+ mWidth = mHeight;
+ mHeight = tmp;
+ }
+}
+
+void RenderInput::render(MediaBuffer* buffer, uint32_t videoEffect,
+ M4xVSS_MediaRendering renderingMode, bool isExternalBuffer) {
+ mVideoEffect = videoEffect;
+ mRenderingMode = renderingMode;
+ mIsExternalBuffer = isExternalBuffer;
+ mBuffer = buffer;
+
+ mRenderer->startRequest(NativeWindowRenderer::CMD_RENDER_INPUT);
+ mRenderer->mThreadRenderInput = this;
+ mRenderer->sendRequest();
+}
+
+} // namespace android