Initial libmediaformatshaper
mediaformatshaper library to amend MediaFormat parameters before
encoding. This is the initial cut of the library.
Disabled by default, enable with "setprop debug.stagefright.enableshaping 1"
Bug: 182827840
Test: build, boot, encode video
Change-Id: I8cefb6ed6ad286ae2192796bf15760e873e0d2f3
diff --git a/apex/Android.bp b/apex/Android.bp
index aacb9a8..eedd603 100644
--- a/apex/Android.bp
+++ b/apex/Android.bp
@@ -51,7 +51,10 @@
},
},
// JNI
- native_shared_libs: ["libmediaparser-jni"],
+ native_shared_libs: [
+ "libmediaparser-jni",
+ "libmediaformatshaper",
+ ],
compile_multilib: "both",
prebuilts: [
"code_coverage.policy",
diff --git a/media/libmedia/Android.bp b/media/libmedia/Android.bp
index 9e33610..b2056ad 100644
--- a/media/libmedia/Android.bp
+++ b/media/libmedia/Android.bp
@@ -432,5 +432,8 @@
},
},
- apex_available: ["com.android.media"],
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media"
+ ],
}
diff --git a/media/libmediaformatshaper/Android.bp b/media/libmediaformatshaper/Android.bp
new file mode 100644
index 0000000..731ff4c
--- /dev/null
+++ b/media/libmediaformatshaper/Android.bp
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+// these headers include the structure of needed function pointers
+cc_library_headers {
+ name: "libmediaformatshaper_headers",
+ export_include_dirs: ["include"],
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media",
+ ],
+ min_sdk_version: "29",
+ host_supported: true,
+ target: {
+ darwin: {
+ enabled: false,
+ },
+ },
+}
+
+cc_defaults {
+ name: "libmediaformatshaper_defaults",
+ srcs: [
+ "CodecProperties.cpp",
+ "FormatShaper.cpp",
+ "ManageShapingCodecs.cpp",
+ "VideoShaper.cpp",
+ "VQApply.cpp",
+ ],
+
+ local_include_dirs: [
+ "include",
+ ],
+
+ shared_libs: [
+ "liblog",
+ "libutils",
+ ],
+
+ cflags: [
+ "-Werror",
+ "-Wall",
+ "-fvisibility=hidden",
+ "-Wthread-safety", // enables GUARDED_BY()
+ ],
+
+ target: {
+ android: {
+ shared_libs: [
+ "libmediandk#29",
+ ],
+ },
+ },
+
+ sanitize: {
+ cfi: true,
+ misc_undefined: [
+ "unsigned-integer-overflow",
+ "signed-integer-overflow",
+ ],
+ },
+}
+
+cc_library {
+ name: "libmediaformatshaper",
+ defaults: ["libmediaformatshaper_defaults"],
+
+ min_sdk_version: "29",
+
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media",
+ ],
+
+ version_script: "exports.lds",
+
+}
diff --git a/media/libmediaformatshaper/CodecProperties.cpp b/media/libmediaformatshaper/CodecProperties.cpp
new file mode 100644
index 0000000..dccfd95
--- /dev/null
+++ b/media/libmediaformatshaper/CodecProperties.cpp
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "CodecProperties"
+#include <utils/Log.h>
+
+#include <string>
+
+#include <media/formatshaper/CodecProperties.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+CodecProperties::CodecProperties(std::string name, std::string mediaType) {
+ mName = name;
+ mMediaType = mediaType;
+}
+
+std::string CodecProperties::getName(){
+ return mName;
+}
+
+std::string CodecProperties::getMediaType(){
+ return mMediaType;
+}
+
+int CodecProperties::supportedMinimumQuality() {
+ return mMinimumQuality;
+}
+void CodecProperties::setSupportedMinimumQuality(int vmaf) {
+ mMinimumQuality = vmaf;
+}
+
+int CodecProperties::targetQpMax() {
+ return mTargetQpMax;
+}
+void CodecProperties::setTargetQpMax(int qpMax) {
+ mTargetQpMax = qpMax;
+}
+
+// what API is this codec set up for (e.g. API of the associated partition)
+// vendor-side (OEM) codecs may be older, due to 'vendor freeze' and treble
+int CodecProperties::supportedApi() {
+ return mApi;
+}
+
+std::string CodecProperties::getMapping(std::string key, std::string kind) {
+ ALOGV("getMapping(key %s, kind %s )", key.c_str(), kind.c_str());
+ //play with mMappings
+ auto mapped = mMappings.find(kind + "-" + key);
+ if (mapped != mMappings.end()) {
+ std::string result = mapped->second;
+ ALOGV("getMapping(%s, %s) -> %s", key.c_str(), kind.c_str(), result.c_str());
+ return result;
+ }
+ ALOGV("nope, return unchanged key");
+ return key;
+}
+
+
+// really a bit of debugging code here.
+void CodecProperties::showMappings() {
+ ALOGD("Mappings:");
+ int count = 0;
+ for (const auto& [key, value] : mMappings) {
+ count++;
+ ALOGD("'%s' -> '%s'", key.c_str(), value.c_str());
+ }
+ ALOGD("total %d mappings", count);
+}
+
+void CodecProperties::setMapping(std::string kind, std::string key, std::string value) {
+ ALOGV("setMapping(%s,%s,%s)", kind.c_str(), key.c_str(), value.c_str());
+ std::string metaKey = kind + "-" + key;
+ mMappings.insert({metaKey, value});
+}
+
+const char **CodecProperties::getMappings(std::string kind, bool reverse) {
+ ALOGV("getMappings(kind %s, reverse %d", kind.c_str(), reverse);
+ // how many do we need?
+ int count = mMappings.size();
+ if (count == 0) {
+ ALOGV("empty mappings");
+ return nullptr;
+ }
+ size_t size = sizeof(char *) * (2 * count + 2);
+ const char **result = (const char **)malloc(size);
+ if (result == nullptr) {
+ ALOGW("no memory to return mappings");
+ return nullptr;
+ }
+ memset(result, '\0', size);
+
+ const char **pp = result;
+ for (const auto& [key, value] : mMappings) {
+ // split out the kind/key
+ size_t pos = key.find('-');
+ if (pos == std::string::npos) {
+ ALOGD("ignoring malformed key: %s", key.c_str());
+ continue;
+ }
+ std::string actualKind = key.substr(0,pos);
+ if (kind.length() != 0 && kind != actualKind) {
+ ALOGD("kinds don't match: want '%s' got '%s'", kind.c_str(), actualKind.c_str());
+ continue;
+ }
+ if (reverse) {
+ // codec specific -> std aka 'unmapping'
+ pp[0] = strdup( value.c_str());
+ pp[1] = strdup( key.substr(pos+1).c_str());
+ } else {
+ // std -> codec specific
+ pp[0] = strdup( key.substr(pos+1).c_str());
+ pp[1] = strdup( value.c_str());
+ }
+ ALOGV(" %s -> %s", pp[0], pp[1]);
+ pp += 2;
+ }
+
+ pp[0] = nullptr;
+ pp[1] = nullptr;
+
+ return result;
+}
+
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/FormatShaper.cpp b/media/libmediaformatshaper/FormatShaper.cpp
new file mode 100644
index 0000000..ca4dc72
--- /dev/null
+++ b/media/libmediaformatshaper/FormatShaper.cpp
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "FormatShaper"
+#include <utils/Log.h>
+
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+
+#include <media/formatshaper/VQops.h>
+#include <media/formatshaper/CodecProperties.h>
+#include <media/formatshaper/FormatShaper.h>
+#include <media/formatshaper/VideoShaper.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+//
+// Caller retains ownership of and responsibility for inFormat
+//
+
+//
+// the interface to the outside
+//
+
+int shapeFormat(shaperHandle_t shaper, AMediaFormat* inFormat, int flags) {
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return -1;
+ }
+ if (!codec->isRegistered()) {
+ return -1;
+ }
+
+ // run through the list of possible transformations
+ //
+
+ std::string mediaType = codec->getMediaType();
+ if (strncmp(mediaType.c_str(), "video/", 6) == 0) {
+ // video specific shaping
+ (void) videoShaper(codec, inFormat, flags);
+
+ } else if (strncmp(mediaType.c_str(), "audio/", 6) == 0) {
+ // audio specific shaping
+
+ } else {
+ ALOGV("unknown mediatype '%s', left untouched", mediaType.c_str());
+
+ }
+
+ return 0;
+}
+
+int setMap(shaperHandle_t shaper, const char *kind, const char *key, const char *value) {
+ ALOGV("setMap: kind %s key %s -> value %s", kind, key, value);
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return -1;
+ }
+ // must not yet be registered
+ if (codec->isRegistered()) {
+ return -1;
+ }
+
+ codec->setMapping(kind, key, value);
+ return 0;
+}
+
+int setFeature(shaperHandle_t shaper, const char *feature, int value) {
+ ALOGV("set_feature: feature %s value %d", feature, value);
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return -1;
+ }
+ // must not yet be registered
+ if (codec->isRegistered()) {
+ return -1;
+ }
+
+ if (!strcmp(feature, "vq-minimum-quality")) {
+ codec->setSupportedMinimumQuality(value);
+ } else if (!strcmp(feature, "vq-supports-qp")) {
+ codec->setSupportsQp(value != 0);
+ } else if (!strcmp(feature, "vq-target-qpmax")) {
+ codec->setTargetQpMax(value);
+ } else if (!strcmp(feature, "vq-target-bppx100")) {
+ double bpp = value / 100.0;
+ codec->setBpp(bpp);
+ } else {
+ // changed nothing, don't mark as configured
+ return 0;
+ }
+ return 0;
+}
+
+/*
+ * The routines that manage finding, creating, and registering the shapers.
+ */
+
+shaperHandle_t findShaper(const char *codecName, const char *mediaType) {
+ CodecProperties *codec = findCodec(codecName, mediaType);
+ return (shaperHandle_t) codec;
+}
+
+shaperHandle_t createShaper(const char *codecName, const char *mediaType) {
+ CodecProperties *codec = new CodecProperties(codecName, mediaType);
+ return (shaperHandle_t) codec;
+}
+
+shaperHandle_t registerShaper(shaperHandle_t shaper, const char *codecName, const char *mediaType) {
+ ALOGV("registerShaper(handle, codecName %s, mediaType %s", codecName, mediaType);
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return nullptr;
+ }
+ // must not yet be registered
+ if (codec->isRegistered()) {
+ return nullptr;
+ }
+
+ codec = registerCodec(codec, codecName, mediaType);
+ return (shaperHandle_t) codec;
+}
+
+// mapping & unmapping
+// give me the mappings for 'kind'.
+// kind==null (or empty string), means *all* mappings
+
+const char **getMappings(shaperHandle_t shaper, const char *kind) {
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr)
+ return nullptr;
+ if (kind == nullptr)
+ kind = "";
+
+ return codec->getMappings(kind, /* reverse */ false);
+}
+
+const char **getReverseMappings(shaperHandle_t shaper, const char *kind) {
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr)
+ return nullptr;
+ if (kind == nullptr)
+ kind = "";
+
+ return codec->getMappings(kind, /* reverse */ true);
+}
+
+
+// the system grabs this structure
+__attribute__ ((visibility ("default")))
+extern "C" FormatShaperOps_t shaper_ops = {
+ .version = SHAPER_VERSION_V1,
+
+ .findShaper = findShaper,
+ .createShaper = createShaper,
+ .setMap = setMap,
+ .setFeature = setFeature,
+ .registerShaper = registerShaper,
+
+ .shapeFormat = shapeFormat,
+ .getMappings = getMappings,
+ .getReverseMappings = getReverseMappings,
+};
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/ManageShapingCodecs.cpp b/media/libmediaformatshaper/ManageShapingCodecs.cpp
new file mode 100644
index 0000000..bdc395f
--- /dev/null
+++ b/media/libmediaformatshaper/ManageShapingCodecs.cpp
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "ManageShapingCodecs"
+#include <utils/Log.h>
+
+#include <mutex>
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+#include <media/formatshaper/CodecProperties.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+// manage the list of codec information.
+//
+// XXX: the mutex here is too heavy; rework that.
+//
+
+static std::mutex sCodecMutex;
+static std::map<std::string, CodecProperties*> sCodecTraits;
+
+CodecProperties *findCodec(const char *codecName, const char *mediaType) {
+ CodecProperties *codec = nullptr;
+
+ // synthesize a name from both codecName + mediaType
+ // some codecs support multiple media types and may have different capabilities
+ // for each media type
+ //
+ std::string codecKey = codecName;
+ codecKey += "-";
+ codecKey += mediaType;
+
+ std::lock_guard _l(sCodecMutex);
+
+ auto it = sCodecTraits.find(codecKey);
+ if (it != sCodecTraits.end()) {
+ codec = it->second;
+ }
+
+ return codec;
+}
+
+CodecProperties *registerCodec(CodecProperties *codec, const char *codecName,
+ const char *mediaType) {
+
+ CodecProperties *registeredCodec = nullptr;
+
+ if (codec->isRegistered()) {
+ return nullptr;
+ }
+
+ // synthesize a name from both codecName + mediaType
+ // some codecs support multiple media types and may have different capabilities
+ // for each media type
+ //
+ std::string codecKey = codecName;
+ codecKey += "-";
+ codecKey += mediaType;
+
+ std::lock_guard _l(sCodecMutex);
+
+ auto it = sCodecTraits.find(codecKey);
+ if (it != sCodecTraits.end()) {
+ registeredCodec = it->second;
+ }
+
+ if (registeredCodec == nullptr) {
+ // register the one that was passed to us
+ ALOGV("Creating entry for codec %s, mediaType %s, key %s", codecName, mediaType,
+ codecKey.c_str());
+ sCodecTraits.insert({codecKey, codec});
+ registeredCodec = codec;
+ codec->setRegistered(true);
+ } else {
+ // one has already been registered, use that
+ // and discard the candidate
+ delete codec;
+ codec = nullptr;
+ }
+
+ return registeredCodec;
+}
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/VQApply.cpp b/media/libmediaformatshaper/VQApply.cpp
new file mode 100644
index 0000000..6f6f33c
--- /dev/null
+++ b/media/libmediaformatshaper/VQApply.cpp
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "VQApply"
+#include <utils/Log.h>
+
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+
+#include <media/formatshaper/VQops.h>
+#include <media/formatshaper/CodecProperties.h>
+#include <media/formatshaper/VideoShaper.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+
+// these are all NDK#31 and we run as NDK#29 (to be within the module)
+// the __builtin_available(android 31, *) constructs didn't work for me.
+//
+#define AMEDIAFORMAT_VIDEO_QP_MAX "video-qp-max"
+#define AMEDIAFORMAT_VIDEO_QP_MIN "video-qp-min"
+
+#define AMEDIAFORMAT_VIDEO_QP_B_MAX "video-qp-b-max"
+#define AMEDIAFORMAT_VIDEO_QP_B_MIN "video-qp-b-min"
+#define AMEDIAFORMAT_VIDEO_QP_I_MAX "video-qp-i-max"
+#define AMEDIAFORMAT_VIDEO_QP_I_MIN "video-qp-i-min"
+#define AMEDIAFORMAT_VIDEO_QP_P_MAX "video-qp-p-max"
+#define AMEDIAFORMAT_VIDEO_QP_P_MIN "video-qp-p-min"
+
+//
+// Caller retains ownership of and responsibility for inFormat
+//
+int VQApply(CodecProperties *codec, vqOps_t *info, AMediaFormat* inFormat, int flags) {
+ ALOGV("codecName %s inFormat %p flags x%x", codec->getName().c_str(), inFormat, flags);
+
+ if (codec->supportedMinimumQuality() > 0) {
+ // allow the codec provided minimum quality behavior to work at it
+ ALOGD("minquality(codec): codec says %d", codec->supportedMinimumQuality());
+ return 0;
+ }
+
+ ALOGD("considering other ways to improve quality...");
+
+ //
+ // apply any and all tools that we have.
+ // -- qp
+ // -- minimum bits-per-pixel
+ //
+ if (codec->supportsQp()) {
+ // use a (configurable) QP value to force better quality
+ //
+ // XXX: augment this so that we don't lower an existing QP setting
+ // (e.g. if user set it to 40, we don't want to set it back to 45)
+ int qpmax = codec->targetQpMax();
+ if (qpmax <= 0) {
+ qpmax = 45;
+ ALOGD("use default substitute QpMax == %d", qpmax);
+ }
+ ALOGD("minquality by QP: inject %s=%d", AMEDIAFORMAT_VIDEO_QP_MAX, qpmax);
+ AMediaFormat_setInt32(inFormat, AMEDIAFORMAT_VIDEO_QP_MAX, qpmax);
+
+ // force spreading the QP across frame types, since we imposing a value
+ qpSpreadMaxPerFrameType(inFormat, info->qpDelta, info->qpMax, /* override */ true);
+ } else {
+ ALOGD("codec %s: no qp bounding", codec->getName().c_str());
+ }
+
+ double bpp = codec->getBpp();
+ if (bpp > 0.0) {
+ // if we've decided to use bits-per-pixel (per second) to drive the quality
+ //
+ // (properly phrased as 'bits per second per pixel' so that it's resolution
+ // and framerate agnostic
+ //
+ // all of these is structured so that a missing value cleanly gets us to a
+ // non-faulting value of '0' for the minimum bits-per-pixel.
+ //
+ int32_t width = 0;
+ (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_WIDTH, &width);
+ int32_t height = 0;
+ (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_HEIGHT, &height);
+ int32_t bitrateConfigured = 0;
+ (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_BIT_RATE, &bitrateConfigured);
+
+ int64_t pixels = ((int64_t)width) * height;
+ int64_t bitrateFloor = pixels * bpp;
+
+ if (bitrateFloor > INT32_MAX) bitrateFloor = INT32_MAX;
+
+ ALOGD("minquality/bitrate: target %d floor %" PRId64 "(%.3f bpp * (%d w * %d h)",
+ bitrateConfigured, bitrateFloor, codec->getBpp(), height, width);
+
+ if (bitrateConfigured < bitrateFloor) {
+ ALOGD("minquality/target bitrate raised from %d to %" PRId64 " to maintain quality",
+ bitrateConfigured, bitrateFloor);
+ AMediaFormat_setInt32(inFormat, AMEDIAFORMAT_KEY_BIT_RATE, (int32_t)bitrateFloor);
+ }
+ }
+
+ return 0;
+}
+
+
+bool hasQpPerFrameType(AMediaFormat *format) {
+ int32_t value;
+
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &value)) {
+ return true;
+ }
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &value)) {
+ return true;
+ }
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &value)) {
+ return true;
+ }
+ return false;
+}
+
+bool hasQp(AMediaFormat *format) {
+ int32_t value;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MIN, &value)) {
+ return true;
+ }
+ return hasQpPerFrameType(format);
+}
+
+void qpSpreadPerFrameType(AMediaFormat *format, int delta,
+ int qplow, int qphigh, bool override) {
+ qpSpreadMaxPerFrameType(format, delta, qphigh, override);
+ qpSpreadMinPerFrameType(format, qplow, override);
+}
+
+void qpSpreadMaxPerFrameType(AMediaFormat *format, int delta, int qphigh, bool override) {
+ ALOGV("format %p delta %d hi %d override %d", format, delta, qphigh, override);
+
+ int32_t qpOffered = 0;
+ if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MAX, &qpOffered)) {
+ // propagate to otherwise unspecified frame-specific keys
+ int32_t maxI;
+ if (override || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &maxI)) {
+ int32_t value = std::min(qphigh, qpOffered);
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, value);
+ }
+ int32_t maxP;
+ if (override || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &maxP)) {
+ int32_t value = std::min(qphigh, (std::min(qpOffered, INT32_MAX-delta) + delta));
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, value);
+ }
+ int32_t maxB;
+ if (override || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &maxB)) {
+ int32_t value = std::min(qphigh, (std::min(qpOffered, INT32_MAX-2*delta) + 2*delta));
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, value);
+ }
+ }
+}
+
+void qpSpreadMinPerFrameType(AMediaFormat *format, int qplow, bool override) {
+ ALOGV("format %p lo %d override %d", format, qplow, override);
+
+ int32_t qpOffered = 0;
+ if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MIN, &qpOffered)) {
+ int value = std::max(qplow, qpOffered);
+ // propagate to otherwise unspecified frame-specific keys
+ int32_t minI;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &minI)) {
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, value);
+ }
+ int32_t minP;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &minP)) {
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, value);
+ }
+ int32_t minB;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &minB)) {
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, value);
+ }
+ }
+}
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/VideoShaper.cpp b/media/libmediaformatshaper/VideoShaper.cpp
new file mode 100644
index 0000000..fecd3a1
--- /dev/null
+++ b/media/libmediaformatshaper/VideoShaper.cpp
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "VideoShaper"
+#include <utils/Log.h>
+
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+
+#include <media/formatshaper/VQops.h>
+#include <media/formatshaper/CodecProperties.h>
+#include <media/formatshaper/VideoShaper.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+// mediatype-specific operations
+
+vqOps_t mediaInfo[] = {
+ {
+ .mediaType = "video/avc",
+ .qpMin = 0,
+ .qpMax = 51,
+ .qpDelta = 3,
+ },
+ {
+ .mediaType = "video/hevc",
+ .qpMin = 0,
+ .qpMax = 51,
+ .qpDelta = 3,
+ },
+ {
+ .mediaType = NULL, // matches everything, it must come last
+ .qpMin = INT32_MIN,
+ .qpMax = INT32_MAX,
+ .qpDelta = 3,
+ }
+};
+int nMediaInfos = sizeof(mediaInfo) / sizeof(mediaInfo[0]);
+
+//
+// Caller retains ownership of and responsibility for inFormat
+//
+
+int videoShaper(CodecProperties *codec, AMediaFormat* inFormat, int flags) {
+ if (codec == nullptr) {
+ return -1;
+ }
+ ALOGV("codec %s inFormat %p flags x%x", codec->getName().c_str(), inFormat, flags);
+
+ int ix;
+
+ std::string mediaType = codec->getMediaType();
+ // we should always come out of this with a selection, because the final entry
+ // is deliberaly a NULL -- so that it will act as a default
+ for(ix = 0; mediaInfo[ix].mediaType != NULL; ix++) {
+ if (strcmp(mediaType.c_str(), mediaInfo[ix].mediaType) == 0) {
+ break;
+ }
+ }
+ if (ix >= nMediaInfos) {
+ // shouldn't happen, but if it does .....
+ }
+
+ vqOps_t *info = &mediaInfo[ix];
+
+ // apply any quality transforms in here..
+ (void) VQApply(codec, info, inFormat, flags);
+
+ // We must always spread and map any QP parameters.
+ // Sometimes it's something we inserted here, sometimes it's a value that the user injected.
+ //
+ qpSpreadPerFrameType(inFormat, info->qpDelta, info->qpMin, info->qpMax, /* override */ false);
+
+ //
+ return 0;
+
+}
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/exports.lds b/media/libmediaformatshaper/exports.lds
new file mode 100644
index 0000000..a29cadb
--- /dev/null
+++ b/media/libmediaformatshaper/exports.lds
@@ -0,0 +1,6 @@
+{
+ global:
+ shaper_ops;
+ local:
+ *;
+};
diff --git a/media/libmediaformatshaper/include/media/formatshaper/CodecProperties.h b/media/libmediaformatshaper/include/media/formatshaper/CodecProperties.h
new file mode 100644
index 0000000..f7177a4
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/CodecProperties.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _LIBMEDIAFORMATSHAPER_CODECPROPERTIES_H_
+#define _LIBMEDIAFORMATSHAPER_CODECPROPERTIES_H_
+
+#include <map>
+#include <mutex>
+#include <string>
+
+#include <utils/RefBase.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+class CodecProperties {
+
+ public:
+ CodecProperties(std::string name, std::string mediaType);
+
+ std::string getName();
+ std::string getMediaType();
+
+ // establish a mapping from standard 'key' to non-standard 'value' in the namespace 'kind'
+ void setMapping(std::string kind, std::string key, std::string value);
+
+ // translate from from standard key to non-standard key
+ // return original standard key if there is no mapping
+ std::string getMapping(std::string key, std::string kind);
+
+ // returns an array of char *, which are paired "from" and "to" values
+ // for mapping (or unmapping). it's always expressed as from->to
+ // and 'reverse' describes which strings are to be on which side.
+ const char **getMappings(std::string kind, bool reverse);
+
+ // debugging of what's in the mapping dictionary
+ void showMappings();
+
+ // does the codec support the Android S minimum quality rules
+ void setSupportedMinimumQuality(int vmaf);
+ int supportedMinimumQuality();
+
+ // qp max bound used to compensate when SupportedMinimumQuality == 0
+ // 0 == let a system default handle it
+ void setTargetQpMax(int qpmax);
+ int targetQpMax();
+
+ // target bits-per-pixel (per second) for encoding operations.
+ // This is used to calculate a minimum bitrate for any particular resolution.
+ // A 1080p (1920*1080 = 2073600 pixels) to be encoded at 5Mbps has a bpp == 2.41
+ void setBpp(double bpp) { mBpp = bpp;}
+ double getBpp() {return mBpp;}
+
+ // Does this codec support QP bounding
+ // The getMapping() methods provide any needed mapping to non-standard keys.
+ void setSupportsQp(bool supported) { mSupportsQp = supported;}
+ bool supportsQp() { return mSupportsQp;}
+
+ int supportedApi();
+
+ // a codec is not usable until it has been registered with its
+ // name/mediaType.
+ bool isRegistered() { return mIsRegistered;}
+ void setRegistered(bool registered) { mIsRegistered = registered;}
+
+ private:
+ std::string mName;
+ std::string mMediaType;
+ int mApi = 0;
+ int mMinimumQuality = 0;
+ int mTargetQpMax = 0;
+ bool mSupportsQp = false;
+ double mBpp = 0.0;
+
+ std::mutex mMappingLock;
+ // XXX figure out why I'm having problems getting compiler to like GUARDED_BY
+ std::map<std::string, std::string> mMappings /*GUARDED_BY(mMappingLock)*/ ;
+ std::map<std::string, std::string> mUnMappings /*GUARDED_BY(mMappingLock)*/ ;
+
+ bool mIsRegistered = false;
+
+ // DISALLOW_EVIL_CONSTRUCTORS(CodecProperties);
+};
+
+extern CodecProperties *findCodec(const char *codecName, const char *mediaType);
+extern CodecProperties *registerCodec(CodecProperties *codec, const char *codecName,
+ const char *mediaType);
+
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // _LIBMEDIAFORMATSHAPER_CODECPROPERTIES_H_
diff --git a/media/libmediaformatshaper/include/media/formatshaper/FormatShaper.h b/media/libmediaformatshaper/include/media/formatshaper/FormatShaper.h
new file mode 100644
index 0000000..8ad81cd
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/FormatShaper.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * structure defining the function pointers that system-side folks
+ * use to invoke operations within the MediaFormat shaping library
+ *
+ * This is the include file the outside world uses.
+ */
+
+#ifndef LIBMEDIAFORMATSHAPER_FORMATSHAPER_H_
+#define LIBMEDIAFORMATSHAPER_FORMATSHAPER_H_
+
+namespace android {
+namespace mediaformatshaper {
+
+/*
+ * An opaque handle clients use to refer to codec+mediatype being shaped.
+ */
+typedef void (*shaperHandle_t);
+
+/*
+ * shapeFormat applies any re-shaping on the passed AMediaFormat.
+ * The updated format is returned in-place.
+ */
+typedef int (*shapeFormat_t)(shaperHandle_t shaperHandle,
+ AMediaFormat* inFormat, int flags);
+
+/*
+ * getMapping returns any mappings from standard keys to codec-specific keys.
+ * The return is a vector of const char* which are set up in pairs
+ * of "from", and "to".
+ * This array is always finished with a pair of nulls (to indicate a null from
+ * and a null to)
+ */
+
+typedef const char **(*getMappings_t)(shaperHandle_t shaperHandle, const char *kind);
+
+/*
+ * Returns a handle to the shaperHandle for the specified codec and mediatype.
+ * If none exists, it returns null.
+ */
+typedef shaperHandle_t (*findShaper_t)(const char *codecName, const char *mediaType);
+
+/*
+ * Creates and returns an empty shaperHandle that the client can populate using the
+ * setFeature() and setMap() operations.
+ */
+typedef shaperHandle_t (*createShaper_t)(const char *codecName, const char *mediaType);
+
+/*
+ * Registers the indicated shaperHandle for the indicated codec and mediatype.
+ * This call returns the shaperHandle that is to be used for further shaper operations.
+ * The returned value may be different than the one passed as an argument if another
+ * shaperinfo was registered while the passed one was being configured.
+ */
+typedef shaperHandle_t (*registerShaper_t)(shaperHandle_t shaper, const char *codecName,
+ const char *mediaType);
+
+/*
+ * establishes a mapping between the standard key "from" and the codec-specific key "to"
+ * in the "kind" namespace. This mapping is specific to the indicated codecName when
+ * encoding for the indicated mediaType.
+ */
+typedef int (*setMap_t)(shaperHandle_t shaper, const char *kind, const char *from, const char *to);
+
+/*
+ * establishes that codec "codecName" encoding for "mediaType" supports the indicated
+ * feature at the indicated value
+ */
+typedef int (*setFeature_t)(shaperHandle_t shaper, const char *feature, int value);
+
+/*
+ * The expectation is that the client will implement a flow similar to the following when
+ * setting up an encoding.
+ *
+ * if ((shaper=formatShaperops->findShaper(codecName, mediaType)) == NULL) {
+ * for (all codec features) {
+ * get feature name, feature value
+ * formatShaperops->setFeature(shaper,, featurename, featurevalue)
+ * }
+ * for (all codec mappings) {
+ * get mapping 'kind', mapping 'from', mapping 'to'
+ * formatShaperops->setMap(shaper, kind, from, to)
+ * }
+ * }
+ *
+ */
+
+typedef struct FormatShaperOps {
+ const uint32_t version;
+
+ /*
+ * find, create, setup, and register the shaper info
+ */
+ findShaper_t findShaper;
+ createShaper_t createShaper;
+ setMap_t setMap;
+ setFeature_t setFeature;
+ registerShaper_t registerShaper;
+
+ /*
+ * use the shaper info
+ */
+ shapeFormat_t shapeFormat;
+ getMappings_t getMappings;
+ getMappings_t getReverseMappings;
+} FormatShaperOps_t;
+
+// versioninf information
+const uint32_t SHAPER_VERSION_UNKNOWN = 0;
+const uint32_t SHAPER_VERSION_V1 = 1;
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // LIBMEDIAFORMATSHAPER_FORMATSHAPER_H_
diff --git a/media/libmediaformatshaper/include/media/formatshaper/VQops.h b/media/libmediaformatshaper/include/media/formatshaper/VQops.h
new file mode 100644
index 0000000..807e8af
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/VQops.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBMEDIAFORMATSHAPER_VQOPS_H_
+#define LIBMEDIAFORMATSHAPER_VQOPS_H_
+
+#include <media/formatshaper/CodecProperties.h>
+#include <media/NdkMediaFormat.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+// parameterized info for the different mediaType types
+typedef struct {
+ const char *mediaType;
+
+ int32_t qpMin; // codec type limit (e.g. h264, not c2.android.avc.encoder)
+ int32_t qpMax;
+ int32_t qpDelta; // from I to P to B
+
+} vqOps_t;
+
+int VQApply(CodecProperties *codec, vqOps_t *info, AMediaFormat* inFormat, int flags);
+
+// spread the overall QP setting to any un-set per-frame-type settings
+void qpSpreadPerFrameType(AMediaFormat *format, int delta, int qplow, int qphigh, bool override);
+void qpSpreadMaxPerFrameType(AMediaFormat *format, int delta, int qphigh, bool override);
+void qpSpreadMinPerFrameType(AMediaFormat *format, int qplow, bool override);
+
+// does the format have QP bounding entries
+bool hasQp(AMediaFormat *format);
+bool hasQpPerFrameType(AMediaFormat *format);
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // LIBMEDIAFORMATSHAPER_VQOPS_H_
diff --git a/media/libmediaformatshaper/include/media/formatshaper/VideoShaper.h b/media/libmediaformatshaper/include/media/formatshaper/VideoShaper.h
new file mode 100644
index 0000000..53f1b13
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/VideoShaper.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBMEDIAFORMATSHAPER_VIDEOSHAPER_H_
+#define LIBMEDIAFORMATSHAPER_VIDEOSHAPER_H_
+
+namespace android {
+namespace mediaformatshaper {
+
+/*
+ * runs through video-specific shaping operations for the codec/format combination.
+ * updates inFormat in place.
+ */
+int videoShaper(CodecProperties *codec, AMediaFormat* inFormat, int flags);
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // LIBMEDIAFORMATSHAPER_VIDEOSHAPER_H_
diff --git a/media/libstagefright/Android.bp b/media/libstagefright/Android.bp
index 930bc0f..52434b3 100644
--- a/media/libstagefright/Android.bp
+++ b/media/libstagefright/Android.bp
@@ -352,6 +352,8 @@
"libwebm",
"libstagefright_id3",
"media_permission-aidl-cpp",
+ "libmediandk_format",
+ "libmedia_ndkformatpriv",
],
header_libs:[
@@ -359,6 +361,7 @@
"libnativeloader-headers",
"libstagefright_xmlparser_headers",
"media_ndk_headers",
+ "libmediaformatshaper_headers",
],
export_shared_lib_headers: [
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index f80b22f..755986c 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -20,9 +20,11 @@
#include <utils/Log.h>
#include <set>
+#include <stdlib.h>
#include <inttypes.h>
#include <stdlib.h>
+#include <dlfcn.h>
#include <C2Buffer.h>
@@ -35,6 +37,7 @@
#include <aidl/android/media/IResourceManagerService.h>
#include <android/binder_ibinder.h>
#include <android/binder_manager.h>
+#include <android/dlext.h>
#include <binder/IMemory.h>
#include <binder/MemoryDealer.h>
#include <cutils/properties.h>
@@ -47,6 +50,10 @@
#include <media/MediaCodecInfo.h>
#include <media/MediaMetricsItem.h>
#include <media/MediaResource.h>
+#include <media/NdkMediaErrorPriv.h>
+#include <media/NdkMediaFormat.h>
+#include <media/NdkMediaFormatPriv.h>
+#include <media/formatshaper/FormatShaper.h>
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
@@ -68,6 +75,7 @@
#include <media/stagefright/OMXClient.h>
#include <media/stagefright/PersistentSurface.h>
#include <media/stagefright/SurfaceUtils.h>
+#include <nativeloader/dlext_namespaces.h>
#include <private/android_filesystem_config.h>
#include <utils/Singleton.h>
@@ -1326,6 +1334,9 @@
return msg->post();
}
+static void mapFormat(AString componentName, const sp<AMessage> &format, const char *kind,
+ bool reverse);
+
status_t MediaCodec::configure(
const sp<AMessage> &format,
const sp<Surface> &nativeWindow,
@@ -1382,6 +1393,7 @@
ALOGE("Invalid size(s), width=%d, height=%d", mVideoWidth, mVideoHeight);
return BAD_VALUE;
}
+
} else {
if (mMetricsHandle != 0) {
int32_t channelCount;
@@ -1395,6 +1407,19 @@
}
}
+ // apply framework level modifications to the mediaformat for encoding
+ // XXX: default off for a while during dogfooding
+ static const char *enable_property = "debug.stagefright.enableshaping";
+ int8_t enableShaping = property_get_bool(enable_property, 0);
+ if (!enableShaping) {
+ ALOGD("format shaping disabled via property '%s'", enable_property);
+ } else {
+ if (flags & CONFIGURE_FLAG_ENCODE) {
+ (void) shapeMediaFormat(format, flags);
+ mapFormat(mComponentName, format, nullptr, false);
+ }
+ }
+
updateLowLatency(format);
msg->setMessage("format", format);
@@ -1464,6 +1489,316 @@
return err;
}
+// Media Format Shaping support
+//
+
+static android::mediaformatshaper::FormatShaperOps_t *sShaperOps = NULL;
+
+static bool connectFormatShaper() {
+ static std::once_flag sCheckOnce;
+ static void *libHandle = NULL;
+
+ std::call_once(sCheckOnce, [&](){
+
+ // prefer any copy in the mainline module
+ //
+ android_namespace_t *mediaNs = android_get_exported_namespace("com_android_media");
+ AString libraryName = "libmediaformatshaper.so";
+
+ if (mediaNs != NULL) {
+ static const android_dlextinfo dlextinfo = {
+ .flags = ANDROID_DLEXT_USE_NAMESPACE,
+ .library_namespace = mediaNs,
+ };
+
+ AString libraryMainline = "/apex/com.android.media/";
+#if __LP64__
+ libraryMainline.append("lib64/");
+#else
+ libraryMainline.append("lib/");
+#endif
+ libraryMainline.append(libraryName);
+
+ libHandle = android_dlopen_ext(libraryMainline.c_str(), RTLD_NOW|RTLD_NODELETE,
+ &dlextinfo);
+
+ if (libHandle != NULL) {
+ sShaperOps = (android::mediaformatshaper::FormatShaperOps_t*)
+ dlsym(libHandle, "shaper_ops");
+ } else {
+ ALOGW("connectFormatShaper: unable to load mainline formatshaper %s",
+ libraryMainline.c_str());
+ }
+ } else {
+ ALOGV("connectFormatShaper: couldn't find media namespace.");
+ }
+
+ // fall back to the system partition, if present.
+ //
+ if (sShaperOps == NULL) {
+
+ libHandle = dlopen(libraryName.c_str(), RTLD_NOW|RTLD_NODELETE);
+
+ if (libHandle != NULL) {
+ sShaperOps = (android::mediaformatshaper::FormatShaperOps_t*)
+ dlsym(libHandle, "shaper_ops");
+ } else {
+ ALOGW("connectFormatShaper: unable to load formatshaper %s", libraryName.c_str());
+ }
+ }
+
+ if (sShaperOps != nullptr
+ && sShaperOps->version != android::mediaformatshaper::SHAPER_VERSION_V1) {
+ ALOGW("connectFormatShaper: unhandled version ShaperOps: %d, DISABLED",
+ sShaperOps->version);
+ sShaperOps = nullptr;
+ }
+
+ if (sShaperOps != nullptr) {
+ ALOGI("connectFormatShaper: connected to library %s", libraryName.c_str());
+ }
+
+ });
+
+ return true;
+}
+
+// a construct to force the above dlopen() to run very early.
+// goal: so the dlopen() doesn't happen on critical path of latency sensitive apps
+// failure of this means that cold start of those apps is slower by the time to dlopen()
+//
+static bool forceEarlyLoadingShaper = connectFormatShaper();
+
+// parse the codec's properties: mapping, whether it meets min quality, etc
+// and pass them into the video quality code
+//
+status_t MediaCodec::setupFormatShaper(AString mediaType) {
+ ALOGV("setupFormatShaper: initializing shaper data for codec %s mediaType %s",
+ mComponentName.c_str(), mediaType.c_str());
+
+ nsecs_t mapping_started = systemTime(SYSTEM_TIME_MONOTONIC);
+
+ // see if the shaper is already present, if so return
+ mediaformatshaper::shaperHandle_t shaperHandle;
+ shaperHandle = sShaperOps->findShaper(mComponentName.c_str(), mediaType.c_str());
+ if (shaperHandle != nullptr) {
+ ALOGV("shaperhandle %p -- no initialization needed", shaperHandle);
+ return OK;
+ }
+
+ // not there, so we get to build & register one
+ shaperHandle = sShaperOps->createShaper(mComponentName.c_str(), mediaType.c_str());
+ if (shaperHandle == nullptr) {
+ ALOGW("unable to create a shaper for cocodec %s mediaType %s",
+ mComponentName.c_str(), mediaType.c_str());
+ return OK;
+ }
+
+ sp<MediaCodecInfo::Capabilities> capabilities =
+ mCodecInfo->getCapabilitiesFor(mediaType.c_str());
+ if (capabilities == nullptr) {
+ ALOGI("no capabilities as part of the codec?");
+ } else {
+ const sp<AMessage> &details = capabilities->getDetails();
+ AString mapTarget;
+ int count = details->countEntries();
+ for(int ix = 0; ix < count; ix++) {
+ AMessage::Type entryType;
+ const char *mapSrc = details->getEntryNameAt(ix, &entryType);
+ // XXX: re-use ix from getEntryAt() to avoid additional findXXX() invocation
+ //
+ static const char *featurePrefix = "feature-";
+ static const int featurePrefixLen = strlen(featurePrefix);
+ static const char *mappingPrefix = "mapping-";
+ static const int mappingPrefixLen = strlen(mappingPrefix);
+
+ if (mapSrc == NULL) {
+ continue;
+ } else if (!strncmp(mapSrc, featurePrefix, featurePrefixLen)) {
+ int32_t intValue;
+ if (details->findInt32(mapSrc, &intValue)) {
+ ALOGV("-- feature '%s' -> %d", mapSrc, intValue);
+ (void)(sShaperOps->setFeature)(shaperHandle, &mapSrc[featurePrefixLen],
+ intValue);
+ }
+ continue;
+ } else if (!strncmp(mapSrc, mappingPrefix, mappingPrefixLen)) {
+ AString target;
+ if (details->findString(mapSrc, &target)) {
+ ALOGV("-- mapping %s: map %s to %s", mapSrc, &mapSrc[mappingPrefixLen],
+ target.c_str());
+ // key is really "kind-key"
+ // separate that, so setMap() sees the triple kind, key, value
+ const char *kind = &mapSrc[mappingPrefixLen];
+ const char *sep = strchr(kind, '-');
+ const char *key = sep+1;
+ if (sep != NULL) {
+ std::string xkind = std::string(kind, sep-kind);
+ (void)(sShaperOps->setMap)(shaperHandle, xkind.c_str(),
+ key, target.c_str());
+ }
+ }
+ }
+ }
+ }
+ shaperHandle = sShaperOps->registerShaper(shaperHandle,
+ mComponentName.c_str(), mediaType.c_str());
+
+ nsecs_t mapping_finished = systemTime(SYSTEM_TIME_MONOTONIC);
+ ALOGD("setupFormatShaper: populated shaper node for codec %s: %" PRId64 " us",
+ mComponentName.c_str(), (mapping_finished - mapping_started)/1000);
+
+ return OK;
+}
+
+
+// Format Shaping
+// Mapping and Manipulation of encoding parameters
+//
+
+status_t MediaCodec::shapeMediaFormat(
+ const sp<AMessage> &format,
+ uint32_t flags) {
+ ALOGV("shapeMediaFormat entry");
+
+ if (!(flags & CONFIGURE_FLAG_ENCODE)) {
+ ALOGW("shapeMediaFormat: not encoder");
+ return OK;
+ }
+ if (mCodecInfo == NULL) {
+ ALOGW("shapeMediaFormat: no codecinfo");
+ return OK;
+ }
+
+ AString mediaType;
+ if (!format->findString("mime", &mediaType)) {
+ ALOGW("shapeMediaFormat: no mediaType information");
+ return OK;
+ }
+
+ // make sure we have the function entry points for the shaper library
+ //
+
+ connectFormatShaper();
+ if (sShaperOps == nullptr) {
+ ALOGW("shapeMediaFormat: no MediaFormatShaper hooks available");
+ return OK;
+ }
+
+ // find the shaper information for this codec+mediaType pair
+ //
+ mediaformatshaper::shaperHandle_t shaperHandle;
+ shaperHandle = sShaperOps->findShaper(mComponentName.c_str(), mediaType.c_str());
+ if (shaperHandle == nullptr) {
+ setupFormatShaper(mediaType);
+ shaperHandle = sShaperOps->findShaper(mComponentName.c_str(), mediaType.c_str());
+ }
+ if (shaperHandle == nullptr) {
+ ALOGW("shapeMediaFormat: no handler for codec %s mediatype %s",
+ mComponentName.c_str(), mediaType.c_str());
+ return OK;
+ }
+
+ // run the shaper
+ //
+
+ ALOGV("Shaping input: %s", format->debugString(0).c_str());
+
+ sp<AMessage> updatedFormat = format->dup();
+ AMediaFormat *updatedNdkFormat = AMediaFormat_fromMsg(&updatedFormat);
+
+ int result = (*sShaperOps->shapeFormat)(shaperHandle, updatedNdkFormat, flags);
+ if (result == 0) {
+ AMediaFormat_getFormat(updatedNdkFormat, &updatedFormat);
+
+ sp<AMessage> deltas = updatedFormat->changesFrom(format, false /* deep */);
+ ALOGD("shapeMediaFormat: deltas: %s", deltas->debugString(2).c_str());
+
+ // note that this means that for anything in both, the copy in deltas wins
+ format->extend(deltas);
+ }
+
+ AMediaFormat_delete(updatedNdkFormat);
+ return OK;
+}
+
+static void mapFormat(AString componentName, const sp<AMessage> &format, const char *kind,
+ bool reverse) {
+ AString mediaType;
+ if (!format->findString("mime", &mediaType)) {
+ ALOGW("mapFormat: no mediaType information");
+ return;
+ }
+ ALOGV("mapFormat: codec %s mediatype %s kind %s reverse %d", componentName.c_str(),
+ mediaType.c_str(), kind ? kind : "<all>", reverse);
+
+ // make sure we have the function entry points for the shaper library
+ //
+
+ connectFormatShaper();
+ if (sShaperOps == nullptr) {
+ ALOGW("mapFormat: no MediaFormatShaper hooks available");
+ return;
+ }
+
+ // find the shaper information for this codec+mediaType pair
+ //
+ mediaformatshaper::shaperHandle_t shaperHandle;
+ shaperHandle = sShaperOps->findShaper(componentName.c_str(), mediaType.c_str());
+ if (shaperHandle == nullptr) {
+ ALOGW("mapFormat: no shaper handle");
+ return;
+ }
+
+ const char **mappings;
+ if (reverse)
+ mappings = sShaperOps->getReverseMappings(shaperHandle, kind);
+ else
+ mappings = sShaperOps->getMappings(shaperHandle, kind);
+
+ if (mappings == nullptr) {
+ ALOGV("no mappings returned");
+ return;
+ }
+
+ ALOGV("Pre-mapping: %s", format->debugString(2).c_str());
+ // do the mapping
+ //
+ int entries = format->countEntries();
+ for (int i = 0; ; i += 2) {
+ if (mappings[i] == nullptr) {
+ break;
+ }
+
+ size_t ix = format->findEntryByName(mappings[i]);
+ if (ix < entries) {
+ ALOGV("map '%s' to '%s'", mappings[i], mappings[i+1]);
+ status_t status = format->setEntryNameAt(ix, mappings[i+1]);
+ if (status != OK) {
+ ALOGW("Unable to map from '%s' to '%s': status %d",
+ mappings[i], mappings[i+1], status);
+ }
+ }
+ }
+ ALOGV("Post-mapping: %s", format->debugString(2).c_str());
+
+
+ // reclaim the mapping memory
+ for (int i = 0; ; i += 2) {
+ if (mappings[i] == nullptr) {
+ break;
+ }
+ free((void*)mappings[i]);
+ free((void*)mappings[i + 1]);
+ }
+ free(mappings);
+ mappings = nullptr;
+}
+
+//
+// end of Format Shaping hooks within MediaCodec
+//
+
status_t MediaCodec::releaseCrypto()
{
ALOGV("releaseCrypto");
@@ -3691,6 +4026,7 @@
buffer->meta()->setObject("changedKeys", changedKeys);
}
mOutputFormat = format;
+ mapFormat(mComponentName, format, nullptr, true);
ALOGV("[%s] output format changed to: %s",
mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
@@ -4469,6 +4805,7 @@
status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) {
updateLowLatency(params);
+ mapFormat(mComponentName, params, nullptr, false);
mCodec->signalSetParameters(params);
return OK;
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 5f64686..9b7da3a 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -408,6 +408,17 @@
// configure parameter
sp<AMessage> mConfigureMsg;
+ // rewrites the format description during configure() for encoding.
+ // format and flags as they exist within configure()
+ // the (possibly) updated format is returned in place.
+ status_t shapeMediaFormat(
+ const sp<AMessage> &format,
+ uint32_t flags);
+
+ // populate the format shaper library with information for this codec encoding
+ // for the indicated media type
+ status_t setupFormatShaper(AString mediaType);
+
// Used only to synchronize asynchronous getBufferAndFormat
// across all the other (synchronous) buffer state change
// operations, such as de/queueIn/OutputBuffer, start and