Define dvrTrackingFeatureExtractor API
Bug: 78173557
Bug: 78169521
Test: dvr_tracking-test
Change-Id: I975989d9e7bd86dff10ef4300d6f0a4d3117caaa
diff --git a/libs/vr/libdvr/dvr_tracking.cpp b/libs/vr/libdvr/dvr_tracking.cpp
index 77a9438..73addc9 100644
--- a/libs/vr/libdvr/dvr_tracking.cpp
+++ b/libs/vr/libdvr/dvr_tracking.cpp
@@ -29,6 +29,34 @@
return -ENOSYS;
}
+int dvrTrackingFeatureExtractorCreate(DvrTrackingFeatureExtractor**) {
+ ALOGE("dvrTrackingFeatureExtractorCreate is not implemented.");
+ return -ENOSYS;
+}
+
+void dvrTrackingFeatureExtractorDestroy(DvrTrackingFeatureExtractor*) {
+ ALOGE("dvrTrackingFeatureExtractorDestroy is not implemented.");
+}
+
+int dvrTrackingFeatureExtractorStart(DvrTrackingFeatureExtractor*,
+ DvrTrackingFeatureCallback, void*) {
+ ALOGE("dvrTrackingFeatureExtractorCreate is not implemented.");
+ return -ENOSYS;
+}
+
+int dvrTrackingFeatureExtractorStop(DvrTrackingFeatureExtractor*) {
+ ALOGE("dvrTrackingFeatureExtractorCreate is not implemented.");
+ return -ENOSYS;
+}
+
+int dvrTrackingFeatureExtractorProcessBuffer(DvrTrackingFeatureExtractor*,
+ DvrReadBuffer*,
+ const DvrTrackingBufferMetadata*,
+ bool*) {
+ ALOGE("dvrTrackingFeatureExtractorProcessBuffer is not implemented.");
+ return -ENOSYS;
+}
+
int dvrTrackingSensorsCreate(DvrTrackingSensors**, const char*) {
ALOGE("dvrTrackingSensorsCreate is not implemented.");
return -ENOSYS;
diff --git a/libs/vr/libdvr/include/dvr/dvr_api.h b/libs/vr/libdvr/include/dvr/dvr_api.h
index 1e532a6..024cae9 100644
--- a/libs/vr/libdvr/include/dvr/dvr_api.h
+++ b/libs/vr/libdvr/include/dvr/dvr_api.h
@@ -375,6 +375,19 @@
DvrWriteBufferQueue* write_queue);
typedef int (*DvrTrackingCameraStopPtr)(DvrTrackingCamera* camera);
+typedef int (*DvrTrackingFeatureExtractorCreatePtr)(
+ DvrTrackingFeatureExtractor** out_extractor);
+typedef void (*DvrTrackingFeatureExtractorDestroyPtr)(
+ DvrTrackingFeatureExtractor* extractor);
+typedef int (*DvrTrackingFeatureExtractorStartPtr)(
+ DvrTrackingFeatureExtractor* extractor,
+ DvrTrackingFeatureCallback callback, void* context);
+typedef int (*DvrTrackingFeatureExtractorStopPtr)(
+ DvrTrackingFeatureExtractor* extractor);
+typedef int (*DvrTrackingFeatureExtractorProcessBufferPtr)(
+ DvrTrackingFeatureExtractor* extractor, DvrReadBuffer* buffer,
+ const DvrTrackingBufferMetadata* metadata, bool* out_skipped);
+
typedef int (*DvrTrackingSensorsCreatePtr)(DvrTrackingSensors** out_sensors,
const char* mode);
typedef void (*DvrTrackingSensorsDestroyPtr)(DvrTrackingSensors* sensors);
diff --git a/libs/vr/libdvr/include/dvr/dvr_api_entries.h b/libs/vr/libdvr/include/dvr/dvr_api_entries.h
index 989980e..6620a37 100644
--- a/libs/vr/libdvr/include/dvr/dvr_api_entries.h
+++ b/libs/vr/libdvr/include/dvr/dvr_api_entries.h
@@ -187,6 +187,13 @@
DVR_V1_API_ENTRY(TrackingCameraDestroy);
DVR_V1_API_ENTRY(TrackingCameraStart);
DVR_V1_API_ENTRY(TrackingCameraStop);
+
+DVR_V1_API_ENTRY(TrackingFeatureExtractorCreate);
+DVR_V1_API_ENTRY(TrackingFeatureExtractorDestroy);
+DVR_V1_API_ENTRY(TrackingFeatureExtractorStart);
+DVR_V1_API_ENTRY(TrackingFeatureExtractorStop);
+DVR_V1_API_ENTRY(TrackingFeatureExtractorProcessBuffer);
+
DVR_V1_API_ENTRY(TrackingSensorsCreate);
DVR_V1_API_ENTRY(TrackingSensorsDestroy);
DVR_V1_API_ENTRY(TrackingSensorsStart);
diff --git a/libs/vr/libdvr/include/dvr/dvr_tracking.h b/libs/vr/libdvr/include/dvr/dvr_tracking.h
index ca97345..b5591a0 100644
--- a/libs/vr/libdvr/include/dvr/dvr_tracking.h
+++ b/libs/vr/libdvr/include/dvr/dvr_tracking.h
@@ -4,30 +4,21 @@
#include <stdint.h>
#include <sys/cdefs.h>
+#include <dvr/dvr_tracking_types.h>
+
__BEGIN_DECLS
-// Represents a sensor event.
-typedef struct DvrTrackingSensorEvent {
- // The sensor type.
- int32_t sensor;
-
- // Event type.
- int32_t type;
-
- // This is the timestamp recorded from the device. Taken in the middle
- // of the integration interval and adjusted for any low pass filtering.
- int64_t timestamp_ns;
-
- // The event data.
- float x;
- float y;
- float z;
-} DvrTrackingSensorEvent;
-
-typedef struct DvrTrackingSensors DvrTrackingSensors;
+typedef struct DvrReadBuffer DvrReadBuffer;
typedef struct DvrTrackingCamera DvrTrackingCamera;
+typedef struct DvrTrackingFeatureExtractor DvrTrackingFeatureExtractor;
+typedef struct DvrTrackingSensors DvrTrackingSensors;
typedef struct DvrWriteBufferQueue DvrWriteBufferQueue;
+// The callback for DvrTrackingFeatureExtractor that will deliver the feature
+// events. This callback is passed to dvrTrackingFeatureExtractorStart.
+typedef void (*DvrTrackingFeatureCallback)(void* context,
+ const DvrTrackingFeatures* event);
+
// The callback for DvrTrackingSensors session that will deliver the events.
// This callback is passed to dvrTrackingSensorsStart.
typedef void (*DvrTrackingSensorEventCallback)(void* context,
@@ -67,7 +58,8 @@
// @param write_queue A DvrWriteBufferQueue that the camera stack can use to
// populate the buffer into. The queue must be empty and the camera stack
// will request buffer allocation with proper buffer dimension, format, and
-// usage.
+// usage. Note that the write queue must be created with user_metadata_size
+// set to sizeof(DvrTrackingBufferMetadata).
// @return Zero on success, or negative error code.
int dvrTrackingCameraStart(DvrTrackingCamera* camera,
DvrWriteBufferQueue* write_queue);
@@ -96,12 +88,12 @@
int dvrTrackingSensorsCreate(DvrTrackingSensors** out_sensors,
const char* mode);
-// Destroy a DvrTrackingSensors session.
+// Destroys a DvrTrackingSensors session.
//
// @param sensors The DvrTrackingSensors struct to destroy.
void dvrTrackingSensorsDestroy(DvrTrackingSensors* sensors);
-// Start the tracking.
+// Starts the tracking sensor session.
//
// This will start the device sensors and start pumping the feature and sensor
// events as they arrive.
@@ -115,7 +107,7 @@
DvrTrackingSensorEventCallback callback,
void* context);
-// Stop the tracking.
+// Stops a DvrTrackingSensors session.
//
// This will stop the device sensors. dvrTrackingSensorsStart can be called to
// restart them again.
@@ -124,6 +116,67 @@
// @return Zero on success, or negative error code.
int dvrTrackingSensorsStop(DvrTrackingSensors* sensors);
+// Creates a tracking feature extractor.
+//
+// This will initialize but not start the feature extraction session. Upon
+// successful creation, the client can call dvrTrackingFeatureExtractorStart to
+// start receiving features.
+//
+// @param out_extractor The pointer of a DvrTrackingFeatureExtractor will be
+// filled here if the method call succeeds.
+int dvrTrackingFeatureExtractorCreate(
+ DvrTrackingFeatureExtractor** out_extractor);
+
+// Destroys a tracking feature extractor.
+//
+// @param extractor The DvrTrackingFeatureExtractor to destroy.
+void dvrTrackingFeatureExtractorDestroy(DvrTrackingFeatureExtractor* extractor);
+
+// Starts the tracking feature extractor.
+//
+// This will start the extractor and start pumping the output feature events to
+// the registered callback. Note that this method will create one or more
+// threads to handle feature processing.
+//
+// @param extractor The DvrTrackingFeatureExtractor to destroy.
+int dvrTrackingFeatureExtractorStart(DvrTrackingFeatureExtractor* extractor,
+ DvrTrackingFeatureCallback callback,
+ void* context);
+
+// Stops the tracking feature extractor.
+//
+// This will stop the extractor session and clean up all internal resourcse
+// related to this extractor. On succssful return, all internal therad started
+// by dvrTrackingFeatureExtractorStart should be stopped.
+//
+// @param extractor The DvrTrackingFeatureExtractor to destroy.
+int dvrTrackingFeatureExtractorStop(DvrTrackingFeatureExtractor* extractor);
+
+// Processes one buffer to extract features from.
+//
+// The buffer will be sent over to DSP for feature extraction. Once the process
+// is done, the processing thread will invoke DvrTrackingFeatureCallback with
+// newly extracted features. Note that not all buffers will be processed, as the
+// underlying DSP can only process buffers at a certain framerate. If a buffer
+// needs to be skipped, out_skipped filed will be set to true. Also note that
+// for successfully processed stereo buffer, two callbacks (one for each eye)
+// will be fired.
+//
+// @param extractor The DvrTrackingFeatureExtractor to destroy.
+// @param buffer The buffer to extract features from. Note that the buffer must
+// be in acquired state for the buffer to be processed. Also note that the
+// buffer will be released back to its producer on successful return of the
+// method.
+// @param metadata The metadata associated with the buffer. Should be populated
+// by DvrTrackingCamera session as user defined metadata.
+// @param out_skipped On successful return, the field will be set to true iff
+// the buffer was skipped; and false iff the buffer was processed. This
+// field is optional and nullptr can be passed here to ignore the field.
+// @return Zero on success, or negative error code.
+int dvrTrackingFeatureExtractorProcessBuffer(
+ DvrTrackingFeatureExtractor* extractor, DvrReadBuffer* buffer,
+ const DvrTrackingBufferMetadata* metadata, bool* out_skipped);
+
__END_DECLS
#endif // ANDROID_DVR_TRACKING_H_
diff --git a/libs/vr/libdvr/include/dvr/dvr_tracking_types.h b/libs/vr/libdvr/include/dvr/dvr_tracking_types.h
new file mode 100644
index 0000000..81310d2
--- /dev/null
+++ b/libs/vr/libdvr/include/dvr/dvr_tracking_types.h
@@ -0,0 +1,104 @@
+#ifndef ANDROID_DVR_TRACKING_TYPES_H_
+#define ANDROID_DVR_TRACKING_TYPES_H_
+
+#include <stdint.h>
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+
+typedef struct DvrTrackingBufferMetadata {
+ // Specifies the source of this image.
+ uint32_t camera_mask;
+ // Specifies the memory format of this image.
+ uint32_t format;
+ /// The width of the image data.
+ uint32_t width;
+ /// The height of the image data.
+ uint32_t height;
+ /// The number of bytes per scanline of image data.
+ uint32_t stride;
+ /// The frame number of this image.
+ int32_t frame_number;
+ /// The timestamp of this image in nanoseconds. Taken in the middle of the
+ /// exposure interval.
+ int64_t timestamp_ns;
+ // This is the timestamp for recording when the system using the HAL
+ // received the callback. It will not be populated by the HAL.
+ int64_t callback_timestamp_ns;
+ /// The exposure duration of this image in nanoseconds.
+ int64_t exposure_duration_ns;
+} DvrTrackingBufferMetadata;
+
+// Represents a set of features extracted from a camera frame. Note that this
+// should be in sync with TangoHalCallbacks defined in tango-hal.h.
+typedef struct DvrTrackingFeatures {
+ // Specifies the source of the features.
+ uint32_t camera_mask;
+
+ // This is unused.
+ uint32_t unused;
+
+ // The timestamp in nanoseconds from the image that generated the features.
+ // Taken in the middle of the exposure interval.
+ int64_t timestamp_ns;
+
+ // This is the timestamp for recording when the system using the HAL
+ // received the callback. It will not be populated by the HAL.
+ int64_t callback_timestamp_ns;
+
+ // The frame number from the image that generated the features.
+ int64_t frame_number;
+
+ // The number of features.
+ int count;
+
+ // An array of 2D image points for each feature in the current image.
+ // This is sub-pixel refined extremum location at the fine resolution.
+ float (*positions)[2];
+
+ // The id of these measurements.
+ int32_t* ids;
+
+ // The feature descriptors.
+ uint64_t (*descriptors)[8];
+
+ // Laplacian scores for each feature.
+ float* scores;
+
+ // Is this feature a minimum or maximum in the Laplacian image.
+ // 0 if the feature is a maximum, 1 if it is a minimum.
+ int32_t* is_minimum;
+
+ // This corresponds to the sub-pixel index of the laplacian image
+ // that the extremum was found.
+ float* scales;
+
+ // Computed orientation of keypoint as part of FREAK extraction, except
+ // it's represented in radians and measured anti-clockwise.
+ float* angles;
+
+ // Edge scores for each feature.
+ float* edge_scores;
+} DvrTrackingFeatures;
+
+// Represents a sensor event.
+typedef struct DvrTrackingSensorEvent {
+ // The sensor type.
+ int32_t sensor;
+
+ // Event type.
+ int32_t type;
+
+ // This is the timestamp recorded from the device. Taken in the middle
+ // of the integration interval and adjusted for any low pass filtering.
+ int64_t timestamp_ns;
+
+ // The event data.
+ float x;
+ float y;
+ float z;
+} DvrTrackingSensorEvent;
+
+__END_DECLS
+
+#endif // ANDROID_DVR_TRACKING_TYPES_H_
diff --git a/libs/vr/libdvr/tests/dvr_tracking-test.cpp b/libs/vr/libdvr/tests/dvr_tracking-test.cpp
index 6a52801..3b6d6e1 100644
--- a/libs/vr/libdvr/tests/dvr_tracking-test.cpp
+++ b/libs/vr/libdvr/tests/dvr_tracking-test.cpp
@@ -13,9 +13,15 @@
ASSERT_TRUE(api_.TrackingCameraCreate != nullptr);
ASSERT_TRUE(api_.TrackingCameraStart != nullptr);
ASSERT_TRUE(api_.TrackingCameraStop != nullptr);
+
+ ASSERT_TRUE(api_.TrackingFeatureExtractorCreate != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorDestroy != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorStart != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorStop != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorProcessBuffer != nullptr);
}
-TEST_F(DvrTrackingTest, CreateFailsForInvalidInput) {
+TEST_F(DvrTrackingTest, CameraCreateFailsForInvalidInput) {
int ret;
ret = api_.TrackingCameraCreate(nullptr);
EXPECT_EQ(ret, -EINVAL);
@@ -25,7 +31,7 @@
EXPECT_EQ(ret, -EINVAL);
}
-TEST_F(DvrTrackingTest, CreateDestroy) {
+TEST_F(DvrTrackingTest, CameraCreateDestroy) {
DvrTrackingCamera* camera = nullptr;
int ret = api_.TrackingCameraCreate(&camera);
@@ -35,6 +41,27 @@
api_.TrackingCameraDestroy(camera);
}
+TEST_F(DvrTrackingTest, FeatureExtractorCreateFailsForInvalidInput) {
+ int ret;
+ ret = api_.TrackingFeatureExtractorCreate(nullptr);
+ EXPECT_EQ(ret, -EINVAL);
+
+ DvrTrackingFeatureExtractor* camera =
+ reinterpret_cast<DvrTrackingFeatureExtractor*>(42);
+ ret = api_.TrackingFeatureExtractorCreate(&camera);
+ EXPECT_EQ(ret, -EINVAL);
+}
+
+TEST_F(DvrTrackingTest, FeatureExtractorCreateDestroy) {
+ DvrTrackingFeatureExtractor* camera = nullptr;
+ int ret = api_.TrackingFeatureExtractorCreate(&camera);
+
+ EXPECT_EQ(ret, 0);
+ ASSERT_TRUE(camera != nullptr);
+
+ api_.TrackingFeatureExtractorDestroy(camera);
+}
+
#else // !DVR_TRACKING_IMPLEMENTED
TEST_F(DvrTrackingTest, NotImplemented) {
@@ -47,6 +74,20 @@
EXPECT_EQ(api_.TrackingCameraStart(nullptr, nullptr), -ENOSYS);
EXPECT_EQ(api_.TrackingCameraStop(nullptr), -ENOSYS);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorCreate != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorDestroy != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorStart != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorStop != nullptr);
+ ASSERT_TRUE(api_.TrackingFeatureExtractorProcessBuffer != nullptr);
+
+ EXPECT_EQ(api_.TrackingFeatureExtractorCreate(nullptr), -ENOSYS);
+ EXPECT_EQ(api_.TrackingFeatureExtractorStart(nullptr, nullptr, nullptr),
+ -ENOSYS);
+ EXPECT_EQ(api_.TrackingFeatureExtractorStop(nullptr), -ENOSYS);
+ EXPECT_EQ(api_.TrackingFeatureExtractorProcessBuffer(nullptr, nullptr,
+ nullptr, nullptr),
+ -ENOSYS);
+
ASSERT_TRUE(api_.TrackingSensorsCreate != nullptr);
ASSERT_TRUE(api_.TrackingSensorsDestroy != nullptr);
ASSERT_TRUE(api_.TrackingSensorsStart != nullptr);