blob: 8382b4610a7f782b15eb1d4dbfd3e5c38a9250a3 [file] [log] [blame]
Chia-I Wuf1405182017-11-27 11:29:21 -08001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef ANDROID_BUFFERLAYERCONSUMER_H
18#define ANDROID_BUFFERLAYERCONSUMER_H
19
20#include <EGL/egl.h>
21#include <EGL/eglext.h>
22
23#include <gui/BufferQueueDefs.h>
24#include <gui/ConsumerBase.h>
25
26#include <ui/FenceTime.h>
27#include <ui/GraphicBuffer.h>
28
29#include <utils/String8.h>
30#include <utils/Vector.h>
31#include <utils/threads.h>
32
33namespace android {
34// ----------------------------------------------------------------------------
35
36class String8;
37
38/*
39 * BufferLayerConsumer consumes buffers of graphics data from a BufferQueue,
40 * and makes them available to OpenGL as a texture.
41 *
42 * A typical usage pattern is to set up the BufferLayerConsumer with the
43 * desired options, and call updateTexImage() when a new frame is desired.
44 * If a new frame is available, the texture will be updated. If not,
45 * the previous contents are retained.
46 *
Chia-I Wuc91077c2017-11-27 13:32:04 -080047 * The texture is attached to the GL_TEXTURE_EXTERNAL_OES texture target, in
48 * the EGL context of the first thread that calls updateTexImage(). After that
49 * point, all calls to updateTexImage must be made with the same OpenGL ES
50 * context current.
Chia-I Wuf1405182017-11-27 11:29:21 -080051 *
52 * This class was previously called SurfaceTexture.
53 */
54class BufferLayerConsumer : public ConsumerBase {
55public:
Chia-I Wuf1405182017-11-27 11:29:21 -080056 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener;
57
Chia-I Wuc91077c2017-11-27 13:32:04 -080058 // BufferLayerConsumer constructs a new BufferLayerConsumer object.
59 // The tex parameter indicates the name of the OpenGL ES
Chia-I Wubd854bf2017-11-27 13:41:26 -080060 // texture to which images are to be streamed.
61 BufferLayerConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t tex);
Chia-I Wuf1405182017-11-27 11:29:21 -080062
Chia-I Wuf1405182017-11-27 11:29:21 -080063 // updateTexImage acquires the most recently queued buffer, and sets the
64 // image contents of the target texture to it.
65 //
66 // This call may only be made while the OpenGL ES context to which the
67 // target texture belongs is bound to the calling thread.
68 //
69 // This calls doGLFenceWait to ensure proper synchronization.
70 status_t updateTexImage();
71
Chia-I Wuf1405182017-11-27 11:29:21 -080072 // setReleaseFence stores a fence that will signal when the current buffer
73 // is no longer being read. This fence will be returned to the producer
74 // when the current buffer is released by updateTexImage(). Multiple
75 // fences can be set for a given buffer; they will be merged into a single
76 // union fence.
77 virtual void setReleaseFence(const sp<Fence>& fence);
78
79 // getTransformMatrix retrieves the 4x4 texture coordinate transform matrix
80 // associated with the texture image set by the most recent call to
81 // updateTexImage.
82 //
83 // This transform matrix maps 2D homogeneous texture coordinates of the form
84 // (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture
85 // coordinate that should be used to sample that location from the texture.
86 // Sampling the texture outside of the range of this transform is undefined.
87 //
88 // This transform is necessary to compensate for transforms that the stream
89 // content producer may implicitly apply to the content. By forcing users of
90 // a BufferLayerConsumer to apply this transform we avoid performing an extra
91 // copy of the data that would be needed to hide the transform from the
92 // user.
93 //
94 // The matrix is stored in column-major order so that it may be passed
95 // directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv
96 // functions.
97 void getTransformMatrix(float mtx[16]);
98
Chia-I Wuf1405182017-11-27 11:29:21 -080099 // getTimestamp retrieves the timestamp associated with the texture image
100 // set by the most recent call to updateTexImage.
101 //
102 // The timestamp is in nanoseconds, and is monotonically increasing. Its
103 // other semantics (zero point, etc) are source-dependent and should be
104 // documented by the source.
105 int64_t getTimestamp();
106
107 // getDataSpace retrieves the DataSpace associated with the texture image
108 // set by the most recent call to updateTexImage.
109 android_dataspace getCurrentDataSpace();
110
111 // getFrameNumber retrieves the frame number associated with the texture
112 // image set by the most recent call to updateTexImage.
113 //
114 // The frame number is an incrementing counter set to 0 at the creation of
115 // the BufferQueue associated with this consumer.
116 uint64_t getFrameNumber();
117
118 // setDefaultBufferSize is used to set the size of buffers returned by
119 // requestBuffers when a with and height of zero is requested.
120 // A call to setDefaultBufferSize() may trigger requestBuffers() to
121 // be called from the client.
122 // The width and height parameters must be no greater than the minimum of
123 // GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv).
124 // An error due to invalid dimensions might not be reported until
125 // updateTexImage() is called.
126 status_t setDefaultBufferSize(uint32_t width, uint32_t height);
127
128 // setFilteringEnabled sets whether the transform matrix should be computed
129 // for use with bilinear filtering.
130 void setFilteringEnabled(bool enabled);
131
132 // getCurrentBuffer returns the buffer associated with the current image.
133 // When outSlot is not nullptr, the current buffer slot index is also
134 // returned.
135 sp<GraphicBuffer> getCurrentBuffer(int* outSlot = nullptr) const;
136
Chia-I Wuf1405182017-11-27 11:29:21 -0800137 // getCurrentCrop returns the cropping rectangle of the current buffer.
138 Rect getCurrentCrop() const;
139
140 // getCurrentTransform returns the transform of the current buffer.
141 uint32_t getCurrentTransform() const;
142
143 // getCurrentScalingMode returns the scaling mode of the current buffer.
144 uint32_t getCurrentScalingMode() const;
145
146 // getCurrentFence returns the fence indicating when the current buffer is
147 // ready to be read from.
148 sp<Fence> getCurrentFence() const;
149
150 // getCurrentFence returns the FenceTime indicating when the current
151 // buffer is ready to be read from.
152 std::shared_ptr<FenceTime> getCurrentFenceTime() const;
153
154 // setConsumerUsageBits overrides the ConsumerBase method to OR
155 // DEFAULT_USAGE_FLAGS to usage.
156 status_t setConsumerUsageBits(uint64_t usage);
157
Chia-I Wuf1405182017-11-27 11:29:21 -0800158protected:
159 // abandonLocked overrides the ConsumerBase method to clear
160 // mCurrentTextureImage in addition to the ConsumerBase behavior.
161 virtual void abandonLocked();
162
163 // dumpLocked overrides the ConsumerBase method to dump BufferLayerConsumer-
164 // specific info in addition to the ConsumerBase behavior.
165 virtual void dumpLocked(String8& result, const char* prefix) const;
166
167 // acquireBufferLocked overrides the ConsumerBase method to update the
168 // mEglSlots array in addition to the ConsumerBase behavior.
169 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen,
170 uint64_t maxFrameNumber = 0) override;
171
Chia-I Wuf1405182017-11-27 11:29:21 -0800172 struct PendingRelease {
Chia-I Wu6aff69b2017-11-27 14:08:48 -0800173 PendingRelease() : isPending(false), currentTexture(-1), graphicBuffer() {}
Chia-I Wuf1405182017-11-27 11:29:21 -0800174
175 bool isPending;
176 int currentTexture;
177 sp<GraphicBuffer> graphicBuffer;
Chia-I Wuf1405182017-11-27 11:29:21 -0800178 };
179
180 // This releases the buffer in the slot referenced by mCurrentTexture,
181 // then updates state to refer to the BufferItem, which must be a
182 // newly-acquired buffer. If pendingRelease is not null, the parameters
183 // which would have been passed to releaseBufferLocked upon the successful
184 // completion of the method will instead be returned to the caller, so that
185 // it may call releaseBufferLocked itself later.
186 status_t updateAndReleaseLocked(const BufferItem& item,
187 PendingRelease* pendingRelease = nullptr);
188
Chia-I Wubd854bf2017-11-27 13:41:26 -0800189 // Binds mTexName and the current buffer to sTexTarget. Uses
Chia-I Wuf1405182017-11-27 11:29:21 -0800190 // mCurrentTexture if it's set, mCurrentTextureImage if not. If the
191 // bind succeeds, this calls doGLFenceWait.
192 status_t bindTextureImageLocked();
193
194 // Gets the current EGLDisplay and EGLContext values, and compares them
195 // to mEglDisplay and mEglContext. If the fields have been previously
196 // set, the values must match; if not, the fields are set to the current
197 // values.
Chia-I Wuc91077c2017-11-27 13:32:04 -0800198 status_t checkAndUpdateEglStateLocked();
Chia-I Wuf1405182017-11-27 11:29:21 -0800199
200private:
201 // EglImage is a utility class for tracking and creating EGLImageKHRs. There
202 // is primarily just one image per slot, but there is also special cases:
Chia-I Wuf1405182017-11-27 11:29:21 -0800203 // - After freeBuffer, we must still keep the current image/buffer
204 // Reference counting EGLImages lets us handle all these cases easily while
205 // also only creating new EGLImages from buffers when required.
206 class EglImage : public LightRefBase<EglImage> {
207 public:
208 EglImage(sp<GraphicBuffer> graphicBuffer);
209
210 // createIfNeeded creates an EGLImage if required (we haven't created
211 // one yet, or the EGLDisplay or crop-rect has changed).
Chia-I Wuc91077c2017-11-27 13:32:04 -0800212 status_t createIfNeeded(EGLDisplay display, const Rect& cropRect);
Chia-I Wuf1405182017-11-27 11:29:21 -0800213
214 // This calls glEGLImageTargetTexture2DOES to bind the image to the
215 // texture in the specified texture target.
216 void bindToTextureTarget(uint32_t texTarget);
217
218 const sp<GraphicBuffer>& graphicBuffer() { return mGraphicBuffer; }
219 const native_handle* graphicBufferHandle() {
220 return mGraphicBuffer == NULL ? NULL : mGraphicBuffer->handle;
221 }
222
223 private:
224 // Only allow instantiation using ref counting.
225 friend class LightRefBase<EglImage>;
226 virtual ~EglImage();
227
228 // createImage creates a new EGLImage from a GraphicBuffer.
229 EGLImageKHR createImage(EGLDisplay dpy, const sp<GraphicBuffer>& graphicBuffer,
230 const Rect& crop);
231
232 // Disallow copying
233 EglImage(const EglImage& rhs);
234 void operator=(const EglImage& rhs);
235
236 // mGraphicBuffer is the buffer that was used to create this image.
237 sp<GraphicBuffer> mGraphicBuffer;
238
239 // mEglImage is the EGLImage created from mGraphicBuffer.
240 EGLImageKHR mEglImage;
241
242 // mEGLDisplay is the EGLDisplay that was used to create mEglImage.
243 EGLDisplay mEglDisplay;
244
245 // mCropRect is the crop rectangle passed to EGL when mEglImage
246 // was created.
247 Rect mCropRect;
248 };
249
250 // freeBufferLocked frees up the given buffer slot. If the slot has been
251 // initialized this will release the reference to the GraphicBuffer in that
252 // slot and destroy the EGLImage in that slot. Otherwise it has no effect.
253 //
254 // This method must be called with mMutex locked.
255 virtual void freeBufferLocked(int slotIndex);
256
257 // computeCurrentTransformMatrixLocked computes the transform matrix for the
258 // current texture. It uses mCurrentTransform and the current GraphicBuffer
259 // to compute this matrix and stores it in mCurrentTransformMatrix.
260 // mCurrentTextureImage must not be NULL.
261 void computeCurrentTransformMatrixLocked();
262
263 // doGLFenceWaitLocked inserts a wait command into the OpenGL ES command
264 // stream to ensure that it is safe for future OpenGL ES commands to
265 // access the current texture buffer.
266 status_t doGLFenceWaitLocked() const;
267
268 // syncForReleaseLocked performs the synchronization needed to release the
269 // current slot from an OpenGL ES context. If needed it will set the
270 // current slot's fence to guard against a producer accessing the buffer
271 // before the outstanding accesses have completed.
272 status_t syncForReleaseLocked(EGLDisplay dpy);
273
Chia-I Wubd854bf2017-11-27 13:41:26 -0800274 // sTexTarget is the GL texture target with which the GL texture object is
275 // associated.
276 static constexpr uint32_t sTexTarget = 0x8D65; // GL_TEXTURE_EXTERNAL_OES
277
Chia-I Wuf1405182017-11-27 11:29:21 -0800278 // The default consumer usage flags that BufferLayerConsumer always sets on its
279 // BufferQueue instance; these will be OR:d with any additional flags passed
280 // from the BufferLayerConsumer user. In particular, BufferLayerConsumer will always
281 // consume buffers as hardware textures.
282 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE;
283
284 // mCurrentTextureImage is the EglImage/buffer of the current texture. It's
285 // possible that this buffer is not associated with any buffer slot, so we
286 // must track it separately in order to support the getCurrentBuffer method.
287 sp<EglImage> mCurrentTextureImage;
288
289 // mCurrentCrop is the crop rectangle that applies to the current texture.
290 // It gets set each time updateTexImage is called.
291 Rect mCurrentCrop;
292
293 // mCurrentTransform is the transform identifier for the current texture. It
294 // gets set each time updateTexImage is called.
295 uint32_t mCurrentTransform;
296
297 // mCurrentScalingMode is the scaling mode for the current texture. It gets
298 // set each time updateTexImage is called.
299 uint32_t mCurrentScalingMode;
300
301 // mCurrentFence is the fence received from BufferQueue in updateTexImage.
302 sp<Fence> mCurrentFence;
303
304 // The FenceTime wrapper around mCurrentFence.
305 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE};
306
307 // mCurrentTransformMatrix is the transform matrix for the current texture.
308 // It gets computed by computeTransformMatrix each time updateTexImage is
309 // called.
310 float mCurrentTransformMatrix[16];
311
312 // mCurrentTimestamp is the timestamp for the current texture. It
313 // gets set each time updateTexImage is called.
314 int64_t mCurrentTimestamp;
315
316 // mCurrentDataSpace is the dataspace for the current texture. It
317 // gets set each time updateTexImage is called.
318 android_dataspace mCurrentDataSpace;
319
320 // mCurrentFrameNumber is the frame counter for the current texture.
321 // It gets set each time updateTexImage is called.
322 uint64_t mCurrentFrameNumber;
323
324 uint32_t mDefaultWidth, mDefaultHeight;
325
326 // mFilteringEnabled indicates whether the transform matrix is computed for
327 // use with bilinear filtering. It defaults to true and is changed by
328 // setFilteringEnabled().
329 bool mFilteringEnabled;
330
331 // mTexName is the name of the OpenGL texture to which streamed images will
Chia-I Wuc91077c2017-11-27 13:32:04 -0800332 // be bound when updateTexImage is called. It is set at construction time.
333 const uint32_t mTexName;
Chia-I Wuf1405182017-11-27 11:29:21 -0800334
Chia-I Wuf1405182017-11-27 11:29:21 -0800335 // EGLSlot contains the information and object references that
336 // BufferLayerConsumer maintains about a BufferQueue buffer slot.
337 struct EglSlot {
Chia-I Wuf1405182017-11-27 11:29:21 -0800338 // mEglImage is the EGLImage created from mGraphicBuffer.
339 sp<EglImage> mEglImage;
Chia-I Wuf1405182017-11-27 11:29:21 -0800340 };
341
342 // mEglDisplay is the EGLDisplay with which this BufferLayerConsumer is currently
343 // associated. It is intialized to EGL_NO_DISPLAY and gets set to the
Chia-I Wuc91077c2017-11-27 13:32:04 -0800344 // current display when updateTexImage is called for the first time.
Chia-I Wuf1405182017-11-27 11:29:21 -0800345 EGLDisplay mEglDisplay;
346
347 // mEglContext is the OpenGL ES context with which this BufferLayerConsumer is
348 // currently associated. It is initialized to EGL_NO_CONTEXT and gets set
349 // to the current GL context when updateTexImage is called for the first
Chia-I Wuc91077c2017-11-27 13:32:04 -0800350 // time.
Chia-I Wuf1405182017-11-27 11:29:21 -0800351 EGLContext mEglContext;
352
353 // mEGLSlots stores the buffers that have been allocated by the BufferQueue
354 // for each buffer slot. It is initialized to null pointers, and gets
355 // filled in with the result of BufferQueue::acquire when the
356 // client dequeues a buffer from a
357 // slot that has not yet been used. The buffer allocated to a slot will also
358 // be replaced if the requested buffer usage or geometry differs from that
359 // of the buffer allocated to a slot.
360 EglSlot mEglSlots[BufferQueueDefs::NUM_BUFFER_SLOTS];
361
362 // mCurrentTexture is the buffer slot index of the buffer that is currently
363 // bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT,
364 // indicating that no buffer slot is currently bound to the texture. Note,
365 // however, that a value of INVALID_BUFFER_SLOT does not necessarily mean
366 // that no buffer is bound to the texture. A call to setBufferCount will
367 // reset mCurrentTexture to INVALID_BUFFER_SLOT.
368 int mCurrentTexture;
Chia-I Wuf1405182017-11-27 11:29:21 -0800369};
370
371// ----------------------------------------------------------------------------
372}; // namespace android
373
374#endif // ANDROID_BUFFERLAYERCONSUMER_H