FreeRDP
Loading...
Searching...
No Matches
h264_mediacodec.c
1
20#include <winpr/wlog.h>
21#include <winpr/assert.h>
22#include <winpr/library.h>
23
24#include <freerdp/log.h>
25#include <freerdp/codec/h264.h>
26
27#include <media/NdkMediaCodec.h>
28#include <media/NdkMediaFormat.h>
29
30#include "h264.h"
31
32static const char* CODEC_NAME = "video/avc";
33
34static const int COLOR_FormatYUV420Planar = 19;
35static const int COLOR_FormatYUV420Flexible = 0x7f420888;
36
37/* https://developer.android.com/reference/android/media/MediaCodec#qualityFloor */
38static const int MEDIACODEC_MINIMUM_WIDTH = 320;
39static const int MEDIACODEC_MINIMUM_HEIGHT = 240;
40
41typedef struct
42{
43 AMediaCodec* decoder;
44 AMediaFormat* inputFormat;
45 AMediaFormat* outputFormat;
46 int32_t width;
47 int32_t height;
48 int32_t outputWidth;
49 int32_t outputHeight;
50 ssize_t currentOutputBufferIndex;
51} H264_CONTEXT_MEDIACODEC;
52
53static AMediaFormat* mediacodec_format_new(wLog* log, int width, int height)
54{
55 const char* media_format;
56 AMediaFormat* format = AMediaFormat_new();
57 if (format == NULL)
58 {
59 WLog_Print(log, WLOG_ERROR, "AMediaFormat_new failed");
60 return NULL;
61 }
62
63 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
64 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
65 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
66 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420Planar);
67
68 media_format = AMediaFormat_toString(format);
69 if (media_format == NULL)
70 {
71 WLog_Print(log, WLOG_ERROR, "AMediaFormat_toString failed");
72 AMediaFormat_delete(format);
73 return NULL;
74 }
75
76 WLog_Print(log, WLOG_DEBUG, "MediaCodec configuring with desired output format [%s]",
77 media_format);
78
79 return format;
80}
81
82static void set_mediacodec_format(H264_CONTEXT* h264, AMediaFormat** formatVariable,
83 AMediaFormat* newFormat)
84{
85 media_status_t status = AMEDIA_OK;
86 H264_CONTEXT_MEDIACODEC* sys;
87
88 WINPR_ASSERT(h264);
89 WINPR_ASSERT(formatVariable);
90
91 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
92 WINPR_ASSERT(sys);
93
94 if (*formatVariable == newFormat)
95 return;
96
97 if (*formatVariable != NULL)
98 {
99 status = AMediaFormat_delete(*formatVariable);
100 if (status != AMEDIA_OK)
101 {
102 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaFormat_delete %d", status);
103 }
104 }
105
106 *formatVariable = newFormat;
107}
108
109static int update_mediacodec_inputformat(H264_CONTEXT* h264)
110{
111 H264_CONTEXT_MEDIACODEC* sys;
112 AMediaFormat* inputFormat;
113 const char* mediaFormatName;
114
115 WINPR_ASSERT(h264);
116
117 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
118 WINPR_ASSERT(sys);
119
120#if __ANDROID__ >= 21
121 inputFormat = AMediaCodec_getInputFormat(sys->decoder);
122 if (inputFormat == NULL)
123 {
124 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getInputFormat failed");
125 return -1;
126 }
127#else
128 inputFormat = sys->inputFormat;
129#endif
130 set_mediacodec_format(h264, &sys->inputFormat, inputFormat);
131
132 mediaFormatName = AMediaFormat_toString(sys->inputFormat);
133 if (mediaFormatName == NULL)
134 {
135 WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_toString failed");
136 return -1;
137 }
138 WLog_Print(h264->log, WLOG_DEBUG, "Using MediaCodec with input MediaFormat [%s]",
139 mediaFormatName);
140
141 return 1;
142}
143
144static int update_mediacodec_outputformat(H264_CONTEXT* h264)
145{
146 H264_CONTEXT_MEDIACODEC* sys;
147 AMediaFormat* outputFormat;
148 const char* mediaFormatName;
149 int32_t outputWidth, outputHeight;
150
151 WINPR_ASSERT(h264);
152
153 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
154 WINPR_ASSERT(sys);
155
156 outputFormat = AMediaCodec_getOutputFormat(sys->decoder);
157 if (outputFormat == NULL)
158 {
159 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getOutputFormat failed");
160 return -1;
161 }
162 set_mediacodec_format(h264, &sys->outputFormat, outputFormat);
163
164 mediaFormatName = AMediaFormat_toString(sys->outputFormat);
165 if (mediaFormatName == NULL)
166 {
167 WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_toString failed");
168 return -1;
169 }
170 WLog_Print(h264->log, WLOG_DEBUG, "Using MediaCodec with output MediaFormat [%s]",
171 mediaFormatName);
172
173 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_WIDTH, &outputWidth))
174 {
175 WLog_Print(h264->log, WLOG_ERROR, "fnAMediaFormat_getInt32 failed getting width");
176 return -1;
177 }
178
179 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_HEIGHT, &outputHeight))
180 {
181 WLog_Print(h264->log, WLOG_ERROR, "fnAMediaFormat_getInt32 failed getting height");
182 return -1;
183 }
184
185 sys->outputWidth = outputWidth;
186 sys->outputHeight = outputHeight;
187
188 return 1;
189}
190
191static void release_current_outputbuffer(H264_CONTEXT* h264)
192{
193 media_status_t status = AMEDIA_OK;
194 H264_CONTEXT_MEDIACODEC* sys;
195
196 WINPR_ASSERT(h264);
197 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
198 WINPR_ASSERT(sys);
199
200 if (sys->currentOutputBufferIndex < 0)
201 {
202 return;
203 }
204
205 status = AMediaCodec_releaseOutputBuffer(sys->decoder, sys->currentOutputBufferIndex, FALSE);
206 if (status != AMEDIA_OK)
207 {
208 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_releaseOutputBuffer %d", status);
209 }
210
211 sys->currentOutputBufferIndex = -1;
212}
213
214static int mediacodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const UINT32* pStride,
215 BYTE** ppDstData, UINT32* pDstSize)
216{
217 WINPR_ASSERT(h264);
218 WINPR_ASSERT(pSrcYuv);
219 WINPR_ASSERT(pStride);
220 WINPR_ASSERT(ppDstData);
221 WINPR_ASSERT(pDstSize);
222
223 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec is not supported as an encoder");
224 return -1;
225}
226
227static int mediacodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize)
228{
229 ssize_t inputBufferId = -1;
230 size_t inputBufferSize, outputBufferSize;
231 uint8_t* inputBuffer;
232 media_status_t status;
233 BYTE** pYUVData;
234 UINT32* iStride;
235 H264_CONTEXT_MEDIACODEC* sys;
236
237 WINPR_ASSERT(h264);
238 WINPR_ASSERT(pSrcData);
239
240 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
241 WINPR_ASSERT(sys);
242
243 pYUVData = h264->pYUVData;
244 WINPR_ASSERT(pYUVData);
245
246 iStride = h264->iStride;
247 WINPR_ASSERT(iStride);
248
249 release_current_outputbuffer(h264);
250
251 if (sys->width != h264->width || sys->height != h264->height)
252 {
253 sys->width = h264->width;
254 sys->height = h264->height;
255
256 if (sys->width < MEDIACODEC_MINIMUM_WIDTH || sys->height < MEDIACODEC_MINIMUM_HEIGHT)
257 {
258 WLog_Print(h264->log, WLOG_ERROR,
259 "MediaCodec got width or height smaller than minimum [%d,%d]", sys->width,
260 sys->height);
261 return -1;
262 }
263
264 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec setting new input width and height [%d,%d]",
265 sys->width, sys->height);
266
267#if __ANDROID__ >= 26
268 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_WIDTH, sys->width);
269 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_HEIGHT, sys->height);
270 status = AMediaCodec_setParameters(sys->decoder, sys->inputFormat);
271 if (status != AMEDIA_OK)
272 {
273 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_setParameters failed: %d", status);
274 return -1;
275 }
276#else
277 set_mediacodec_format(h264, &sys->inputFormat,
278 mediacodec_format_new(h264->log, sys->width, sys->height));
279#endif
280
281 // The codec can change output width and height
282 if (update_mediacodec_outputformat(h264) < 0)
283 {
284 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating input format");
285 return -1;
286 }
287 }
288
289 while (true)
290 {
291 UINT32 inputBufferCurrnetOffset = 0;
292 while (inputBufferCurrnetOffset < SrcSize)
293 {
294 UINT32 numberOfBytesToCopy = SrcSize - inputBufferCurrnetOffset;
295 inputBufferId = AMediaCodec_dequeueInputBuffer(sys->decoder, -1);
296 if (inputBufferId < 0)
297 {
298 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_dequeueInputBuffer failed [%d]",
299 inputBufferId);
300 // TODO: sleep?
301 continue;
302 }
303
304 inputBuffer = AMediaCodec_getInputBuffer(sys->decoder, inputBufferId, &inputBufferSize);
305 if (inputBuffer == NULL)
306 {
307 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getInputBuffer failed");
308 return -1;
309 }
310
311 if (numberOfBytesToCopy > inputBufferSize)
312 {
313 WLog_Print(h264->log, WLOG_WARN,
314 "MediaCodec inputBufferSize: got [%d] but wanted [%d]", inputBufferSize,
315 numberOfBytesToCopy);
316 numberOfBytesToCopy = inputBufferSize;
317 }
318
319 memcpy(inputBuffer, pSrcData + inputBufferCurrnetOffset, numberOfBytesToCopy);
320 inputBufferCurrnetOffset += numberOfBytesToCopy;
321
322 status = AMediaCodec_queueInputBuffer(sys->decoder, inputBufferId, 0,
323 numberOfBytesToCopy, 0, 0);
324 if (status != AMEDIA_OK)
325 {
326 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_queueInputBuffer %d", status);
327 return -1;
328 }
329 }
330
331 while (true)
332 {
333 AMediaCodecBufferInfo bufferInfo;
334 ssize_t outputBufferId = AMediaCodec_dequeueOutputBuffer(sys->decoder, &bufferInfo, -1);
335 if (outputBufferId >= 0)
336 {
337 sys->currentOutputBufferIndex = outputBufferId;
338
339 uint8_t* outputBuffer;
340 outputBuffer =
341 AMediaCodec_getOutputBuffer(sys->decoder, outputBufferId, &outputBufferSize);
342 sys->currentOutputBufferIndex = outputBufferId;
343
344 if (outputBufferSize !=
345 (sys->outputWidth * sys->outputHeight +
346 ((sys->outputWidth + 1) / 2) * ((sys->outputHeight + 1) / 2) * 2))
347 {
348 WLog_Print(h264->log, WLOG_ERROR,
349 "Error MediaCodec unexpected output buffer size %d",
350 outputBufferSize);
351 return -1;
352 }
353
354 // TODO: work with AImageReader and get AImage object instead of
355 // COLOR_FormatYUV420Planar buffer.
356 iStride[0] = sys->outputWidth;
357 iStride[1] = (sys->outputWidth + 1) / 2;
358 iStride[2] = (sys->outputWidth + 1) / 2;
359 pYUVData[0] = outputBuffer;
360 pYUVData[1] = outputBuffer + iStride[0] * sys->outputHeight;
361 pYUVData[2] = outputBuffer + iStride[0] * sys->outputHeight +
362 iStride[1] * ((sys->outputHeight + 1) / 2);
363 break;
364 }
365 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
366 {
367 if (update_mediacodec_outputformat(h264) < 0)
368 {
369 WLog_Print(h264->log, WLOG_ERROR,
370 "MediaCodec failed updating output format in decompress");
371 return -1;
372 }
373 }
374 else if (outputBufferId == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
375 {
376 WLog_Print(h264->log, WLOG_WARN,
377 "AMediaCodec_dequeueOutputBuffer need to try again later");
378 // TODO: sleep?
379 }
380 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
381 {
382 WLog_Print(h264->log, WLOG_WARN,
383 "AMediaCodec_dequeueOutputBuffer returned deprecated value "
384 "AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED, ignoring");
385 }
386 else
387 {
388 WLog_Print(h264->log, WLOG_ERROR,
389 "AMediaCodec_dequeueOutputBuffer returned unknown value [%d]",
390 outputBufferId);
391 return -1;
392 }
393 }
394
395 break;
396 }
397
398 return 1;
399}
400
401static void mediacodec_uninit(H264_CONTEXT* h264)
402{
403 media_status_t status = AMEDIA_OK;
404 H264_CONTEXT_MEDIACODEC* sys;
405
406 WINPR_ASSERT(h264);
407
408 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
409
410 WLog_Print(h264->log, WLOG_DEBUG, "Uninitializing MediaCodec");
411
412 if (!sys)
413 return;
414
415 if (sys->decoder != NULL)
416 {
417 release_current_outputbuffer(h264);
418 status = AMediaCodec_stop(sys->decoder);
419 if (status != AMEDIA_OK)
420 {
421 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_stop %d", status);
422 }
423
424 status = AMediaCodec_delete(sys->decoder);
425 if (status != AMEDIA_OK)
426 {
427 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_delete %d", status);
428 }
429
430 sys->decoder = NULL;
431 }
432
433 set_mediacodec_format(h264, &sys->inputFormat, NULL);
434 set_mediacodec_format(h264, &sys->outputFormat, NULL);
435
436 free(sys);
437 h264->pSystemData = NULL;
438}
439
440static BOOL mediacodec_init(H264_CONTEXT* h264)
441{
442 H264_CONTEXT_MEDIACODEC* sys;
443 media_status_t status;
444
445 WINPR_ASSERT(h264);
446
447 if (h264->Compressor)
448 {
449 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec is not supported as an encoder");
450 goto EXCEPTION;
451 }
452
453 WLog_Print(h264->log, WLOG_DEBUG, "Initializing MediaCodec");
454
455 sys = (H264_CONTEXT_MEDIACODEC*)calloc(1, sizeof(H264_CONTEXT_MEDIACODEC));
456
457 if (!sys)
458 {
459 goto EXCEPTION;
460 }
461
462 h264->pSystemData = (void*)sys;
463
464 sys->currentOutputBufferIndex = -1;
465
466 // updated when we're given the height and width for the first time
467 sys->width = sys->outputWidth = MEDIACODEC_MINIMUM_WIDTH;
468 sys->height = sys->outputHeight = MEDIACODEC_MINIMUM_HEIGHT;
469 sys->decoder = AMediaCodec_createDecoderByType(CODEC_NAME);
470 if (sys->decoder == NULL)
471 {
472 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_createCodecByName failed");
473 goto EXCEPTION;
474 }
475
476#if __ANDROID_API__ >= 28
477 char* codec_name;
478 status = AMediaCodec_getName(sys->decoder, &codec_name);
479 if (status != AMEDIA_OK)
480 {
481 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getName failed: %d", status);
482 goto EXCEPTION;
483 }
484
485 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec using %s codec [%s]", CODEC_NAME, codec_name);
486 AMediaCodec_releaseName(sys->decoder, codec_name);
487#endif
488
489 set_mediacodec_format(h264, &sys->inputFormat,
490 mediacodec_format_new(h264->log, sys->width, sys->height));
491
492 status = AMediaCodec_configure(sys->decoder, sys->inputFormat, NULL, NULL, 0);
493 if (status != AMEDIA_OK)
494 {
495 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_configure failed: %d", status);
496 goto EXCEPTION;
497 }
498
499 if (update_mediacodec_inputformat(h264) < 0)
500 {
501 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating input format");
502 goto EXCEPTION;
503 }
504
505 if (update_mediacodec_outputformat(h264) < 0)
506 {
507 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating output format");
508 goto EXCEPTION;
509 }
510
511 WLog_Print(h264->log, WLOG_DEBUG, "Starting MediaCodec");
512 status = AMediaCodec_start(sys->decoder);
513 if (status != AMEDIA_OK)
514 {
515 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_start failed %d", status);
516 goto EXCEPTION;
517 }
518
519 return TRUE;
520EXCEPTION:
521 mediacodec_uninit(h264);
522 return FALSE;
523}
524
525const H264_CONTEXT_SUBSYSTEM g_Subsystem_mediacodec = { "MediaCodec", mediacodec_init,
526 mediacodec_uninit, mediacodec_decompress,
527 mediacodec_compress };