FreeRDP
h264_mediacodec.c
1 
20 #include <winpr/wlog.h>
21 #include <winpr/assert.h>
22 #include <winpr/library.h>
23 
24 #include <freerdp/log.h>
25 #include <freerdp/codec/h264.h>
26 
27 #include <media/NdkMediaCodec.h>
28 #include <media/NdkMediaFormat.h>
29 
30 #include "h264.h"
31 
32 static const char* CODEC_NAME = "video/avc";
33 
34 static const int COLOR_FormatYUV420Planar = 19;
35 static const int COLOR_FormatYUV420Flexible = 0x7f420888;
36 
37 /* https://developer.android.com/reference/android/media/MediaCodec#qualityFloor */
38 static const int MEDIACODEC_MINIMUM_WIDTH = 320;
39 static const int MEDIACODEC_MINIMUM_HEIGHT = 240;
40 
41 typedef struct
42 {
43  AMediaCodec* decoder;
44  AMediaFormat* inputFormat;
45  AMediaFormat* outputFormat;
46  int32_t width;
47  int32_t height;
48  int32_t outputWidth;
49  int32_t outputHeight;
50  ssize_t currentOutputBufferIndex;
51 } H264_CONTEXT_MEDIACODEC;
52 
53 static AMediaFormat* mediacodec_format_new(wLog* log, int width, int height)
54 {
55  const char* media_format;
56  AMediaFormat* format = AMediaFormat_new();
57  if (format == NULL)
58  {
59  WLog_Print(log, WLOG_ERROR, "AMediaFormat_new failed");
60  return NULL;
61  }
62 
63  AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
64  AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
65  AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
66  AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420Planar);
67 
68  media_format = AMediaFormat_toString(format);
69  if (media_format == NULL)
70  {
71  WLog_Print(log, WLOG_ERROR, "AMediaFormat_toString failed");
72  AMediaFormat_delete(format);
73  return NULL;
74  }
75 
76  WLog_Print(log, WLOG_DEBUG, "MediaCodec configuring with desired output format [%s]",
77  media_format);
78 
79  return format;
80 }
81 
82 static void set_mediacodec_format(H264_CONTEXT* h264, AMediaFormat** formatVariable,
83  AMediaFormat* newFormat)
84 {
85  media_status_t status = AMEDIA_OK;
86  H264_CONTEXT_MEDIACODEC* sys;
87 
88  WINPR_ASSERT(h264);
89  WINPR_ASSERT(formatVariable);
90 
91  sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
92  WINPR_ASSERT(sys);
93 
94  if (*formatVariable == newFormat)
95  return;
96 
97  if (*formatVariable != NULL)
98  {
99  status = AMediaFormat_delete(*formatVariable);
100  if (status != AMEDIA_OK)
101  {
102  WLog_Print(h264->log, WLOG_ERROR, "Error AMediaFormat_delete %d", status);
103  }
104  }
105 
106  *formatVariable = newFormat;
107 }
108 
109 static int update_mediacodec_inputformat(H264_CONTEXT* h264)
110 {
111  H264_CONTEXT_MEDIACODEC* sys;
112  AMediaFormat* inputFormat;
113  const char* mediaFormatName;
114 
115  WINPR_ASSERT(h264);
116 
117  sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
118  WINPR_ASSERT(sys);
119 
120 #if __ANDROID__ >= 21
121  inputFormat = AMediaCodec_getInputFormat(sys->decoder);
122  if (inputFormat == NULL)
123  {
124  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getInputFormat failed");
125  return -1;
126  }
127 #else
128  inputFormat = sys->inputFormat;
129 #endif
130  set_mediacodec_format(h264, &sys->inputFormat, inputFormat);
131 
132  mediaFormatName = AMediaFormat_toString(sys->inputFormat);
133  if (mediaFormatName == NULL)
134  {
135  WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_toString failed");
136  return -1;
137  }
138  WLog_Print(h264->log, WLOG_DEBUG, "Using MediaCodec with input MediaFormat [%s]",
139  mediaFormatName);
140 
141  return 1;
142 }
143 
144 static int update_mediacodec_outputformat(H264_CONTEXT* h264)
145 {
146  H264_CONTEXT_MEDIACODEC* sys;
147  AMediaFormat* outputFormat;
148  const char* mediaFormatName;
149  int32_t outputWidth, outputHeight;
150 
151  WINPR_ASSERT(h264);
152 
153  sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
154  WINPR_ASSERT(sys);
155 
156  outputFormat = AMediaCodec_getOutputFormat(sys->decoder);
157  if (outputFormat == NULL)
158  {
159  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getOutputFormat failed");
160  return -1;
161  }
162  set_mediacodec_format(h264, &sys->outputFormat, outputFormat);
163 
164  mediaFormatName = AMediaFormat_toString(sys->outputFormat);
165  if (mediaFormatName == NULL)
166  {
167  WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_toString failed");
168  return -1;
169  }
170  WLog_Print(h264->log, WLOG_DEBUG, "Using MediaCodec with output MediaFormat [%s]",
171  mediaFormatName);
172 
173  if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_WIDTH, &outputWidth))
174  {
175  WLog_Print(h264->log, WLOG_ERROR, "fnAMediaFormat_getInt32 failed getting width");
176  return -1;
177  }
178 
179  if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_HEIGHT, &outputHeight))
180  {
181  WLog_Print(h264->log, WLOG_ERROR, "fnAMediaFormat_getInt32 failed getting height");
182  return -1;
183  }
184 
185  sys->outputWidth = outputWidth;
186  sys->outputHeight = outputHeight;
187 
188  return 1;
189 }
190 
191 static void release_current_outputbuffer(H264_CONTEXT* h264)
192 {
193  media_status_t status = AMEDIA_OK;
194  H264_CONTEXT_MEDIACODEC* sys;
195 
196  WINPR_ASSERT(h264);
197  sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
198  WINPR_ASSERT(sys);
199 
200  if (sys->currentOutputBufferIndex < 0)
201  {
202  return;
203  }
204 
205  status = AMediaCodec_releaseOutputBuffer(sys->decoder, sys->currentOutputBufferIndex, FALSE);
206  if (status != AMEDIA_OK)
207  {
208  WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_releaseOutputBuffer %d", status);
209  }
210 
211  sys->currentOutputBufferIndex = -1;
212 }
213 
214 static int mediacodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const UINT32* pStride,
215  BYTE** ppDstData, UINT32* pDstSize)
216 {
217  WINPR_ASSERT(h264);
218  WINPR_ASSERT(pSrcYuv);
219  WINPR_ASSERT(pStride);
220  WINPR_ASSERT(ppDstData);
221  WINPR_ASSERT(pDstSize);
222 
223  WLog_Print(h264->log, WLOG_ERROR, "MediaCodec is not supported as an encoder");
224  return -1;
225 }
226 
227 static int mediacodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize)
228 {
229  ssize_t inputBufferId = -1;
230  size_t inputBufferSize, outputBufferSize;
231  uint8_t* inputBuffer;
232  media_status_t status;
233  BYTE** pYUVData;
234  UINT32* iStride;
235  H264_CONTEXT_MEDIACODEC* sys;
236 
237  WINPR_ASSERT(h264);
238  WINPR_ASSERT(pSrcData);
239 
240  sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
241  WINPR_ASSERT(sys);
242 
243  pYUVData = h264->pYUVData;
244  WINPR_ASSERT(pYUVData);
245 
246  iStride = h264->iStride;
247  WINPR_ASSERT(iStride);
248 
249  release_current_outputbuffer(h264);
250 
251  if (sys->width != h264->width || sys->height != h264->height)
252  {
253  sys->width = h264->width;
254  sys->height = h264->height;
255 
256  if (sys->width < MEDIACODEC_MINIMUM_WIDTH || sys->height < MEDIACODEC_MINIMUM_HEIGHT)
257  {
258  WLog_Print(h264->log, WLOG_ERROR,
259  "MediaCodec got width or height smaller than minimum [%d,%d]", sys->width,
260  sys->height);
261  return -1;
262  }
263 
264  WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec setting new input width and height [%d,%d]",
265  sys->width, sys->height);
266 
267 #if __ANDROID__ >= 26
268  AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_WIDTH, sys->width);
269  AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_HEIGHT, sys->height);
270  status = AMediaCodec_setParameters(sys->decoder, sys->inputFormat);
271  if (status != AMEDIA_OK)
272  {
273  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_setParameters failed: %d", status);
274  return -1;
275  }
276 #else
277  set_mediacodec_format(h264, &sys->inputFormat,
278  mediacodec_format_new(h264->log, sys->width, sys->height));
279 #endif
280 
281  // The codec can change output width and height
282  if (update_mediacodec_outputformat(h264) < 0)
283  {
284  WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating input format");
285  return -1;
286  }
287  }
288 
289  while (true)
290  {
291  UINT32 inputBufferCurrnetOffset = 0;
292  while (inputBufferCurrnetOffset < SrcSize)
293  {
294  UINT32 numberOfBytesToCopy = SrcSize - inputBufferCurrnetOffset;
295  inputBufferId = AMediaCodec_dequeueInputBuffer(sys->decoder, -1);
296  if (inputBufferId < 0)
297  {
298  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_dequeueInputBuffer failed [%d]",
299  inputBufferId);
300  // TODO: sleep?
301  continue;
302  }
303 
304  inputBuffer = AMediaCodec_getInputBuffer(sys->decoder, inputBufferId, &inputBufferSize);
305  if (inputBuffer == NULL)
306  {
307  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getInputBuffer failed");
308  return -1;
309  }
310 
311  if (numberOfBytesToCopy > inputBufferSize)
312  {
313  WLog_Print(h264->log, WLOG_WARN,
314  "MediaCodec inputBufferSize: got [%d] but wanted [%d]", inputBufferSize,
315  numberOfBytesToCopy);
316  numberOfBytesToCopy = inputBufferSize;
317  }
318 
319  memcpy(inputBuffer, pSrcData + inputBufferCurrnetOffset, numberOfBytesToCopy);
320  inputBufferCurrnetOffset += numberOfBytesToCopy;
321 
322  status = AMediaCodec_queueInputBuffer(sys->decoder, inputBufferId, 0,
323  numberOfBytesToCopy, 0, 0);
324  if (status != AMEDIA_OK)
325  {
326  WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_queueInputBuffer %d", status);
327  return -1;
328  }
329  }
330 
331  while (true)
332  {
333  AMediaCodecBufferInfo bufferInfo;
334  ssize_t outputBufferId = AMediaCodec_dequeueOutputBuffer(sys->decoder, &bufferInfo, -1);
335  if (outputBufferId >= 0)
336  {
337  sys->currentOutputBufferIndex = outputBufferId;
338 
339  uint8_t* outputBuffer;
340  outputBuffer =
341  AMediaCodec_getOutputBuffer(sys->decoder, outputBufferId, &outputBufferSize);
342  sys->currentOutputBufferIndex = outputBufferId;
343 
344  if (outputBufferSize !=
345  (sys->outputWidth * sys->outputHeight +
346  ((sys->outputWidth + 1) / 2) * ((sys->outputHeight + 1) / 2) * 2))
347  {
348  WLog_Print(h264->log, WLOG_ERROR,
349  "Error MediaCodec unexpected output buffer size %d",
350  outputBufferSize);
351  return -1;
352  }
353 
354  // TODO: work with AImageReader and get AImage object instead of
355  // COLOR_FormatYUV420Planar buffer.
356  iStride[0] = sys->outputWidth;
357  iStride[1] = (sys->outputWidth + 1) / 2;
358  iStride[2] = (sys->outputWidth + 1) / 2;
359  pYUVData[0] = outputBuffer;
360  pYUVData[1] = outputBuffer + iStride[0] * sys->outputHeight;
361  pYUVData[2] = outputBuffer + iStride[0] * sys->outputHeight +
362  iStride[1] * ((sys->outputHeight + 1) / 2);
363  break;
364  }
365  else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
366  {
367  if (update_mediacodec_outputformat(h264) < 0)
368  {
369  WLog_Print(h264->log, WLOG_ERROR,
370  "MediaCodec failed updating output format in decompress");
371  return -1;
372  }
373  }
374  else if (outputBufferId == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
375  {
376  WLog_Print(h264->log, WLOG_WARN,
377  "AMediaCodec_dequeueOutputBuffer need to try again later");
378  // TODO: sleep?
379  }
380  else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
381  {
382  WLog_Print(h264->log, WLOG_WARN,
383  "AMediaCodec_dequeueOutputBuffer returned deprecated value "
384  "AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED, ignoring");
385  }
386  else
387  {
388  WLog_Print(h264->log, WLOG_ERROR,
389  "AMediaCodec_dequeueOutputBuffer returned unknown value [%d]",
390  outputBufferId);
391  return -1;
392  }
393  }
394 
395  break;
396  }
397 
398  return 1;
399 }
400 
401 static void mediacodec_uninit(H264_CONTEXT* h264)
402 {
403  media_status_t status = AMEDIA_OK;
404  H264_CONTEXT_MEDIACODEC* sys;
405 
406  WINPR_ASSERT(h264);
407 
408  sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
409 
410  WLog_Print(h264->log, WLOG_DEBUG, "Uninitializing MediaCodec");
411 
412  if (!sys)
413  return;
414 
415  if (sys->decoder != NULL)
416  {
417  release_current_outputbuffer(h264);
418  status = AMediaCodec_stop(sys->decoder);
419  if (status != AMEDIA_OK)
420  {
421  WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_stop %d", status);
422  }
423 
424  status = AMediaCodec_delete(sys->decoder);
425  if (status != AMEDIA_OK)
426  {
427  WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_delete %d", status);
428  }
429 
430  sys->decoder = NULL;
431  }
432 
433  set_mediacodec_format(h264, &sys->inputFormat, NULL);
434  set_mediacodec_format(h264, &sys->outputFormat, NULL);
435 
436  free(sys);
437  h264->pSystemData = NULL;
438 }
439 
440 static BOOL mediacodec_init(H264_CONTEXT* h264)
441 {
442  H264_CONTEXT_MEDIACODEC* sys;
443  media_status_t status;
444 
445  WINPR_ASSERT(h264);
446 
447  if (h264->Compressor)
448  {
449  WLog_Print(h264->log, WLOG_ERROR, "MediaCodec is not supported as an encoder");
450  goto EXCEPTION;
451  }
452 
453  WLog_Print(h264->log, WLOG_DEBUG, "Initializing MediaCodec");
454 
455  sys = (H264_CONTEXT_MEDIACODEC*)calloc(1, sizeof(H264_CONTEXT_MEDIACODEC));
456 
457  if (!sys)
458  {
459  goto EXCEPTION;
460  }
461 
462  h264->pSystemData = (void*)sys;
463 
464  sys->currentOutputBufferIndex = -1;
465 
466  // updated when we're given the height and width for the first time
467  sys->width = sys->outputWidth = MEDIACODEC_MINIMUM_WIDTH;
468  sys->height = sys->outputHeight = MEDIACODEC_MINIMUM_HEIGHT;
469  sys->decoder = AMediaCodec_createDecoderByType(CODEC_NAME);
470  if (sys->decoder == NULL)
471  {
472  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_createCodecByName failed");
473  goto EXCEPTION;
474  }
475 
476 #if __ANDROID_API__ >= 28
477  char* codec_name;
478  status = AMediaCodec_getName(sys->decoder, &codec_name);
479  if (status != AMEDIA_OK)
480  {
481  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getName failed: %d", status);
482  goto EXCEPTION;
483  }
484 
485  WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec using %s codec [%s]", CODEC_NAME, codec_name);
486  AMediaCodec_releaseName(sys->decoder, codec_name);
487 #endif
488 
489  set_mediacodec_format(h264, &sys->inputFormat,
490  mediacodec_format_new(h264->log, sys->width, sys->height));
491 
492  status = AMediaCodec_configure(sys->decoder, sys->inputFormat, NULL, NULL, 0);
493  if (status != AMEDIA_OK)
494  {
495  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_configure failed: %d", status);
496  goto EXCEPTION;
497  }
498 
499  if (update_mediacodec_inputformat(h264) < 0)
500  {
501  WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating input format");
502  goto EXCEPTION;
503  }
504 
505  if (update_mediacodec_outputformat(h264) < 0)
506  {
507  WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating output format");
508  goto EXCEPTION;
509  }
510 
511  WLog_Print(h264->log, WLOG_DEBUG, "Starting MediaCodec");
512  status = AMediaCodec_start(sys->decoder);
513  if (status != AMEDIA_OK)
514  {
515  WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_start failed %d", status);
516  goto EXCEPTION;
517  }
518 
519  return TRUE;
520 EXCEPTION:
521  mediacodec_uninit(h264);
522  return FALSE;
523 }
524 
525 const H264_CONTEXT_SUBSYSTEM g_Subsystem_mediacodec = { "MediaCodec", mediacodec_init,
526  mediacodec_uninit, mediacodec_decompress,
527  mediacodec_compress };