21#include <freerdp/config.h>
23#include <winpr/wlog.h>
24#include <winpr/assert.h>
25#include <winpr/cast.h>
26#include <winpr/library.h>
28#include <freerdp/log.h>
29#include <freerdp/codec/h264.h>
31#include <media/NdkMediaCodec.h>
32#include <media/NdkMediaFormat.h>
36static const char* CODEC_NAME =
"video/avc";
38static const int COLOR_FormatYUV420Planar = 19;
39static const int COLOR_FormatYUV420SemiPlanar = 21;
40static const int COLOR_FormatYUV420Flexible = 0x7f420888;
44static const char* MEDIACODEC_KEY_PREPEND_HEADER =
"prepend-sps-pps-to-idr-frames";
47static const char* MEDIACODEC_KEY_LOW_LATENCY =
"low-latency";
50static const char* MEDIACODEC_KEY_CROP_LEFT =
"crop-left";
51static const char* MEDIACODEC_KEY_CROP_RIGHT =
"crop-right";
52static const char* MEDIACODEC_KEY_CROP_TOP =
"crop-top";
53static const char* MEDIACODEC_KEY_CROP_BOTTOM =
"crop-bottom";
56static const int64_t MEDIACODEC_INPUT_DEQUEUE_TIMEOUT = 10000;
57static const int64_t MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT = 5000;
60static const int MEDIACODEC_DECODE_MAX_RETRIES = 100;
63static const int MEDIACODEC_MINIMUM_WIDTH = 320;
64static const int MEDIACODEC_MINIMUM_HEIGHT = 240;
65static const int32_t MEDIACODEC_MAXIMUM_DIMENSION = 16384;
68static const int MEDIACODEC_IFRAME_INTERVAL = 1;
70static const int MEDIACODEC_DEFAULT_FRAMERATE = 30;
71static const int MEDIACODEC_DEFAULT_BITRATE = 4000000;
74static const int MEDIACODEC_BITRATE_MODE_VBR = 1;
77static const char* MEDIACODEC_PARAM_VIDEO_BITRATE =
"video-bitrate";
83 AMediaFormat* inputFormat;
84 AMediaFormat* outputFormat;
90 int32_t outputSliceHeight;
94 ssize_t currentOutputBufferIndex;
99 BYTE* decodeI420Buffer;
100 size_t decodeI420Capacity;
104 int32_t encoderWidth;
105 int32_t encoderHeight;
106 int32_t encoderStride;
107 int32_t encoderSliceHeight;
108 int32_t encoderBitrate;
112 size_t outputCapacity;
113 int32_t encoderFailedWidth;
114 int32_t encoderFailedHeight;
119} H264_CONTEXT_MEDIACODEC;
121#define mediacodec_warn(log, status, what) \
122 mediacodec_warn_((log), (status), (what), __FILE__, __func__, __LINE__)
124static void mediacodec_warn_(wLog* log, media_status_t status,
const char* what,
const char* file,
125 const char* fkt,
size_t line)
127 if ((status != AMEDIA_OK) && WLog_IsLevelActive(log, WLOG_WARN))
128 WLog_PrintTextMessage(log, WLOG_WARN, line, file, fkt,
"%s failed: %d", what, status);
132static AMediaFormat* mediacodec_format_new(wLog* log,
int width,
int height)
134 const char* media_format =
nullptr;
135 AMediaFormat* format = AMediaFormat_new();
136 if (format ==
nullptr)
138 WLog_Print(log, WLOG_ERROR,
"AMediaFormat_new failed");
142 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
143 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
144 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
145 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420Planar);
147 AMediaFormat_setInt32(format, MEDIACODEC_KEY_LOW_LATENCY, 1);
148 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PRIORITY, 0);
150 media_format = AMediaFormat_toString(format);
151 if (media_format ==
nullptr)
153 WLog_Print(log, WLOG_ERROR,
"AMediaFormat_toString failed");
154 AMediaFormat_delete(format);
158 WLog_Print(log, WLOG_DEBUG,
"MediaCodec configuring with desired output format [%s]",
164static void set_mediacodec_format(H264_CONTEXT* h264, AMediaFormat** formatVariable,
165 AMediaFormat* newFormat)
167 media_status_t status = AMEDIA_OK;
168 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
171 WINPR_ASSERT(formatVariable);
173 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
176 if (*formatVariable == newFormat)
179 if (*formatVariable !=
nullptr)
181 status = AMediaFormat_delete(*formatVariable);
182 if (status != AMEDIA_OK)
184 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaFormat_delete %d", status);
188 *formatVariable = newFormat;
192static int update_mediacodec_inputformat(H264_CONTEXT* h264)
194 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
195 AMediaFormat* inputFormat =
nullptr;
196 const char* mediaFormatName =
nullptr;
200 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
203 inputFormat = AMediaCodec_getInputFormat(sys->decoder);
204 if (inputFormat ==
nullptr)
206 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getInputFormat failed");
209 set_mediacodec_format(h264, &sys->inputFormat, inputFormat);
211 mediaFormatName = AMediaFormat_toString(sys->inputFormat);
212 if (mediaFormatName ==
nullptr)
214 WLog_Print(h264->log, WLOG_ERROR,
"AMediaFormat_toString failed");
217 WLog_Print(h264->log, WLOG_DEBUG,
"Using MediaCodec with input MediaFormat [%s]",
224static int update_mediacodec_outputformat(H264_CONTEXT* h264)
226 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
227 AMediaFormat* outputFormat =
nullptr;
228 const char* mediaFormatName =
nullptr;
229 int32_t outputWidth = 0;
230 int32_t outputHeight = 0;
234 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
237 outputFormat = AMediaCodec_getOutputFormat(sys->decoder);
238 if (outputFormat ==
nullptr)
240 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getOutputFormat failed");
243 set_mediacodec_format(h264, &sys->outputFormat, outputFormat);
245 mediaFormatName = AMediaFormat_toString(sys->outputFormat);
246 if (mediaFormatName ==
nullptr)
248 WLog_Print(h264->log, WLOG_ERROR,
"AMediaFormat_toString failed");
251 WLog_Print(h264->log, WLOG_DEBUG,
"Using MediaCodec with output MediaFormat [%s]",
254 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_WIDTH, &outputWidth))
256 WLog_Print(h264->log, WLOG_ERROR,
"fnAMediaFormat_getInt32 failed getting width");
260 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_HEIGHT, &outputHeight))
262 WLog_Print(h264->log, WLOG_ERROR,
"fnAMediaFormat_getInt32 failed getting height");
267 int32_t outputStride = 0;
268 int32_t outputSliceHeight = 0;
269 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_STRIDE, &outputStride) ||
270 outputStride < outputWidth)
271 outputStride = outputWidth;
272 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_SLICE_HEIGHT,
273 &outputSliceHeight) ||
274 outputSliceHeight < outputHeight)
275 outputSliceHeight = outputHeight;
276 if ((outputWidth <= 0) || (outputHeight <= 0) ||
277 (outputStride > MEDIACODEC_MAXIMUM_DIMENSION) ||
278 (outputSliceHeight > MEDIACODEC_MAXIMUM_DIMENSION))
280 WLog_Print(h264->log, WLOG_ERROR,
281 "MediaCodec output layout out of range: %dx%d stride %d slice-height %d",
282 outputWidth, outputHeight, outputStride, outputSliceHeight);
285 sys->outputStride = outputStride;
286 sys->outputSliceHeight = outputSliceHeight;
290 sys->outputWidth = outputWidth;
291 sys->outputHeight = outputHeight;
295 int32_t cropLeft = 0;
296 int32_t cropRight = 0;
298 int32_t cropBottom = 0;
300 if (!AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_LEFT, &cropLeft))
302 if (!AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_TOP, &cropTop))
305 if (AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_RIGHT, &cropRight) &&
306 AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_BOTTOM, &cropBottom) &&
307 (cropLeft >= 0) && (cropTop >= 0) && (cropRight >= cropLeft) && (cropBottom >= cropTop) &&
308 (cropRight < outputWidth) && (cropBottom < outputHeight))
310 sys->outputWidth = cropRight - cropLeft + 1;
311 sys->outputHeight = cropBottom - cropTop + 1;
312 sys->cropLeft = cropLeft;
313 sys->cropTop = cropTop;
317 int32_t colorFormat = 0;
318 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat))
319 colorFormat = COLOR_FormatYUV420Planar;
320 sys->colorFormat = colorFormat;
322 if (colorFormat == COLOR_FormatYUV420Flexible)
323 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec output is flexible YUV, assuming NV12");
326 h264->YUVWidth = (UINT32)sys->outputWidth;
327 h264->YUVHeight = (UINT32)sys->outputHeight;
329 WLog_Print(h264->log, WLOG_DEBUG,
330 "MediaCodec output %dx%d stride %d slice-height %d crop [%d,%d] color 0x%x",
331 sys->outputWidth, sys->outputHeight, sys->outputStride, sys->outputSliceHeight,
332 sys->cropLeft, sys->cropTop, colorFormat);
337static void release_current_outputbuffer(H264_CONTEXT* h264)
339 media_status_t status = AMEDIA_OK;
340 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
343 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
346 if (sys->currentOutputBufferIndex < 0)
351 status = AMediaCodec_releaseOutputBuffer(sys->decoder, sys->currentOutputBufferIndex, FALSE);
352 if (status != AMEDIA_OK)
354 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_releaseOutputBuffer %d", status);
357 sys->currentOutputBufferIndex = -1;
361static void mediacodec_encoder_release(wLog* log, H264_CONTEXT_MEDIACODEC* sys)
363 if (!sys || !sys->encoder)
366 if (sys->encoderStarted)
367 mediacodec_warn(log, AMediaCodec_stop(sys->encoder),
"AMediaCodec_stop");
368 mediacodec_warn(log, AMediaCodec_delete(sys->encoder),
"AMediaCodec_delete");
369 sys->encoder =
nullptr;
370 sys->encoderStarted = FALSE;
373 free(sys->spsPpsData);
374 sys->spsPpsData =
nullptr;
380static BOOL mediacodec_encoder_probe(wLog* log)
382 AMediaCodec* probe = AMediaCodec_createEncoderByType(CODEC_NAME);
385 WLog_Print(log, WLOG_WARN,
"No MediaCodec H.264 encoder on this device");
390 AMediaFormat* format = AMediaFormat_new();
393 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
394 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, MEDIACODEC_MINIMUM_WIDTH);
395 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, MEDIACODEC_MINIMUM_HEIGHT);
396 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420SemiPlanar);
397 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, MEDIACODEC_DEFAULT_BITRATE);
398 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE, MEDIACODEC_BITRATE_MODE_VBR);
399 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, MEDIACODEC_DEFAULT_FRAMERATE);
400 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL,
401 MEDIACODEC_IFRAME_INTERVAL);
403 if (AMediaCodec_configure(probe, format,
nullptr,
nullptr,
404 AMEDIACODEC_CONFIGURE_FLAG_ENCODE) == AMEDIA_OK &&
405 AMediaCodec_start(probe) == AMEDIA_OK)
408 mediacodec_warn(log, AMediaCodec_stop(probe),
"AMediaCodec_stop");
410 mediacodec_warn(log, AMediaFormat_delete(format),
"AMediaFormat_delete");
413 mediacodec_warn(log, AMediaCodec_delete(probe),
"AMediaCodec_delete");
415 WLog_Print(log, WLOG_WARN,
416 "MediaCodec H.264 encoder cannot be configured (color format unsupported?)");
421static int32_t mediacodec_rate(UINT32 value, int32_t fallback)
425 return (value > INT32_MAX) ? INT32_MAX : WINPR_ASSERTING_INT_CAST(int32_t, value);
430static BOOL mediacodec_encoder_configure(H264_CONTEXT* h264)
433 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
436 mediacodec_encoder_release(h264->log, sys);
438 if (h264->width < MEDIACODEC_MINIMUM_WIDTH || h264->height < MEDIACODEC_MINIMUM_HEIGHT ||
439 h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION ||
440 h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION)
442 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec encoder dimensions out of range [%u,%u]",
443 h264->width, h264->height);
447 const int32_t width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
448 const int32_t height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
449 const int32_t framerate = mediacodec_rate(h264->FrameRate, MEDIACODEC_DEFAULT_FRAMERATE);
450 const int32_t bitrate = mediacodec_rate(h264->BitRate, MEDIACODEC_DEFAULT_BITRATE);
452 sys->encoder = AMediaCodec_createEncoderByType(CODEC_NAME);
455 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_createEncoderByType failed");
459 AMediaFormat* format = AMediaFormat_new();
462 WLog_Print(h264->log, WLOG_ERROR,
"AMediaFormat_new failed");
463 mediacodec_encoder_release(h264->log, sys);
467 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
468 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
469 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
470 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420SemiPlanar);
471 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, bitrate);
473 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE, MEDIACODEC_BITRATE_MODE_VBR);
474 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, framerate);
475 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, MEDIACODEC_IFRAME_INTERVAL);
476 AMediaFormat_setInt32(format, MEDIACODEC_KEY_PREPEND_HEADER, 1);
478 media_status_t status = AMediaCodec_configure(sys->encoder, format,
nullptr,
nullptr,
479 AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
480 mediacodec_warn(h264->log, AMediaFormat_delete(format),
"AMediaFormat_delete");
481 if (status != AMEDIA_OK)
483 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_configure (encoder) failed: %d", status);
484 mediacodec_encoder_release(h264->log, sys);
488 status = AMediaCodec_start(sys->encoder);
489 if (status != AMEDIA_OK)
491 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_start (encoder) failed: %d", status);
492 mediacodec_encoder_release(h264->log, sys);
496 sys->encoderStarted = TRUE;
497 sys->encoderWidth = width;
498 sys->encoderHeight = height;
499 sys->encoderBitrate = bitrate;
500 sys->syncRequested = FALSE;
504 sys->encoderStride = sys->encoderWidth + (sys->encoderWidth & 1);
505 sys->encoderSliceHeight = sys->encoderHeight;
506 AMediaFormat* inputFormat = AMediaCodec_getInputFormat(sys->encoder);
510 int32_t sliceHeight = 0;
511 if (AMediaFormat_getInt32(inputFormat, AMEDIAFORMAT_KEY_STRIDE, &stride) &&
512 (stride >= sys->encoderWidth))
513 sys->encoderStride = stride;
514 if (AMediaFormat_getInt32(inputFormat, AMEDIAFORMAT_KEY_SLICE_HEIGHT, &sliceHeight) &&
515 (sliceHeight >= sys->encoderHeight))
516 sys->encoderSliceHeight = sliceHeight;
517 mediacodec_warn(h264->log, AMediaFormat_delete(inputFormat),
"AMediaFormat_delete");
520 if ((sys->encoderStride > MEDIACODEC_MAXIMUM_DIMENSION) ||
521 (sys->encoderSliceHeight > MEDIACODEC_MAXIMUM_DIMENSION))
523 WLog_Print(h264->log, WLOG_ERROR,
524 "MediaCodec encoder input layout out of range: stride %d slice-height %d",
525 sys->encoderStride, sys->encoderSliceHeight);
526 mediacodec_encoder_release(h264->log, sys);
531 h264->log, WLOG_DEBUG,
532 "MediaCodec H.264 encoder started: %ux%u @ %d fps, %d bps, stride %d, slice-height %d",
533 h264->width, h264->height, framerate, bitrate, sys->encoderStride, sys->encoderSliceHeight);
540static BOOL mediacodec_copy_i420_to_nv12(uint8_t* dst,
size_t dstCapacity,
const BYTE** pSrcYuv,
541 const UINT32* pStride, int32_t width, int32_t height,
542 int32_t dstStride, int32_t dstSliceHeight)
545 WINPR_ASSERT(pSrcYuv);
546 WINPR_ASSERT(pStride);
548 const int32_t cw = (width + 1) / 2;
549 const int32_t ch = (height + 1) / 2;
550 const size_t ySize = (size_t)dstStride * dstSliceHeight;
552 if (dstCapacity < ySize + (
size_t)dstStride * ((dstSliceHeight + 1) / 2))
555 for (int32_t row = 0; row < height; row++)
556 memcpy(dst + (
size_t)row * dstStride, pSrcYuv[0] + (size_t)row * pStride[0], (
size_t)width);
559 const int32_t pairs = (dstStride / 2 < cw) ? dstStride / 2 : cw;
560 uint8_t* uv = dst + ySize;
561 for (int32_t row = 0; row < ch; row++)
563 const BYTE* uRow = pSrcYuv[1] + (size_t)row * pStride[1];
564 const BYTE* vRow = pSrcYuv[2] + (size_t)row * pStride[2];
565 uint8_t* uvRow = uv + (size_t)row * dstStride;
566 for (int32_t col = 0; col < pairs; col++)
568 uvRow[(size_t)col * 2] = uRow[col];
569 uvRow[(size_t)col * 2 + 1] = vRow[col];
578static BOOL mediacodec_has_sps(
const uint8_t* data,
size_t size)
582 for (
size_t i = 0; i + 3 < size; i++)
584 if ((data[i] != 0) || (data[i + 1] != 0))
588 if (data[i + 2] == 1)
590 else if ((data[i + 2] == 0) && (data[i + 3] == 1))
598 const uint8_t type = data[nal] & 0x1F;
601 if ((type == 1) || (type == 5))
610static void mediacodec_copy_nv12_to_i420(BYTE* dstYuv[3],
const UINT32 dstStride[3],
611 const uint8_t* src, int32_t width, int32_t height,
612 int32_t srcStride, int32_t srcSliceHeight,
613 int32_t cropLeft, int32_t cropTop)
615 WINPR_ASSERT(dstYuv);
616 WINPR_ASSERT(dstStride);
619 const int32_t cw = (width + 1) / 2;
620 const int32_t ch = (height + 1) / 2;
621 const size_t srcLumaOffset = (size_t)cropTop * (
size_t)srcStride + (size_t)cropLeft;
622 const size_t srcChromaOffset = (size_t)srcStride * (
size_t)srcSliceHeight +
623 (size_t)(cropTop / 2) * (size_t)srcStride +
624 ((
size_t)cropLeft & ~1ULL);
626 const uint8_t* srcY = src + srcLumaOffset;
627 const uint8_t* srcUV = src + srcChromaOffset;
630 const int32_t chromaLeft = cropLeft & ~1;
631 const int32_t avail = (srcStride > chromaLeft) ? (srcStride - chromaLeft) / 2 : 0;
632 const int32_t pairs = (avail < cw) ? avail : cw;
634 for (int32_t row = 0; row < height; row++)
636 memcpy(dstYuv[0] + (
size_t)row * (
size_t)dstStride[0],
637 srcY + (
size_t)row * (
size_t)srcStride, (
size_t)width);
640 for (int32_t row = 0; row < ch; row++)
642 const uint8_t* uvRow = srcUV + (size_t)row * (
size_t)srcStride;
643 BYTE* uRow = dstYuv[1] + (size_t)row * (
size_t)dstStride[1];
644 BYTE* vRow = dstYuv[2] + (size_t)row * (
size_t)dstStride[2];
645 for (int32_t col = 0; col < pairs; col++)
647 uRow[col] = uvRow[(size_t)col * 2];
648 vRow[col] = uvRow[(size_t)col * 2 + 1];
652 for (int32_t col = pairs; col < cw; col++)
661static int mediacodec_compress(H264_CONTEXT* h264,
const BYTE** pSrcYuv,
const UINT32* pStride,
662 BYTE** ppDstData, UINT32* pDstSize)
665 WINPR_ASSERT(pSrcYuv);
666 WINPR_ASSERT(pStride);
667 WINPR_ASSERT(ppDstData);
668 WINPR_ASSERT(pDstSize);
670 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
673 if (!pSrcYuv[0] || !pSrcYuv[1] || !pSrcYuv[2])
676 if ((h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION) ||
677 (h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION))
680 const int32_t width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
681 const int32_t height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
683 if (!sys->encoderStarted || sys->encoderWidth != width || sys->encoderHeight != height)
685 if ((width == sys->encoderFailedWidth) && (height == sys->encoderFailedHeight))
688 if (!mediacodec_encoder_configure(h264))
690 sys->encoderFailedWidth = width;
691 sys->encoderFailedHeight = height;
694 sys->encoderFailedWidth = 0;
695 sys->encoderFailedHeight = 0;
699 const int32_t bitrate = mediacodec_rate(h264->BitRate, sys->encoderBitrate);
700 if (bitrate != sys->encoderBitrate)
702 AMediaFormat* params = AMediaFormat_new();
705 AMediaFormat_setInt32(params, MEDIACODEC_PARAM_VIDEO_BITRATE, bitrate);
706 if (AMediaCodec_setParameters(sys->encoder, params) == AMEDIA_OK)
707 sys->encoderBitrate = bitrate;
708 mediacodec_warn(h264->log, AMediaFormat_delete(params),
"AMediaFormat_delete");
712 if (!h264->firstLumaFrameDone && (sys->frameIndex > 0) && !sys->syncRequested)
714 AMediaFormat* syncParams = AMediaFormat_new();
717 AMediaFormat_setInt32(syncParams,
"request-sync", 0);
718 if (AMediaCodec_setParameters(sys->encoder, syncParams) == AMEDIA_OK)
719 sys->syncRequested = TRUE;
720 mediacodec_warn(h264->log, AMediaFormat_delete(syncParams),
"AMediaFormat_delete");
724 const int32_t dstStride = sys->encoderStride;
725 const int32_t dstSliceHeight = sys->encoderSliceHeight;
726 const size_t frameSize =
727 (size_t)dstStride * dstSliceHeight + (
size_t)dstStride * ((dstSliceHeight + 1) / 2);
730 const ssize_t inIdx =
731 AMediaCodec_dequeueInputBuffer(sys->encoder, MEDIACODEC_INPUT_DEQUEUE_TIMEOUT);
735 AMediaCodecBufferInfo stale = { 0 };
736 const ssize_t staleIdx = AMediaCodec_dequeueOutputBuffer(sys->encoder, &stale, 0);
738 mediacodec_warn(h264->log,
739 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)staleIdx, FALSE),
740 "AMediaCodec_releaseOutputBuffer");
742 WLog_Print(h264->log, WLOG_WARN,
"MediaCodec encoder has no input buffer, dropping frame");
746 size_t inCapacity = 0;
747 uint8_t* inBuffer = AMediaCodec_getInputBuffer(sys->encoder, (
size_t)inIdx, &inCapacity);
748 if (!inBuffer || inCapacity < frameSize)
750 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec encoder input buffer too small: %zu < %zu",
751 inCapacity, frameSize);
752 mediacodec_warn(h264->log,
753 AMediaCodec_queueInputBuffer(sys->encoder, (
size_t)inIdx, 0, 0, 0, 0),
754 "AMediaCodec_queueInputBuffer");
758 if (!mediacodec_copy_i420_to_nv12(inBuffer, inCapacity, pSrcYuv, pStride, width, height,
759 dstStride, dstSliceHeight))
761 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec encoder input buffer too small for layout");
762 mediacodec_warn(h264->log,
763 AMediaCodec_queueInputBuffer(sys->encoder, (
size_t)inIdx, 0, 0, 0, 0),
764 "AMediaCodec_queueInputBuffer");
768 const int32_t framerate = mediacodec_rate(h264->FrameRate, MEDIACODEC_DEFAULT_FRAMERATE);
769 const uint64_t pts = (uint64_t)sys->frameIndex++ * 1000000ULL / (uint64_t)framerate;
770 media_status_t status =
771 AMediaCodec_queueInputBuffer(sys->encoder, (
size_t)inIdx, 0, frameSize, pts, 0);
772 if (status != AMEDIA_OK)
774 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_queueInputBuffer (encoder) failed: %d",
782 AMediaCodecBufferInfo info = { 0 };
783 const ssize_t outIdx =
784 AMediaCodec_dequeueOutputBuffer(sys->encoder, &info, MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT);
787 size_t outCapacity = 0;
789 AMediaCodec_getOutputBuffer(sys->encoder, (
size_t)outIdx, &outCapacity);
790 if (!outBuffer || (info.size < 0) || (info.offset < 0) ||
791 ((
size_t)info.offset > outCapacity) ||
792 ((
size_t)info.size > outCapacity - (
size_t)info.offset))
794 WLog_Print(h264->log, WLOG_ERROR,
795 "MediaCodec encoder output out of range: offset %d size %d cap %zu",
796 info.offset, info.size, outCapacity);
798 h264->log, AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE),
799 "AMediaCodec_releaseOutputBuffer");
803 if (((info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) != 0) && (info.size > 0))
805 BYTE* newCfg = (BYTE*)realloc(sys->spsPpsData, (
size_t)info.size);
808 sys->spsPpsData = newCfg;
809 sys->spsPpsSize = (size_t)info.size;
810 memcpy(sys->spsPpsData, outBuffer + info.offset, (
size_t)info.size);
814 WLog_Print(h264->log, WLOG_WARN,
"MediaCodec encoder failed to store SPS/PPS");
818 if (((info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) != 0) || (info.size == 0))
820 const media_status_t rc =
821 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE);
824 WLog_Print(h264->log, WLOG_ERROR,
825 "AMediaCodec_releaseOutputBuffer (encoder) failed: %d", rc);
831 const BOOL isKeyFrame = (info.flags & AMEDIACODEC_BUFFER_FLAG_KEY_FRAME) != 0;
832 const BOOL prependSps = isKeyFrame && sys->spsPpsData && sys->spsPpsSize > 0 &&
833 !mediacodec_has_sps(outBuffer + info.offset, (
size_t)info.size);
834 const size_t totalSize = (size_t)info.size + (prependSps ? sys->spsPpsSize : 0);
836 if (sys->outputCapacity < totalSize)
838 BYTE* resized = (BYTE*)realloc(sys->outputData, totalSize);
843 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE),
844 "AMediaCodec_releaseOutputBuffer");
847 sys->outputData = resized;
848 sys->outputCapacity = totalSize;
853 memcpy(sys->outputData, sys->spsPpsData, sys->spsPpsSize);
854 memcpy(sys->outputData + sys->spsPpsSize, outBuffer + info.offset,
859 memcpy(sys->outputData, outBuffer + info.offset, (
size_t)info.size);
862 *ppDstData = sys->outputData;
863 *pDstSize = WINPR_ASSERTING_INT_CAST(UINT32, totalSize);
864 const media_status_t rc =
865 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE);
867 WLog_Print(h264->log, WLOG_ERROR,
868 "AMediaCodec_releaseOutputBuffer (encoder) failed: %d", rc);
871 else if (outIdx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED ||
872 outIdx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
876 else if (outIdx == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
879 WLog_Print(h264->log, WLOG_DEBUG,
880 "MediaCodec encoder output not ready, dropping frame");
885 WLog_Print(h264->log, WLOG_ERROR,
886 "AMediaCodec_dequeueOutputBuffer (encoder) returned %zd", outIdx);
893static int mediacodec_decompress(H264_CONTEXT* h264,
const BYTE* pSrcData, UINT32 SrcSize)
896 WINPR_ASSERT(pSrcData);
898 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
901 BYTE** pYUVData = h264->pYUVData;
902 WINPR_ASSERT(pYUVData);
904 UINT32* iStride = h264->iStride;
905 WINPR_ASSERT(iStride);
907 release_current_outputbuffer(h264);
909 if ((h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION) ||
910 (h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION))
912 WLog_Print(h264->log, WLOG_ERROR,
913 "MediaCodec size [%" PRIu32
",%" PRIu32
"] exceeds the maximum", h264->width,
918 if ((sys->width != WINPR_ASSERTING_INT_CAST(int32_t, h264->width)) ||
919 (sys->height != WINPR_ASSERTING_INT_CAST(int32_t, h264->height)))
921 sys->width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
922 sys->height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
924 if (sys->width < MEDIACODEC_MINIMUM_WIDTH || sys->height < MEDIACODEC_MINIMUM_HEIGHT)
926 WLog_Print(h264->log, WLOG_ERROR,
927 "MediaCodec got width or height smaller than minimum [%d,%d]", sys->width,
932 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec setting new input width and height [%d,%d]",
933 sys->width, sys->height);
935 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_WIDTH, sys->width);
936 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_HEIGHT, sys->height);
937 const media_status_t status = AMediaCodec_setParameters(sys->decoder, sys->inputFormat);
938 if (status != AMEDIA_OK)
941 WLog_Print(h264->log, WLOG_DEBUG,
942 "AMediaCodec_setParameters returned %d, relying on "
943 "in-band SPS/PPS for the size change",
948 if (update_mediacodec_outputformat(h264) < 0)
950 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec failed updating input format");
957 UINT32 inputBufferCurrnetOffset = 0;
958 int inputRetries = 0;
959 while (inputBufferCurrnetOffset < SrcSize)
961 UINT32 numberOfBytesToCopy = SrcSize - inputBufferCurrnetOffset;
962 const ssize_t inputBufferId =
963 AMediaCodec_dequeueInputBuffer(sys->decoder, MEDIACODEC_INPUT_DEQUEUE_TIMEOUT);
964 if (inputBufferId < 0)
966 if (++inputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
968 WLog_Print(h264->log, WLOG_ERROR,
969 "AMediaCodec_dequeueInputBuffer gave up [%zd]", inputBufferId);
976 size_t inputBufferSize = 0;
977 uint8_t* inputBuffer =
978 AMediaCodec_getInputBuffer(sys->decoder, inputBufferId, &inputBufferSize);
979 if (inputBuffer ==
nullptr)
981 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getInputBuffer failed");
985 if (numberOfBytesToCopy > inputBufferSize)
987 WLog_Print(h264->log, WLOG_WARN,
988 "MediaCodec inputBufferSize: got [%zu] but wanted [%u]", inputBufferSize,
989 numberOfBytesToCopy);
990 numberOfBytesToCopy = WINPR_ASSERTING_INT_CAST(UINT32, inputBufferSize);
993 memcpy(inputBuffer, &pSrcData[inputBufferCurrnetOffset], numberOfBytesToCopy);
994 inputBufferCurrnetOffset += numberOfBytesToCopy;
996 const media_status_t status = AMediaCodec_queueInputBuffer(
997 sys->decoder, inputBufferId, 0, numberOfBytesToCopy, 0, 0);
998 if (status != AMEDIA_OK)
1000 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_queueInputBuffer %d", status);
1005 int outputRetries = 0;
1008 AMediaCodecBufferInfo bufferInfo = WINPR_C_ARRAY_INIT;
1009 ssize_t outputBufferId = AMediaCodec_dequeueOutputBuffer(
1010 sys->decoder, &bufferInfo, MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT);
1011 if (outputBufferId >= 0)
1013 sys->currentOutputBufferIndex = outputBufferId;
1015 size_t outputBufferSize = 0;
1016 uint8_t* outputBuffer =
nullptr;
1018 AMediaCodec_getOutputBuffer(sys->decoder, outputBufferId, &outputBufferSize);
1019 sys->currentOutputBufferIndex = outputBufferId;
1022 if (bufferInfo.size <= 0)
1024 release_current_outputbuffer(h264);
1025 if (++outputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1031 if (!outputBuffer || (bufferInfo.offset < 0) ||
1032 ((
size_t)bufferInfo.offset > outputBufferSize))
1034 WLog_Print(h264->log, WLOG_ERROR,
1035 "MediaCodec output offset %d out of range %zu", bufferInfo.offset,
1039 outputBuffer += bufferInfo.offset;
1040 outputBufferSize -= (size_t)bufferInfo.offset;
1042 const size_t lumaSize = (size_t)sys->outputStride * (
size_t)sys->outputSliceHeight;
1043 const int32_t chromaStride = sys->outputStride / 2;
1044 const int32_t chromaHeight = (sys->outputSliceHeight + 1) / 2;
1045 const size_t chromaSize = (size_t)chromaStride * (
size_t)chromaHeight;
1046 const size_t minRequiredSize =
1047 lumaSize + (size_t)sys->outputStride * (
size_t)chromaHeight;
1049 if (outputBufferSize < minRequiredSize)
1051 WLog_Print(h264->log, WLOG_ERROR,
1052 "Error MediaCodec output buffer too small %zu < %zu",
1053 outputBufferSize, minRequiredSize);
1057 if (sys->colorFormat == COLOR_FormatYUV420Planar)
1059 iStride[0] = (UINT32)sys->outputStride;
1060 iStride[1] = (UINT32)chromaStride;
1061 iStride[2] = (UINT32)chromaStride;
1062 pYUVData[0] = outputBuffer +
1063 ((size_t)sys->cropTop * (
size_t)sys->outputStride) +
1064 (
size_t)sys->cropLeft;
1065 pYUVData[1] = outputBuffer + lumaSize +
1066 ((size_t)(sys->cropTop / 2) * (size_t)chromaStride) +
1067 ((size_t)sys->cropLeft / 2);
1068 pYUVData[2] = outputBuffer + lumaSize + chromaSize +
1069 ((size_t)(sys->cropTop / 2) * (size_t)chromaStride) +
1070 ((size_t)sys->cropLeft / 2);
1075 const UINT32 dstYStride = (UINT32)sys->outputWidth;
1076 const UINT32 dstUVStride = (UINT32)((sys->outputWidth + 1) / 2);
1077 const size_t yBytes = (size_t)dstYStride * (
size_t)sys->outputHeight;
1078 const size_t uvBytes =
1079 (size_t)dstUVStride * (
size_t)((sys->outputHeight + 1) / 2);
1080 const size_t neededCapacity = yBytes + uvBytes * 2;
1082 if (sys->decodeI420Capacity < neededCapacity)
1084 BYTE* newBuf = (BYTE*)realloc(sys->decodeI420Buffer, neededCapacity);
1087 WLog_Print(h264->log, WLOG_ERROR,
1088 "Failed to allocate I420 deinterleave buffer");
1091 sys->decodeI420Buffer = newBuf;
1092 sys->decodeI420Capacity = neededCapacity;
1095 iStride[0] = dstYStride;
1096 iStride[1] = dstUVStride;
1097 iStride[2] = dstUVStride;
1098 pYUVData[0] = sys->decodeI420Buffer;
1099 pYUVData[1] = sys->decodeI420Buffer + yBytes;
1100 pYUVData[2] = sys->decodeI420Buffer + yBytes + uvBytes;
1102 mediacodec_copy_nv12_to_i420(
1103 pYUVData, iStride, outputBuffer, sys->outputWidth, sys->outputHeight,
1104 sys->outputStride, sys->outputSliceHeight, sys->cropLeft, sys->cropTop);
1105 release_current_outputbuffer(h264);
1108 h264->YUVWidth = (UINT32)sys->outputWidth;
1109 h264->YUVHeight = (UINT32)sys->outputHeight;
1112 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
1114 if (update_mediacodec_outputformat(h264) < 0)
1116 WLog_Print(h264->log, WLOG_ERROR,
1117 "MediaCodec failed updating output format in decompress");
1121 else if (outputBufferId == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
1123 if (++outputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1125 WLog_Print(h264->log, WLOG_DEBUG,
1126 "AMediaCodec_dequeueOutputBuffer no buffer ready yet");
1130 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
1132 WLog_Print(h264->log, WLOG_WARN,
1133 "AMediaCodec_dequeueOutputBuffer returned deprecated value "
1134 "AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED, ignoring");
1138 WLog_Print(h264->log, WLOG_ERROR,
1139 "AMediaCodec_dequeueOutputBuffer returned unknown value [%zd]",
1151static void mediacodec_uninit(H264_CONTEXT* h264)
1153 media_status_t status = AMEDIA_OK;
1154 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
1158 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
1160 WLog_Print(h264->log, WLOG_DEBUG,
"Uninitializing MediaCodec");
1165 if (sys->decoder !=
nullptr)
1167 release_current_outputbuffer(h264);
1169 if (sys->decoderStarted)
1171 status = AMediaCodec_stop(sys->decoder);
1172 if (status != AMEDIA_OK)
1174 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_stop %d", status);
1176 sys->decoderStarted = FALSE;
1179 status = AMediaCodec_delete(sys->decoder);
1180 if (status != AMEDIA_OK)
1182 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_delete %d", status);
1185 sys->decoder =
nullptr;
1188 mediacodec_encoder_release(h264->log, sys);
1189 free(sys->outputData);
1190 sys->outputData =
nullptr;
1191 free(sys->decodeI420Buffer);
1192 sys->decodeI420Buffer =
nullptr;
1193 free(sys->spsPpsData);
1194 sys->spsPpsData =
nullptr;
1196 set_mediacodec_format(h264, &sys->inputFormat,
nullptr);
1197 set_mediacodec_format(h264, &sys->outputFormat,
nullptr);
1200 h264->pSystemData =
nullptr;
1204static BOOL mediacodec_init(H264_CONTEXT* h264)
1206 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
1207 media_status_t status = AMEDIA_OK;
1211 WLog_Print(h264->log, WLOG_DEBUG,
"Initializing MediaCodec");
1213 sys = (H264_CONTEXT_MEDIACODEC*)calloc(1,
sizeof(H264_CONTEXT_MEDIACODEC));
1220 h264->pSystemData = (
void*)sys;
1222 sys->currentOutputBufferIndex = -1;
1225 sys->width = sys->outputWidth = MEDIACODEC_MINIMUM_WIDTH;
1226 sys->height = sys->outputHeight = MEDIACODEC_MINIMUM_HEIGHT;
1229 if (h264->Compressor)
1231 if (!mediacodec_encoder_probe(h264->log))
1233 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec H.264 encoder available");
1237 sys->decoder = AMediaCodec_createDecoderByType(CODEC_NAME);
1238 if (sys->decoder ==
nullptr)
1240 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_createCodecByName failed");
1244 char* codec_name =
nullptr;
1245 status = AMediaCodec_getName(sys->decoder, &codec_name);
1246 if (status != AMEDIA_OK)
1248 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getName failed: %d", status);
1252 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec using %s codec [%s]", CODEC_NAME, codec_name);
1253 AMediaCodec_releaseName(sys->decoder, codec_name);
1255 set_mediacodec_format(h264, &sys->inputFormat,
1256 mediacodec_format_new(h264->log, sys->width, sys->height));
1258 status = AMediaCodec_configure(sys->decoder, sys->inputFormat,
nullptr,
nullptr, 0);
1259 if (status != AMEDIA_OK)
1261 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_configure failed: %d", status);
1265 if (update_mediacodec_inputformat(h264) < 0)
1267 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec failed updating input format");
1271 if (update_mediacodec_outputformat(h264) < 0)
1273 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec failed updating output format");
1277 WLog_Print(h264->log, WLOG_DEBUG,
"Starting MediaCodec");
1278 status = AMediaCodec_start(sys->decoder);
1279 if (status != AMEDIA_OK)
1281 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_start failed %d", status);
1285 sys->decoderStarted = TRUE;
1288 mediacodec_uninit(h264);
1292const H264_CONTEXT_SUBSYSTEM g_Subsystem_mediacodec = {
"MediaCodec", mediacodec_init,
1293 mediacodec_uninit, mediacodec_decompress,
1294 mediacodec_compress };