FreeRDP
Loading...
Searching...
No Matches
codec/video.c
1
20#include <freerdp/config.h>
21
22#include <string.h>
23
24#include <winpr/assert.h>
25#include <winpr/wlog.h>
26#include <winpr/stream.h>
27#include <freerdp/log.h>
28
29#include <freerdp/codec/video.h>
30#include <freerdp/codec/h264.h>
31
32#define VTAG FREERDP_TAG("codec.video")
33
34#if defined(WITH_SWSCALE) || defined(WITH_MEDIACODEC) || defined(WITH_APPLE_VIDEOTOOLBOX)
35static UINT32 video_get_h264_bitrate(wLog* log, UINT32 height)
36{
37 static struct
38 {
39 UINT32 height;
40 UINT32 bitrate;
41 } bitrates[] = {
42 { 1080, 2700 }, { 720, 1250 }, { 480, 700 }, { 360, 400 },
43 { 240, 170 }, { 180, 140 }, { 0, 100 },
44 };
45 const size_t nBitrates = ARRAYSIZE(bitrates);
46
47 for (size_t i = 0; i < nBitrates; i++)
48 {
49 if (height >= bitrates[i].height)
50 {
51 UINT32 bitrate = bitrates[i].bitrate;
52 WLog_Print(log, WLOG_DEBUG, "Auto-calculated H.264 bitrate: %u kbps", bitrate);
53 return bitrate * 1000;
54 }
55 }
56
57 WINPR_ASSERT(FALSE);
58 return 0;
59}
60#endif
61
62#if defined(WITH_SWSCALE)
63
64/* Forward declarations for static functions */
65static BOOL freerdp_video_fill_plane_info(wLog* log, BYTE* data[4], int lineSize[4],
66 FREERDP_VIDEO_FORMAT format, UINT32 width, UINT32 height,
67 const BYTE* buffer);
68
69#include "image_ffmpeg.h"
70
71/* MJPEG decoder only available when NOT using runtime loading */
72#if defined(WITH_VIDEO_FFMPEG) && !defined(WITH_SWSCALE_LOADING)
73#define WITH_MJPEG_DECODER
74#include <libavcodec/avcodec.h>
75#endif
76
77struct s_FREERDP_VIDEO_CONTEXT
78{
79 UINT32 width;
80 UINT32 height;
81 struct SwsContext* sws;
82
83#if defined(WITH_MJPEG_DECODER)
84 AVCodecContext* mjpegDecoder;
85 AVPacket* mjpegPacket;
86 AVFrame* mjpegFrame;
87#endif
88
89 H264_CONTEXT* h264;
90 BOOL h264Configured;
91 UINT32 h264Framerate;
92 UINT32 h264Bitrate;
93 UINT32 h264UsageType;
94 wLog* log;
95};
96
100static enum AVPixelFormat video_format_to_av(wLog* log, FREERDP_VIDEO_FORMAT format)
101{
102 switch (format)
103 {
104 /* Compressed formats - not pixel formats */
105 case FREERDP_VIDEO_FORMAT_MJPEG:
106 case FREERDP_VIDEO_FORMAT_H264:
107 return AV_PIX_FMT_NONE;
108
109 /* Planar YUV formats */
110 case FREERDP_VIDEO_FORMAT_YUV420P:
111 return AV_PIX_FMT_YUV420P;
112 case FREERDP_VIDEO_FORMAT_NV12:
113 return AV_PIX_FMT_NV12;
114
115 /* Packed YUV formats */
116 case FREERDP_VIDEO_FORMAT_YUYV422:
117 return AV_PIX_FMT_YUYV422;
118
119 /* RGB formats */
120 case FREERDP_VIDEO_FORMAT_RGB24:
121 return AV_PIX_FMT_RGB24;
122 case FREERDP_VIDEO_FORMAT_RGB32:
123 return AV_PIX_FMT_RGB32;
124
125 case FREERDP_VIDEO_FORMAT_NONE:
126 default:
127 WLog_Print(log, WLOG_WARN, "Could not map FREERDP_VIDEO_FORMAT %s [0x%08" PRIx32 "]",
128 freerdp_video_format_string(format), format);
129 return AV_PIX_FMT_NONE;
130 }
131}
132
133BOOL freerdp_video_available(void)
134{
135 return freerdp_swscale_available() && freerdp_avutil_available();
136}
137
138FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(UINT32 width, UINT32 height)
139{
140 FREERDP_VIDEO_CONTEXT* context = nullptr;
141
142 wLog* log = WLog_Get(VTAG);
143 if (!freerdp_video_available())
144 {
145 WLog_Print(log, WLOG_ERROR, "Video codecs not available - FFmpeg not loaded");
146 return nullptr;
147 }
148
149 context = (FREERDP_VIDEO_CONTEXT*)calloc(1, sizeof(FREERDP_VIDEO_CONTEXT));
150 if (!context)
151 return nullptr;
152
153 context->log = log;
154 context->width = width;
155 context->height = height;
156
157#if defined(WITH_MJPEG_DECODER)
158 /* Initialize MJPEG decoder */
159 const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_MJPEG);
160 if (!codec)
161 {
162 WLog_Print(context->log, WLOG_ERROR, "avcodec_find_decoder failed to find MJPEG codec");
163 goto fail;
164 }
165
166 context->mjpegDecoder = avcodec_alloc_context3(codec);
167 if (!context->mjpegDecoder)
168 {
169 WLog_Print(context->log, WLOG_ERROR, "avcodec_alloc_context3 failed");
170 goto fail;
171 }
172
173 context->mjpegDecoder->width = (int)width;
174 context->mjpegDecoder->height = (int)height;
175 /* Abort on minor errors to skip corrupted frames */
176 context->mjpegDecoder->err_recognition |= AV_EF_EXPLODE;
177
178 if (avcodec_open2(context->mjpegDecoder, codec, nullptr) < 0)
179 {
180 WLog_Print(context->log, WLOG_ERROR, "avcodec_open2 failed");
181 goto fail;
182 }
183
184 context->mjpegPacket = av_packet_alloc();
185 if (!context->mjpegPacket)
186 {
187 WLog_Print(context->log, WLOG_ERROR, "av_packet_alloc failed");
188 goto fail;
189 }
190
191 context->mjpegFrame = av_frame_alloc();
192 if (!context->mjpegFrame)
193 {
194 WLog_Print(context->log, WLOG_ERROR, "av_frame_alloc failed");
195 goto fail;
196 }
197#endif
198
199 return context;
200
201#if defined(WITH_MJPEG_DECODER)
202fail:
203 freerdp_video_context_free(context);
204 return nullptr;
205#endif
206}
207
208void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
209{
210 if (!context)
211 return;
212
213 if (context->sws)
214 {
215 freerdp_sws_freeContext(context->sws);
216 context->sws = nullptr;
217 }
218
219#if defined(WITH_MJPEG_DECODER)
220 if (context->mjpegFrame)
221 av_frame_free(&context->mjpegFrame);
222
223 if (context->mjpegPacket)
224 {
225 context->mjpegPacket->data = nullptr;
226 context->mjpegPacket->size = 0;
227 av_packet_free(&context->mjpegPacket);
228 }
229
230 if (context->mjpegDecoder)
231 avcodec_free_context(&context->mjpegDecoder);
232#endif
233
234 if (context->h264)
235 {
236 h264_context_free(context->h264);
237 context->h264 = nullptr;
238 }
239
240 free(context);
241}
242
243BOOL freerdp_video_context_reconfigure(FREERDP_VIDEO_CONTEXT* context, UINT32 width, UINT32 height,
244 UINT32 framerate, UINT32 bitrate, UINT32 usageType)
245{
246 WINPR_ASSERT(context);
247
248 if (width == 0 || height == 0)
249 {
250 WLog_Print(context->log, WLOG_ERROR, "Invalid dimensions: %ux%u", width, height);
251 return FALSE;
252 }
253
254 if (bitrate == 0)
255 bitrate = video_get_h264_bitrate(context->log, height);
256
257 if (!context->h264)
258 {
259 context->h264 = h264_context_new(TRUE);
260 if (!context->h264)
261 {
262 WLog_Print(context->log, WLOG_ERROR, "h264_context_new failed");
263 return FALSE;
264 }
265 }
266
267 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_USAGETYPE, usageType))
268 goto fail;
269
270 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_FRAMERATE, framerate))
271 goto fail;
272
273 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_BITRATE, bitrate))
274 goto fail;
275
276 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_RATECONTROL,
277 H264_RATECONTROL_CQP))
278 goto fail;
279
280 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_QP, 26))
281 goto fail;
282
283 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL, FALSE))
284 goto fail;
285
286 if (!context->h264Configured || (context->width != width) || (context->height != height))
287 {
288 if (!h264_context_reset(context->h264, width, height))
289 {
290 WLog_Print(context->log, WLOG_ERROR, "h264_context_reset failed");
291 goto fail;
292 }
293 }
294
295 context->h264Framerate = framerate;
296 context->h264Bitrate = bitrate;
297 context->h264UsageType = usageType;
298 context->h264Configured = TRUE;
299 context->width = width;
300 context->height = height;
301
302 return TRUE;
303
304fail:
305 if (context->h264)
306 {
307 h264_context_free(context->h264);
308 context->h264 = nullptr;
309 }
310 return FALSE;
311}
312
313static BOOL freerdp_video_decode_mjpeg(FREERDP_VIDEO_CONTEXT* context, const BYTE* srcData,
314 size_t srcSize, BYTE* dstData[4], int dstLineSize[4],
315 enum AVPixelFormat* dstFormat);
316
317static BOOL freerdp_video_convert_to_yuv(FREERDP_VIDEO_CONTEXT* context, const BYTE* srcData[4],
318 const int srcLineSize[4], enum AVPixelFormat srcFormat,
319 BYTE* dstData[3], const int dstLineSize[3],
320 FREERDP_VIDEO_FORMAT dstFormat, UINT32 width,
321 UINT32 height);
322
323BOOL freerdp_video_sample_convert(FREERDP_VIDEO_CONTEXT* context, FREERDP_VIDEO_FORMAT srcFormat,
324 const void* srcSampleData, size_t srcSampleLength,
325 FREERDP_VIDEO_FORMAT dstFormat, wStream* output)
326{
327 WINPR_ASSERT(context);
328 WINPR_ASSERT(srcSampleData);
329 WINPR_ASSERT(output);
330
331 if (srcFormat == dstFormat)
332 {
333 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
334 return FALSE;
335 Stream_Write(output, srcSampleData, srcSampleLength);
336 return TRUE;
337 }
338
339 if (!freerdp_video_available())
340 {
341 WLog_Print(context->log, WLOG_ERROR, "Video codecs not available");
342 return FALSE;
343 }
344
345 if (srcSampleLength == 0)
346 {
347 WLog_Print(context->log, WLOG_ERROR, "Invalid source sample length: 0");
348 return FALSE;
349 }
350
351 if (!freerdp_video_conversion_supported(srcFormat, dstFormat))
352 {
353 WLog_Print(context->log, WLOG_ERROR, "Conversion from format %u to %u not supported",
354 srcFormat, dstFormat);
355 return FALSE;
356 }
357
358 const BOOL srcCompressed =
359 (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG || srcFormat == FREERDP_VIDEO_FORMAT_H264);
360 const BOOL dstCompressed =
361 (dstFormat == FREERDP_VIDEO_FORMAT_MJPEG || dstFormat == FREERDP_VIDEO_FORMAT_H264);
362
363 BYTE* intermediate_data[4] = WINPR_C_ARRAY_INIT;
364 int intermediate_linesize[4] = WINPR_C_ARRAY_INIT;
365 enum AVPixelFormat intermediate_format = AV_PIX_FMT_NONE;
366
367 if (srcCompressed)
368 {
369 if (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG)
370 {
371 if (!freerdp_video_decode_mjpeg(context, (const BYTE*)srcSampleData, srcSampleLength,
372 intermediate_data, intermediate_linesize,
373 &intermediate_format))
374 {
375 WLog_Print(context->log, WLOG_ERROR, "MJPEG decoding failed");
376 return FALSE;
377 }
378 }
379 else if (srcFormat == FREERDP_VIDEO_FORMAT_H264)
380 {
381 WLog_Print(context->log, WLOG_ERROR, "H264 decoding not supported");
382 return FALSE;
383 }
384 }
385 else
386 {
387 intermediate_format = video_format_to_av(context->log, srcFormat);
388 }
389
390 if (dstFormat == FREERDP_VIDEO_FORMAT_H264)
391 {
392 if (!context->h264Configured)
393 {
394 WLog_Print(context->log, WLOG_ERROR, "H264 encoder not configured");
395 return FALSE;
396 }
397
398 const UINT32 hwAccel = h264_context_get_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL);
399 const FREERDP_VIDEO_FORMAT yuvFormat =
400 hwAccel ? FREERDP_VIDEO_FORMAT_NV12 : FREERDP_VIDEO_FORMAT_YUV420P;
401
402 BYTE* yuvData[3] = WINPR_C_ARRAY_INIT;
403 UINT32 yuvStrides[3] = WINPR_C_ARRAY_INIT;
404
405 if (h264_get_yuv_buffer(context->h264, 0, context->width, context->height, yuvData,
406 yuvStrides) < 0)
407 {
408 WLog_Print(context->log, WLOG_ERROR, "h264_get_yuv_buffer failed");
409 return FALSE;
410 }
411
412 int yuvLineSizes[4] = { (int)yuvStrides[0], (int)yuvStrides[1], (int)yuvStrides[2], 0 };
413
414 if (srcCompressed)
415 {
416 const BYTE* cIntermediateData[4] = { intermediate_data[0], intermediate_data[1],
417 intermediate_data[2], intermediate_data[3] };
418 if (!freerdp_video_convert_to_yuv(context, cIntermediateData, intermediate_linesize,
419 intermediate_format, yuvData, yuvLineSizes, yuvFormat,
420 context->width, context->height))
421 {
422 WLog_Print(context->log, WLOG_ERROR, "YUV conversion failed");
423 return FALSE;
424 }
425 }
426 else
427 {
428 BYTE* srcPlanes[4] = WINPR_C_ARRAY_INIT;
429 int srcStrides[4] = WINPR_C_ARRAY_INIT;
430
431 if (!freerdp_video_fill_plane_info(context->log, srcPlanes, srcStrides, srcFormat,
432 context->width, context->height,
433 (const BYTE*)srcSampleData))
434 {
435 WLog_Print(context->log, WLOG_ERROR, "Failed to fill plane info");
436 return FALSE;
437 }
438
439 const BYTE* cSrcPlanes[4] = { srcPlanes[0], srcPlanes[1], srcPlanes[2], srcPlanes[3] };
440 if (!freerdp_video_convert_to_yuv(
441 context, cSrcPlanes, srcStrides, video_format_to_av(context->log, srcFormat),
442 yuvData, yuvLineSizes, yuvFormat, context->width, context->height))
443 {
444 WLog_Print(context->log, WLOG_ERROR, "YUV conversion failed");
445 return FALSE;
446 }
447 }
448
449 BYTE* h264Data = nullptr;
450 UINT32 h264Size = 0;
451
452 if (h264_compress(context->h264, &h264Data, &h264Size) < 0)
453 {
454 WLog_Print(context->log, WLOG_ERROR, "H264 compression failed");
455 return FALSE;
456 }
457
458 if (!Stream_EnsureRemainingCapacity(output, h264Size))
459 {
460 WLog_Print(context->log, WLOG_ERROR, "Failed to ensure stream capacity");
461 return FALSE;
462 }
463
464 Stream_Write(output, h264Data, h264Size);
465 return TRUE;
466 }
467 else if (dstCompressed)
468 {
469 WLog_Print(context->log, WLOG_ERROR, "Only H264 encoding is supported");
470 return FALSE;
471 }
472
473 WLog_Print(context->log, WLOG_ERROR, "Raw format output not yet implemented");
474 return FALSE;
475}
476
477static BOOL freerdp_video_decode_mjpeg(FREERDP_VIDEO_CONTEXT* context, const BYTE* srcData,
478 size_t srcSize, BYTE* dstData[4], int dstLineSize[4],
479 enum AVPixelFormat* dstFormat)
480{
481#if defined(WITH_MJPEG_DECODER)
482 WINPR_ASSERT(context);
483 WINPR_ASSERT(srcData);
484 WINPR_ASSERT(dstData);
485 WINPR_ASSERT(dstLineSize);
486 WINPR_ASSERT(dstFormat);
487
488 if (!context->mjpegDecoder)
489 {
490 WLog_Print(context->log, WLOG_ERROR, "MJPEG decoder not initialized");
491 return FALSE;
492 }
493
494 context->mjpegPacket->data = WINPR_CAST_CONST_PTR_AWAY(srcData, uint8_t*);
495 WINPR_ASSERT(srcSize <= INT32_MAX);
496 context->mjpegPacket->size = (int)srcSize;
497
498 if (avcodec_send_packet(context->mjpegDecoder, context->mjpegPacket) < 0)
499 {
500 WLog_Print(context->log, WLOG_ERROR, "avcodec_send_packet failed");
501 return FALSE;
502 }
503
504 if (avcodec_receive_frame(context->mjpegDecoder, context->mjpegFrame) < 0)
505 {
506 WLog_Print(context->log, WLOG_ERROR, "avcodec_receive_frame failed");
507 return FALSE;
508 }
509
510 /* Copy plane pointers and line sizes */
511 for (size_t i = 0; i < 4; i++)
512 {
513 dstData[i] = context->mjpegFrame->data[i];
514 dstLineSize[i] = context->mjpegFrame->linesize[i];
515 }
516
517 /* Convert pixel format */
518 *dstFormat = context->mjpegDecoder->pix_fmt;
519
520 return TRUE;
521#else
522 WINPR_UNUSED(context);
523 WINPR_UNUSED(srcData);
524 WINPR_UNUSED(srcSize);
525 WINPR_UNUSED(dstData);
526 WINPR_UNUSED(dstLineSize);
527 WINPR_UNUSED(dstFormat);
528 WLog_Print(context->log, WLOG_ERROR,
529 "MJPEG decoder not available (requires direct FFmpeg linking)");
530 return FALSE;
531#endif
532}
533
534static BOOL freerdp_video_convert_to_yuv(FREERDP_VIDEO_CONTEXT* context, const BYTE* srcData[4],
535 const int srcLineSize[4], enum AVPixelFormat srcFormat,
536 BYTE* dstData[3], const int dstLineSize[3],
537 FREERDP_VIDEO_FORMAT dstFormat, UINT32 width,
538 UINT32 height)
539{
540 WINPR_ASSERT(context);
541 WINPR_ASSERT(srcData);
542 WINPR_ASSERT(srcLineSize);
543 WINPR_ASSERT(dstData);
544 WINPR_ASSERT(dstLineSize);
545
546 if (!freerdp_swscale_available())
547 {
548 WLog_Print(context->log, WLOG_ERROR,
549 "swscale not available - install FFmpeg to enable video processing");
550 return FALSE;
551 }
552
553 enum AVPixelFormat srcPixFmt = srcFormat;
554 enum AVPixelFormat dstPixFmt = video_format_to_av(context->log, dstFormat);
555
556 if (srcPixFmt == AV_PIX_FMT_NONE || dstPixFmt == AV_PIX_FMT_NONE)
557 {
558 WLog_Print(context->log, WLOG_ERROR, "Unsupported pixel format");
559 return FALSE;
560 }
561
562 /* Create or reuse swscale context */
563 struct SwsContext* sws = nullptr;
564 BOOL needFree = FALSE;
565
566 if (context && context->sws)
567 {
568 sws = context->sws;
569 }
570 else
571 {
572 sws = freerdp_sws_getContext((int)width, (int)height, srcPixFmt, (int)width, (int)height,
573 dstPixFmt, 0, nullptr, nullptr, nullptr);
574 if (!sws)
575 {
576 WLog_Print(context->log, WLOG_ERROR, "sws_getContext failed");
577 return FALSE;
578 }
579
580 if (context)
581 context->sws = sws;
582 else
583 needFree = TRUE;
584 }
585
586 /* Perform conversion */
587 const BYTE* cSrcData[4] = { srcData[0], srcData[1], srcData[2], srcData[3] };
588
589 /* sws_scale expects 4-element arrays, but caller may provide 3-element arrays for YUV */
590 uint8_t* localDstData[4] = { dstData[0], dstData[1], dstData[2], nullptr };
591 int localDstLineSize[4] = { dstLineSize[0], dstLineSize[1], dstLineSize[2], 0 };
592
593 const int result = freerdp_sws_scale(sws, cSrcData, srcLineSize, 0, (int)height, localDstData,
594 localDstLineSize);
595
596 if (needFree)
597 {
598 freerdp_sws_freeContext(sws);
599 }
600
601 return (result > 0);
602}
603
604static BOOL freerdp_video_fill_plane_info(wLog* log, BYTE* data[4], int lineSize[4],
605 FREERDP_VIDEO_FORMAT format, UINT32 width, UINT32 height,
606 const BYTE* buffer)
607{
608 WINPR_ASSERT(data);
609 WINPR_ASSERT(lineSize);
610
611 enum AVPixelFormat pixFmt = video_format_to_av(log, format);
612 if (pixFmt == AV_PIX_FMT_NONE)
613 {
614 WLog_Print(log, WLOG_ERROR, "Unsupported pixel format");
615 return FALSE;
616 }
617
618 if (!freerdp_avutil_available())
619 {
620 WLog_Print(log, WLOG_ERROR, "avutil not available");
621 return FALSE;
622 }
623
624 if (freerdp_av_image_fill_linesizes(lineSize, pixFmt, (int)width) < 0)
625 {
626 WLog_Print(log, WLOG_ERROR, "av_image_fill_linesizes failed");
627 return FALSE;
628 }
629
630 if (freerdp_av_image_fill_pointers(data, pixFmt, (int)height,
631 WINPR_CAST_CONST_PTR_AWAY(buffer, BYTE*), lineSize) < 0)
632 {
633 WLog_Print(log, WLOG_ERROR, "av_image_fill_pointers failed");
634 return FALSE;
635 }
636
637 return TRUE;
638}
639
640BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
641 FREERDP_VIDEO_FORMAT dstFormat)
642{
643 if (srcFormat == dstFormat)
644 return TRUE;
645
646 if (!freerdp_video_available())
647 return FALSE;
648
649 if (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG)
650 {
651#if !defined(WITH_MJPEG_DECODER)
652 return FALSE;
653#endif
654 }
655 else if (srcFormat == FREERDP_VIDEO_FORMAT_H264)
656 {
657 return FALSE;
658 }
659
660 if (dstFormat == FREERDP_VIDEO_FORMAT_MJPEG)
661 {
662 return FALSE;
663 }
664 else if (dstFormat == FREERDP_VIDEO_FORMAT_H264)
665 {
666#if !defined(WITH_OPENH264) && !defined(WITH_VIDEO_FFMPEG) && !defined(WITH_MEDIA_FOUNDATION) && \
667 !defined(WITH_MEDIACODEC) && !defined(WITH_APPLE_VIDEOTOOLBOX)
668 return FALSE;
669#endif
670 }
671
672 return TRUE;
673}
674
675#elif defined(WITH_MEDIACODEC) || defined(WITH_APPLE_VIDEOTOOLBOX) /* !WITH_SWSCALE */
676
677/*
678 * Without swscale, the H.264 encoder of MediaCodec (h264_mediacodec.c) still encodes the frames of
679 * the camera of Android (channels/rdpecam/client/android), which come as NV12: their planes are
680 * split here into the three of I420, in the buffer of h264_get_yuv_buffer, and h264_compress hands
681 * them to the encoder. Nothing else is converted.
682 * The same applies to VideoToolbox (h264_videotoolbox.c) and the camera of AVFoundation
683 * (channels/rdpecam/client/avfoundation).
684 */
685
686/* The longest side that the encoder of MediaCodec takes (MEDIACODEC_MAXIMUM_DIMENSION of
687 * h264_mediacodec.c). */
688#define VIDEO_NV12_MAX_DIMENSION 16384
689
690struct s_FREERDP_VIDEO_CONTEXT
691{
692 UINT32 width;
693 UINT32 height;
694 H264_CONTEXT* h264;
695 BOOL h264Configured;
696 wLog* log;
697};
698
699BOOL freerdp_video_available(void)
700{
701 /* FFmpeg is not loaded: NV12 to H.264 is all there is (freerdp_video_conversion_supported) */
702 return FALSE;
703}
704
705FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(UINT32 width, UINT32 height)
706{
707 FREERDP_VIDEO_CONTEXT* context =
708 (FREERDP_VIDEO_CONTEXT*)calloc(1, sizeof(FREERDP_VIDEO_CONTEXT));
709 if (!context)
710 return nullptr;
711
712 context->log = WLog_Get(VTAG);
713 context->width = width;
714 context->height = height;
715 return context;
716}
717
718void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
719{
720 if (!context)
721 return;
722
723 h264_context_free(context->h264);
724 free(context);
725}
726
727BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
728 FREERDP_VIDEO_FORMAT dstFormat)
729{
730 if (srcFormat == dstFormat)
731 return TRUE;
732
733 return (srcFormat == FREERDP_VIDEO_FORMAT_NV12) && (dstFormat == FREERDP_VIDEO_FORMAT_H264);
734}
735
736BOOL freerdp_video_context_reconfigure(FREERDP_VIDEO_CONTEXT* context, UINT32 width, UINT32 height,
737 UINT32 framerate, UINT32 bitrate, UINT32 usageType)
738{
739 if (!context)
740 return FALSE;
741
742 /* NV12 shares one pair of chroma samples between two lines and two columns */
743 if ((width == 0) || (height == 0) || ((width % 2) != 0) || ((height % 2) != 0) ||
744 (width > VIDEO_NV12_MAX_DIMENSION) || (height > VIDEO_NV12_MAX_DIMENSION))
745 {
746 WLog_Print(context->log, WLOG_ERROR, "Invalid dimensions: %" PRIu32 "x%" PRIu32, width,
747 height);
748 return FALSE;
749 }
750
751 if (bitrate == 0)
752 bitrate = video_get_h264_bitrate(context->log, height);
753
754 /*
755 * Each sample reconfigures (ecam_encoder_compress): the encoder starts again on a new size
756 * only, and then as a new codec. h264_context_reset alone sets the new size and keeps the
757 * planes of h264_get_yuv_buffer, which yuv_ensure_buffer then makes again for a new stride
758 * only: a frame of the same width and more lines would be written past their end.
759 */
760 const BOOL newSize =
761 !context->h264Configured || (context->width != width) || (context->height != height);
762 if (newSize)
763 {
764 h264_context_free(context->h264);
765 context->h264 = nullptr;
766 context->h264Configured = FALSE;
767 }
768
769 if (!context->h264)
770 {
771 context->h264 = h264_context_new(TRUE);
772 if (!context->h264)
773 {
774 WLog_Print(context->log, WLOG_ERROR, "h264_context_new failed");
775 return FALSE;
776 }
777 }
778
779 /* HW_ACCEL off: the three planes of I420, which mediacodec_compress takes;
780 * videotoolbox_compress also takes these planes */
781 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_USAGETYPE, usageType) ||
782 !h264_context_set_option(context->h264, H264_CONTEXT_OPTION_FRAMERATE, framerate) ||
783 !h264_context_set_option(context->h264, H264_CONTEXT_OPTION_BITRATE, bitrate) ||
784 !h264_context_set_option(context->h264, H264_CONTEXT_OPTION_RATECONTROL,
785 H264_RATECONTROL_CQP) ||
786 !h264_context_set_option(context->h264, H264_CONTEXT_OPTION_QP, 26) ||
787 !h264_context_set_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL, FALSE))
788 goto fail;
789
790 if (newSize)
791 {
792 if (!h264_context_reset(context->h264, width, height))
793 {
794 WLog_Print(context->log, WLOG_ERROR, "h264_context_reset failed");
795 goto fail;
796 }
797
798 WLog_Print(context->log, WLOG_INFO,
799 "camera frames encoded to H.264: %" PRIu32 "x%" PRIu32 ", %" PRIu32
800 " fps, %" PRIu32 " bps",
801 width, height, framerate, bitrate);
802 }
803
804 context->h264Configured = TRUE;
805 context->width = width;
806 context->height = height;
807 return TRUE;
808
809fail:
810 h264_context_free(context->h264);
811 context->h264 = nullptr;
812 context->h264Configured = FALSE;
813 return FALSE;
814}
815
816/* The Y plane, then the interleaved UV plane of NV12, into the three planes of I420 */
817static void freerdp_video_nv12_to_i420(const BYTE* nv12, UINT32 width, UINT32 height, BYTE* yuv[3],
818 const UINT32 strides[3])
819{
820 for (UINT32 y = 0; y < height; y++)
821 memcpy(&yuv[0][(size_t)y * strides[0]], &nv12[(size_t)y * width], width);
822
823 const BYTE* uv = &nv12[(size_t)width * height];
824 for (UINT32 y = 0; y < height / 2; y++)
825 {
826 const BYTE* src = &uv[(size_t)y * width];
827 BYTE* u = &yuv[1][(size_t)y * strides[1]];
828 BYTE* v = &yuv[2][(size_t)y * strides[2]];
829 for (UINT32 x = 0; x < width / 2; x++)
830 {
831 u[x] = src[2 * (size_t)x];
832 v[x] = src[2 * (size_t)x + 1];
833 }
834 }
835}
836
837BOOL freerdp_video_sample_convert(FREERDP_VIDEO_CONTEXT* context, FREERDP_VIDEO_FORMAT srcFormat,
838 const void* srcSampleData, size_t srcSampleLength,
839 FREERDP_VIDEO_FORMAT dstFormat, wStream* output)
840{
841 if (!context || !srcSampleData || !output)
842 return FALSE;
843
844 if (srcFormat == dstFormat)
845 {
846 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
847 return FALSE;
848
849 Stream_Write(output, srcSampleData, srcSampleLength);
850 return TRUE;
851 }
852
853 if (!freerdp_video_conversion_supported(srcFormat, dstFormat))
854 {
855 WLog_Print(context->log, WLOG_ERROR, "Conversion from %s to %s not supported",
856 freerdp_video_format_string(srcFormat), freerdp_video_format_string(dstFormat));
857 return FALSE;
858 }
859
860 if (!context->h264Configured || !context->h264)
861 {
862 WLog_Print(context->log, WLOG_ERROR, "H264 encoder not configured");
863 return FALSE;
864 }
865
866 /* The sides are even and bounded (freerdp_video_context_reconfigure): no overflow */
867 const size_t frameSize = (size_t)context->width * context->height * 3 / 2;
868 if (srcSampleLength < frameSize)
869 {
870 WLog_Print(context->log, WLOG_ERROR,
871 "NV12 sample of %" PRIuz " bytes, %" PRIuz " expected for %" PRIu32 "x%" PRIu32,
872 srcSampleLength, frameSize, context->width, context->height);
873 return FALSE;
874 }
875
876 /* A stride keeps the width as it is: with 0, yuv_ensure_buffer (h264.c) rounds it up to 16 */
877 BYTE* yuvData[3] = WINPR_C_ARRAY_INIT;
878 UINT32 yuvStrides[3] = WINPR_C_ARRAY_INIT;
879 if ((h264_get_yuv_buffer(context->h264, context->width, context->width, context->height,
880 yuvData, yuvStrides) < 0) ||
881 !yuvData[0] || !yuvData[1] || !yuvData[2] || (yuvStrides[0] < context->width) ||
882 (yuvStrides[1] < context->width / 2) || (yuvStrides[2] < context->width / 2))
883 {
884 WLog_Print(context->log, WLOG_ERROR, "h264_get_yuv_buffer failed");
885 return FALSE;
886 }
887
888 freerdp_video_nv12_to_i420((const BYTE*)srcSampleData, context->width, context->height, yuvData,
889 yuvStrides);
890
891 BYTE* h264Data = nullptr;
892 UINT32 h264Size = 0;
893 if ((h264_compress(context->h264, &h264Data, &h264Size) < 0) || !h264Data || (h264Size == 0))
894 {
895 /* MediaCodec gives no frame while its pipeline fills, nor for a frame it drops */
896 WLog_Print(context->log, WLOG_DEBUG, "No H.264 frame out of this NV12 sample");
897 return FALSE;
898 }
899
900 if (!Stream_EnsureRemainingCapacity(output, h264Size))
901 return FALSE;
902
903 Stream_Write(output, h264Data, h264Size);
904 return TRUE;
905}
906
907#else /* !WITH_SWSCALE && !WITH_MEDIACODEC && !WITH_APPLE_VIDEOTOOLBOX */
908
909/* Stubs when swscale is not available */
910
911BOOL freerdp_video_available(void)
912{
913 return FALSE;
914}
915
916FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(WINPR_ATTR_UNUSED UINT32 width,
917 WINPR_ATTR_UNUSED UINT32 height)
918{
919 WINPR_UNUSED(width);
920 WINPR_UNUSED(height);
921 return calloc(1, sizeof(char));
922}
923
924void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
925{
926 free(context);
927}
928
929BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
930 FREERDP_VIDEO_FORMAT dstFormat)
931{
932 if (srcFormat == dstFormat)
933 return TRUE;
934 return FALSE;
935}
936
937BOOL freerdp_video_context_reconfigure(WINPR_ATTR_UNUSED FREERDP_VIDEO_CONTEXT* context,
938 WINPR_ATTR_UNUSED UINT32 width,
939 WINPR_ATTR_UNUSED UINT32 height,
940 WINPR_ATTR_UNUSED UINT32 framerate,
941 WINPR_ATTR_UNUSED UINT32 bitrate,
942 WINPR_ATTR_UNUSED UINT32 usageType)
943{
944 return TRUE;
945}
946
947BOOL freerdp_video_sample_convert(WINPR_ATTR_UNUSED FREERDP_VIDEO_CONTEXT* context,
948 FREERDP_VIDEO_FORMAT srcFormat, const void* srcSampleData,
949 size_t srcSampleLength, FREERDP_VIDEO_FORMAT dstFormat,
950 wStream* output)
951{
952 if (srcFormat == dstFormat)
953 {
954 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
955 return FALSE;
956
957 Stream_Write(output, srcSampleData, srcSampleLength);
958 return TRUE;
959 }
960
961 return FALSE;
962}
963
964#endif /* WITH_SWSCALE, WITH_MEDIACODEC */
965
966const char* freerdp_video_format_string(UINT32 format)
967{
968#define EVCASE(x) \
969 case x: \
970 return #x
971
972 switch (format)
973 {
974 EVCASE(FREERDP_VIDEO_FORMAT_NONE);
975 EVCASE(FREERDP_VIDEO_FORMAT_MJPEG);
976 EVCASE(FREERDP_VIDEO_FORMAT_H264);
977 EVCASE(FREERDP_VIDEO_FORMAT_YUV420P);
978 EVCASE(FREERDP_VIDEO_FORMAT_NV12);
979 EVCASE(FREERDP_VIDEO_FORMAT_YUYV422);
980 EVCASE(FREERDP_VIDEO_FORMAT_RGB24);
981 EVCASE(FREERDP_VIDEO_FORMAT_RGB32);
982 default:
983 return "FREERDP_VIDEO_FORMAT_UNKNOWN";
984 }
985#undef EVCASE
986}