FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <freerdp/log.h>
25#include <freerdp/codec/h264.h>
26#include <libavcodec/avcodec.h>
27#include <libavutil/opt.h>
28
29#include "h264.h"
30
31#ifdef WITH_VIDEOTOOLBOX
32#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
33#include <libavutil/hwcontext.h>
34#else
35#pragma warning You have asked for VideoToolbox decoding, \
36 but your version of libavutil is too old !Disabling.
37#undef WITH_VIDEOTOOLBOX
38#endif
39#endif
40
41#ifdef WITH_VAAPI
42#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
43#include <libavutil/hwcontext.h>
44#else
45#pragma warning You have asked for VA - API decoding, \
46 but your version of libavutil is too old !Disabling.
47#undef WITH_VAAPI
48#endif
49#endif
50
51/* Fallback support for older libavcodec versions */
52#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
53#define AV_CODEC_ID_H264 CODEC_ID_H264
54#endif
55
56#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
57#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
58#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
59#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
60#endif
61
62#if LIBAVUTIL_VERSION_MAJOR < 52
63#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
64#endif
65
66/* Ubuntu 14.04 ships without the functions provided by avutil,
67 * so define error to string methods here. */
68#if !defined(av_err2str)
69static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
70{
71 av_strerror(errnum, errbuf, errbuf_size);
72 return errbuf;
73}
74
75#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
76#endif
77
78#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING)
79static const char* get_vaapi_device(void)
80{
81 static char device[MAX_PATH] = WINPR_C_ARRAY_INIT;
82 static bool initialized = false;
83 if (!initialized)
84 {
85 initialized = true;
86 // NOLINTNEXTLINE(concurrency-mt-unsafe)
87 const char* env = getenv("FREERDP_VAAPI_DEVICE");
88 if (env)
89 (void)_snprintf(device, sizeof(device), "%s", env);
90 else
91 (void)_snprintf(device, sizeof(device), "/dev/dri/renderD128");
92 }
93 return device;
94}
95#endif
96
97typedef struct
98{
99 const AVCodec* codecDecoder;
100 AVCodecContext* codecDecoderContext;
101 const AVCodec* codecEncoder;
102 AVCodecContext* codecEncoderContext;
103 AVCodecParserContext* codecParser;
104 AVFrame* videoFrame;
105#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
106 AVPacket bufferpacket;
107#endif
108 AVPacket* packet;
109#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
110 AVBufferRef* hwctx;
111 AVFrame* hwVideoFrame;
112 enum AVPixelFormat hw_pix_fmt;
113#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
114 AVBufferRef* hw_frames_ctx;
115#endif
116
117#endif
118} H264_CONTEXT_LIBAVCODEC;
119
120static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
121{
122 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
123
124 if (!h264 || !h264->subsystem)
125 return;
126
127 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
128
129 if (sys->codecEncoderContext)
130 {
131#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
132 avcodec_free_context(&sys->codecEncoderContext);
133#else
134 avcodec_close(sys->codecEncoderContext);
135 av_free(sys->codecEncoderContext);
136#endif
137 }
138
139 sys->codecEncoderContext = nullptr;
140}
141
142#ifdef WITH_VAAPI_H264_ENCODING
143static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
144{
145 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
146 AVBufferRef* hw_frames_ref = nullptr;
147 AVHWFramesContext* frames_ctx = nullptr;
148 int err = 0;
149
150 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
151 {
152 WLog_Print(h264->log, WLOG_ERROR, "Failed to create VAAPI frame context");
153 return -1;
154 }
155 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
156 frames_ctx->format = AV_PIX_FMT_VAAPI;
157 frames_ctx->sw_format = AV_PIX_FMT_NV12;
158 frames_ctx->width = sys->codecEncoderContext->width;
159 frames_ctx->height = sys->codecEncoderContext->height;
160 frames_ctx->initial_pool_size = 20;
161 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
162 {
163 WLog_Print(h264->log, WLOG_ERROR,
164 "Failed to initialize VAAPI frame context."
165 "Error code: %s",
166 av_err2str(err));
167 av_buffer_unref(&hw_frames_ref);
168 return err;
169 }
170 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
171 if (!sys->codecEncoderContext->hw_frames_ctx)
172 err = AVERROR(ENOMEM);
173
174 av_buffer_unref(&hw_frames_ref);
175 return err;
176}
177#endif
178
179static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
180{
181 BOOL recreate = FALSE;
182 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
183
184 if (!h264 || !h264->subsystem)
185 return FALSE;
186
187 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
188 return FALSE;
189
190 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
191 if (!sys || !sys->codecEncoder)
192 return FALSE;
193
194 recreate = !sys->codecEncoderContext;
195
196 if (sys->codecEncoderContext)
197 {
198 if ((sys->codecEncoderContext->width != (int)h264->width) ||
199 (sys->codecEncoderContext->height != (int)h264->height))
200 recreate = TRUE;
201 }
202
203 if (!recreate)
204 return TRUE;
205
206 libavcodec_destroy_encoder_context(h264);
207
208 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
209
210 if (!sys->codecEncoderContext)
211 goto EXCEPTION;
212
213 switch (h264->RateControlMode)
214 {
215 case H264_RATECONTROL_VBR:
216 sys->codecEncoderContext->bit_rate = h264->BitRate;
217 break;
218
219 case H264_RATECONTROL_CQP:
220 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
221 0)
222 {
223 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
224 }
225 break;
226
227 default:
228 break;
229 }
230
231 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
232 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
233 sys->codecEncoderContext->delay = 0;
234#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
235 sys->codecEncoderContext->framerate =
236 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
237#endif
238 sys->codecEncoderContext->time_base =
239 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
240 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
241
242 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
243
244#ifdef WITH_VAAPI_H264_ENCODING
245 if (sys->hwctx)
246 {
247 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
248
249 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_VAAPI;
250 /* set hw_frames_ctx for encoder's AVCodecContext */
251 if (set_hw_frames_ctx(h264) < 0)
252 goto EXCEPTION;
253 }
254 else
255#endif
256 {
257 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
258 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
259 }
260
261 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
262 goto EXCEPTION;
263
264 return TRUE;
265EXCEPTION:
266 libavcodec_destroy_encoder_context(h264);
267 return FALSE;
268}
269
270static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
271 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
272{
273 union
274 {
275 const BYTE* cpv;
276 BYTE* pv;
277 } cnv;
278 int rc = -1;
279 int status = 0;
280 int gotFrame = 0;
281 AVPacket* packet = nullptr;
282
283 WINPR_ASSERT(h264);
284 WINPR_ASSERT(pSrcData || (SrcSize == 0));
285
286 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
287 BYTE** pYUVData = h264->pYUVData;
288 UINT32* iStride = h264->iStride;
289
290 WINPR_ASSERT(sys);
291
292#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
293 packet = &sys->bufferpacket;
294 WINPR_ASSERT(packet);
295 av_init_packet(packet);
296#else
297 packet = av_packet_alloc();
298#endif
299 if (!packet)
300 {
301 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
302 goto fail;
303 }
304
305 cnv.cpv = pSrcData;
306 packet->data = cnv.pv;
307 packet->size = (int)MIN(SrcSize, INT32_MAX);
308
309 WINPR_ASSERT(sys->codecDecoderContext);
310 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
311#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
312 status = avcodec_send_packet(sys->codecDecoderContext, packet);
313
314 if (status < 0)
315 {
316 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
317 goto fail;
318 }
319
320 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
321
322#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
323 status = avcodec_receive_frame(sys->codecDecoderContext,
324 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
325#else
326 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
327#endif
328 if (status == AVERROR(EAGAIN))
329 {
330 rc = 0;
331 goto fail;
332 }
333
334 if (status == 0)
335 gotFrame = 1;
336#else
337#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
338 status =
339 avcodec_decode_video2(sys->codecDecoderContext,
340 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
341#else
342 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
343#endif
344#endif
345 if (status < 0)
346 {
347 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
348 goto fail;
349 }
350
351#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
352
353 if (sys->hwctx)
354 {
355 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
356 {
357 sys->videoFrame->width = sys->hwVideoFrame->width;
358 sys->videoFrame->height = sys->hwVideoFrame->height;
359 status = av_hwframe_transfer_data(sys->videoFrame, sys->hwVideoFrame, 0);
360 }
361 else
362 {
363 status = av_frame_copy(sys->videoFrame, sys->hwVideoFrame);
364 }
365 gotFrame = (status == 0);
366
367 if (status < 0)
368 {
369 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)",
370 status, av_err2str(status));
371 goto fail;
372 }
373 }
374
375#endif
376
377 if (gotFrame)
378 {
379 WINPR_ASSERT(sys->videoFrame);
380
381 pYUVData[0] = sys->videoFrame->data[0];
382 pYUVData[1] = sys->videoFrame->data[1];
383 pYUVData[2] = sys->videoFrame->data[2];
384 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
385 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
386 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
387
388 h264->YUVWidth = WINPR_ASSERTING_INT_CAST(UINT32, sys->videoFrame->width);
389 h264->YUVHeight = WINPR_ASSERTING_INT_CAST(UINT32, sys->videoFrame->height);
390 rc = 1;
391 }
392 else
393 rc = -2;
394
395fail:
396#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
397 av_packet_unref(packet);
398#else
399 av_packet_free(&packet);
400#endif
401
402 return rc;
403}
404
405static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
406 const BYTE** WINPR_RESTRICT pSrcYuv,
407 const UINT32* WINPR_RESTRICT pStride,
408 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
409{
410 union
411 {
412 const BYTE* cpv;
413 uint8_t* pv;
414 } cnv;
415 int rc = -1;
416 int status = 0;
417 int gotFrame = 0;
418
419 WINPR_ASSERT(h264);
420
421 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
422 WINPR_ASSERT(sys);
423
424 if (!libavcodec_create_encoder_context(h264))
425 return -1;
426
427#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
428 sys->packet = &sys->bufferpacket;
429 av_packet_unref(sys->packet);
430 av_init_packet(sys->packet);
431#else
432 av_packet_free(&sys->packet);
433 sys->packet = av_packet_alloc();
434#endif
435 if (!sys->packet)
436 {
437 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
438 goto fail;
439 }
440
441 WINPR_ASSERT(sys->packet);
442 sys->packet->data = nullptr;
443 sys->packet->size = 0;
444
445 WINPR_ASSERT(sys->videoFrame);
446 WINPR_ASSERT(sys->codecEncoderContext);
447 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
448 sys->videoFrame->width = sys->codecEncoderContext->width;
449 sys->videoFrame->height = sys->codecEncoderContext->height;
450#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
451 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
452#endif
453#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
454 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
455#endif
456 cnv.cpv = pSrcYuv[0];
457 sys->videoFrame->data[0] = cnv.pv;
458
459 cnv.cpv = pSrcYuv[1];
460 sys->videoFrame->data[1] = cnv.pv;
461
462 cnv.cpv = pSrcYuv[2];
463 sys->videoFrame->data[2] = cnv.pv;
464
465 sys->videoFrame->linesize[0] = (int)pStride[0];
466 sys->videoFrame->linesize[1] = (int)pStride[1];
467 sys->videoFrame->linesize[2] = (int)pStride[2];
468 sys->videoFrame->pts++;
469
470#ifdef WITH_VAAPI_H264_ENCODING
471 if (sys->hwctx)
472 {
473 av_frame_unref(sys->hwVideoFrame);
474 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
475 sys->hwVideoFrame, 0)) < 0 ||
476 !sys->hwVideoFrame->hw_frames_ctx)
477 {
478 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
479 av_err2str(status), status);
480 goto fail;
481 }
482 sys->videoFrame->format = AV_PIX_FMT_NV12;
483 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
484 {
485 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
486 av_err2str(status), status);
487 goto fail;
488 }
489 }
490#endif
491
492 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
493#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
494#ifdef WITH_VAAPI_H264_ENCODING
495 status = avcodec_send_frame(sys->codecEncoderContext,
496 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
497#else
498 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
499#endif
500
501 if (status < 0)
502 {
503 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
504 av_err2str(status), status);
505 goto fail;
506 }
507
508 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
509
510 if (status < 0)
511 {
512 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
513 av_err2str(status), status);
514 goto fail;
515 }
516
517 gotFrame = (status == 0);
518#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
519
520 do
521 {
522 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
523 &gotFrame);
524 } while ((status >= 0) && (gotFrame == 0));
525
526#else
527 sys->packet->size =
528 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
529 sys->codecDecoderContext->height);
530 sys->packet->data = av_malloc(sys->packet->size);
531
532 if (!sys->packet->data)
533 status = -1;
534 else
535 {
536 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
537 sys->packet->size, sys->videoFrame);
538 }
539
540#endif
541
542 if (status < 0)
543 {
544 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
545 av_err2str(status), status);
546 goto fail;
547 }
548
549 WINPR_ASSERT(sys->packet);
550 *ppDstData = sys->packet->data;
551 *pDstSize = (UINT32)MAX(0, sys->packet->size);
552
553 if (!gotFrame)
554 {
555 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
556 status);
557 rc = -2;
558 }
559 else
560 rc = 1;
561fail:
562 return rc;
563}
564
565static void libavcodec_uninit(H264_CONTEXT* h264)
566{
567 WINPR_ASSERT(h264);
568
569 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
570
571 if (!sys)
572 return;
573
574 if (sys->packet)
575 {
576#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
577 av_packet_unref(sys->packet);
578#else
579 av_packet_free(&sys->packet);
580#endif
581 }
582
583 if (sys->videoFrame)
584 {
585#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
586 av_frame_free(&sys->videoFrame);
587#else
588 av_free(sys->videoFrame);
589#endif
590 }
591
592#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
593 if (sys->hwVideoFrame)
594 {
595#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
596 av_frame_free(&sys->hwVideoFrame);
597#else
598 av_free(sys->hwVideoFrame);
599#endif
600 }
601
602 if (sys->hwctx)
603 av_buffer_unref(&sys->hwctx);
604
605#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
606
607 if (sys->hw_frames_ctx)
608 av_buffer_unref(&sys->hw_frames_ctx);
609
610#endif
611
612#endif
613
614 if (sys->codecParser)
615 av_parser_close(sys->codecParser);
616
617 if (sys->codecDecoderContext)
618 {
619#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
620 avcodec_free_context(&sys->codecDecoderContext);
621#else
622 avcodec_close(sys->codecDecoderContext);
623 av_free(sys->codecDecoderContext);
624#endif
625 }
626
627 libavcodec_destroy_encoder_context(h264);
628 free(sys);
629 h264->pSystemData = nullptr;
630}
631
632#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
633static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
634 const enum AVPixelFormat* fmts)
635{
636 WINPR_ASSERT(ctx);
637
638 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
639 WINPR_ASSERT(h264);
640
641 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
642 WINPR_ASSERT(sys);
643
644 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
645 {
646 if (*p == sys->hw_pix_fmt)
647 {
648#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
649 if (sys->hw_frames_ctx)
650 av_buffer_unref(&sys->hw_frames_ctx);
651
652 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
653
654 if (!sys->hw_frames_ctx)
655 {
656 return AV_PIX_FMT_NONE;
657 }
658
659 sys->codecDecoderContext->pix_fmt = *p;
660 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
661 frames->format = *p;
662 frames->height = sys->codecDecoderContext->coded_height;
663 frames->width = sys->codecDecoderContext->coded_width;
664#ifdef WITH_VIDEOTOOLBOX
665 frames->sw_format = AV_PIX_FMT_YUV420P;
666#else
667 frames->sw_format =
668 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
669 : AV_PIX_FMT_NV12);
670#endif
671 frames->initial_pool_size = 20;
672
673 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
674 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
675
676 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
677
678 if (err < 0)
679 {
680 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
681 av_err2str(err));
682 return AV_PIX_FMT_NONE;
683 }
684
685 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
686#endif
687 return *p;
688 }
689 }
690
691 return AV_PIX_FMT_NONE;
692}
693#endif
694
695static BOOL libavcodec_init(H264_CONTEXT* h264)
696{
697 WINPR_ASSERT(h264);
698 H264_CONTEXT_LIBAVCODEC* sys =
699 (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
700
701 if (!sys)
702 {
703 goto EXCEPTION;
704 }
705
706 h264->pSystemData = (void*)sys;
707
708#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
709 avcodec_register_all();
710#endif
711
712 if (!h264->Compressor)
713 {
714 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
715
716 if (!sys->codecDecoder)
717 {
718 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
719 goto EXCEPTION;
720 }
721
722 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
723
724 if (!sys->codecDecoderContext)
725 {
726 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
727 goto EXCEPTION;
728 }
729
730#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
731 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
732 {
733 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
734 }
735#endif
736
737#ifdef WITH_VAAPI
738
739 if (!sys->hwctx)
740 {
741 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI,
742 get_vaapi_device(), nullptr, 0);
743
744 if (ret < 0)
745 {
746 WLog_Print(h264->log, WLOG_ERROR,
747 "Could not initialize hardware decoder, falling back to software: %s",
748 av_err2str(ret));
749 sys->hwctx = nullptr;
750 goto fail_hwdevice_create;
751 }
752 }
753 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 decoding");
754
755 sys->codecDecoderContext->get_format = libavcodec_get_format;
756 sys->hw_pix_fmt = AV_PIX_FMT_VAAPI;
757#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
758 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
759#endif
760 sys->codecDecoderContext->opaque = (void*)h264;
761 fail_hwdevice_create:
762#endif
763
764#ifdef WITH_VIDEOTOOLBOX
765
766 if (!sys->hwctx)
767 {
768 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr,
769 nullptr, 0);
770
771 if (ret < 0)
772 {
773 WLog_Print(
774 h264->log, WLOG_ERROR,
775 "Could not initialize VideoToolbox decoder, falling back to software: %s",
776 av_err2str(ret));
777 sys->hwctx = nullptr;
778 goto fail_vt_create;
779 }
780 }
781 WLog_Print(h264->log, WLOG_INFO, "Using VideoToolbox for accelerated H264 decoding");
782
783 sys->codecDecoderContext->get_format = libavcodec_get_format;
784 sys->hw_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
785 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
786 sys->codecDecoderContext->opaque = (void*)h264;
787 fail_vt_create:
788#endif
789
790 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
791 {
792 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
793 goto EXCEPTION;
794 }
795
796 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
797
798 if (!sys->codecParser)
799 {
800 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
801 goto EXCEPTION;
802 }
803 }
804 else
805 {
806#ifdef WITH_VAAPI_H264_ENCODING
807 if (h264->hwAccel) /* user requested hw accel */
808 {
809 sys->codecEncoder = avcodec_find_encoder_by_name("h264_vaapi");
810 if (!sys->codecEncoder)
811 {
812 WLog_Print(h264->log, WLOG_ERROR, "H264 VAAPI encoder not found");
813 }
814 else if (av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, get_vaapi_device(),
815 nullptr, 0) < 0)
816 {
817 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create failed");
818 sys->codecEncoder = nullptr;
819 sys->hwctx = nullptr;
820 }
821 else
822 {
823 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 encoding");
824 }
825 }
826#endif
827 if (!sys->codecEncoder)
828 {
829 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
830 h264->hwAccel = FALSE; /* not supported */
831 }
832
833 if (!sys->codecEncoder)
834 {
835 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
836 goto EXCEPTION;
837 }
838 }
839
840#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
841 sys->videoFrame = av_frame_alloc();
842#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
843 sys->hwVideoFrame = av_frame_alloc();
844#endif
845#else
846 sys->videoFrame = avcodec_alloc_frame();
847#endif
848
849 if (!sys->videoFrame)
850 {
851 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
852 goto EXCEPTION;
853 }
854
855#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
856 if (!sys->hwVideoFrame)
857 {
858 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
859 goto EXCEPTION;
860 }
861
862#endif
863 sys->videoFrame->pts = 0;
864 return TRUE;
865EXCEPTION:
866 libavcodec_uninit(h264);
867 return FALSE;
868}
869
870const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
871 libavcodec_uninit, libavcodec_decompress,
872 libavcodec_compress };