FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <freerdp/log.h>
25#include <freerdp/codec/h264.h>
26#include <libavcodec/avcodec.h>
27#include <libavutil/opt.h>
28
29#include "h264.h"
30
31#ifdef WITH_VIDEOTOOLBOX
32#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
33#include <libavutil/hwcontext.h>
34#else
35#pragma warning You have asked for VideoToolbox decoding, \
36 but your version of libavutil is too old !Disabling.
37#undef WITH_VIDEOTOOLBOX
38#endif
39#endif
40
41#ifdef WITH_VAAPI
42#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
43#include <libavutil/hwcontext.h>
44#else
45#pragma warning You have asked for VA - API decoding, \
46 but your version of libavutil is too old !Disabling.
47#undef WITH_VAAPI
48#endif
49#endif
50
51/* Fallback support for older libavcodec versions */
52#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
53#define AV_CODEC_ID_H264 CODEC_ID_H264
54#endif
55
56#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
57#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
58#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
59#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
60#endif
61
62#if LIBAVUTIL_VERSION_MAJOR < 52
63#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
64#endif
65
66/* Ubuntu 14.04 ships without the functions provided by avutil,
67 * so define error to string methods here. */
68#if !defined(av_err2str)
69static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
70{
71 av_strerror(errnum, errbuf, errbuf_size);
72 return errbuf;
73}
74
75#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
76#endif
77
78#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING)
79static const char* get_vaapi_device(void)
80{
81 static char device[MAX_PATH] = WINPR_C_ARRAY_INIT;
82 static bool initialized = false;
83 if (!initialized)
84 {
85 initialized = true;
86 const char* env = getenv("FREERDP_VAAPI_DEVICE");
87 if (env)
88 (void)_snprintf(device, sizeof(device), "%s", env);
89 else
90 (void)_snprintf(device, sizeof(device), "/dev/dri/renderD128");
91 }
92 return device;
93}
94#endif
95
96typedef struct
97{
98 const AVCodec* codecDecoder;
99 AVCodecContext* codecDecoderContext;
100 const AVCodec* codecEncoder;
101 AVCodecContext* codecEncoderContext;
102 AVCodecParserContext* codecParser;
103 AVFrame* videoFrame;
104#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
105 AVPacket bufferpacket;
106#endif
107 AVPacket* packet;
108#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
109 AVBufferRef* hwctx;
110 AVFrame* hwVideoFrame;
111 enum AVPixelFormat hw_pix_fmt;
112#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
113 AVBufferRef* hw_frames_ctx;
114#endif
115
116#endif
117} H264_CONTEXT_LIBAVCODEC;
118
119static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
120{
121 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
122
123 if (!h264 || !h264->subsystem)
124 return;
125
126 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
127
128 if (sys->codecEncoderContext)
129 {
130#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
131 avcodec_free_context(&sys->codecEncoderContext);
132#else
133 avcodec_close(sys->codecEncoderContext);
134 av_free(sys->codecEncoderContext);
135#endif
136 }
137
138 sys->codecEncoderContext = nullptr;
139}
140
141#ifdef WITH_VAAPI_H264_ENCODING
142static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
143{
144 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
145 AVBufferRef* hw_frames_ref = nullptr;
146 AVHWFramesContext* frames_ctx = nullptr;
147 int err = 0;
148
149 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
150 {
151 WLog_Print(h264->log, WLOG_ERROR, "Failed to create VAAPI frame context");
152 return -1;
153 }
154 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
155 frames_ctx->format = AV_PIX_FMT_VAAPI;
156 frames_ctx->sw_format = AV_PIX_FMT_NV12;
157 frames_ctx->width = sys->codecEncoderContext->width;
158 frames_ctx->height = sys->codecEncoderContext->height;
159 frames_ctx->initial_pool_size = 20;
160 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
161 {
162 WLog_Print(h264->log, WLOG_ERROR,
163 "Failed to initialize VAAPI frame context."
164 "Error code: %s",
165 av_err2str(err));
166 av_buffer_unref(&hw_frames_ref);
167 return err;
168 }
169 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
170 if (!sys->codecEncoderContext->hw_frames_ctx)
171 err = AVERROR(ENOMEM);
172
173 av_buffer_unref(&hw_frames_ref);
174 return err;
175}
176#endif
177
178static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
179{
180 BOOL recreate = FALSE;
181 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
182
183 if (!h264 || !h264->subsystem)
184 return FALSE;
185
186 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
187 return FALSE;
188
189 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
190 if (!sys || !sys->codecEncoder)
191 return FALSE;
192
193 recreate = !sys->codecEncoderContext;
194
195 if (sys->codecEncoderContext)
196 {
197 if ((sys->codecEncoderContext->width != (int)h264->width) ||
198 (sys->codecEncoderContext->height != (int)h264->height))
199 recreate = TRUE;
200 }
201
202 if (!recreate)
203 return TRUE;
204
205 libavcodec_destroy_encoder_context(h264);
206
207 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
208
209 if (!sys->codecEncoderContext)
210 goto EXCEPTION;
211
212 switch (h264->RateControlMode)
213 {
214 case H264_RATECONTROL_VBR:
215 sys->codecEncoderContext->bit_rate = h264->BitRate;
216 break;
217
218 case H264_RATECONTROL_CQP:
219 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
220 0)
221 {
222 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
223 }
224 break;
225
226 default:
227 break;
228 }
229
230 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
231 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
232 sys->codecEncoderContext->delay = 0;
233#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
234 sys->codecEncoderContext->framerate =
235 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
236#endif
237 sys->codecEncoderContext->time_base =
238 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
239 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
240
241 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
242
243#ifdef WITH_VAAPI_H264_ENCODING
244 if (sys->hwctx)
245 {
246 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
247
248 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_VAAPI;
249 /* set hw_frames_ctx for encoder's AVCodecContext */
250 if (set_hw_frames_ctx(h264) < 0)
251 goto EXCEPTION;
252 }
253 else
254#endif
255 {
256 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
257 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
258 }
259
260 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
261 goto EXCEPTION;
262
263 return TRUE;
264EXCEPTION:
265 libavcodec_destroy_encoder_context(h264);
266 return FALSE;
267}
268
269static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
270 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
271{
272 union
273 {
274 const BYTE* cpv;
275 BYTE* pv;
276 } cnv;
277 int rc = -1;
278 int status = 0;
279 int gotFrame = 0;
280 AVPacket* packet = nullptr;
281
282 WINPR_ASSERT(h264);
283 WINPR_ASSERT(pSrcData || (SrcSize == 0));
284
285 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
286 BYTE** pYUVData = h264->pYUVData;
287 UINT32* iStride = h264->iStride;
288
289 WINPR_ASSERT(sys);
290
291#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
292 packet = &sys->bufferpacket;
293 WINPR_ASSERT(packet);
294 av_init_packet(packet);
295#else
296 packet = av_packet_alloc();
297#endif
298 if (!packet)
299 {
300 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
301 goto fail;
302 }
303
304 cnv.cpv = pSrcData;
305 packet->data = cnv.pv;
306 packet->size = (int)MIN(SrcSize, INT32_MAX);
307
308 WINPR_ASSERT(sys->codecDecoderContext);
309 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
310#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
311 status = avcodec_send_packet(sys->codecDecoderContext, packet);
312
313 if (status < 0)
314 {
315 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
316 goto fail;
317 }
318
319 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
320
321#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
322 status = avcodec_receive_frame(sys->codecDecoderContext,
323 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
324#else
325 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
326#endif
327 if (status == AVERROR(EAGAIN))
328 {
329 rc = 0;
330 goto fail;
331 }
332
333 gotFrame = (status == 0);
334#else
335#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
336 status =
337 avcodec_decode_video2(sys->codecDecoderContext,
338 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
339#else
340 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
341#endif
342#endif
343 if (status < 0)
344 {
345 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
346 goto fail;
347 }
348
349#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
350
351 if (sys->hwctx)
352 {
353 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
354 {
355 sys->videoFrame->width = sys->hwVideoFrame->width;
356 sys->videoFrame->height = sys->hwVideoFrame->height;
357 status = av_hwframe_transfer_data(sys->videoFrame, sys->hwVideoFrame, 0);
358 }
359 else
360 {
361 status = av_frame_copy(sys->videoFrame, sys->hwVideoFrame);
362 }
363 }
364
365 gotFrame = (status == 0);
366
367 if (status < 0)
368 {
369 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)", status,
370 av_err2str(status));
371 goto fail;
372 }
373
374#endif
375
376 if (gotFrame)
377 {
378 WINPR_ASSERT(sys->videoFrame);
379
380 pYUVData[0] = sys->videoFrame->data[0];
381 pYUVData[1] = sys->videoFrame->data[1];
382 pYUVData[2] = sys->videoFrame->data[2];
383 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
384 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
385 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
386 rc = 1;
387 }
388 else
389 rc = -2;
390
391fail:
392#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
393 av_packet_unref(packet);
394#else
395 av_packet_free(&packet);
396#endif
397
398 return rc;
399}
400
401static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
402 const BYTE** WINPR_RESTRICT pSrcYuv,
403 const UINT32* WINPR_RESTRICT pStride,
404 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
405{
406 union
407 {
408 const BYTE* cpv;
409 uint8_t* pv;
410 } cnv;
411 int rc = -1;
412 int status = 0;
413 int gotFrame = 0;
414
415 WINPR_ASSERT(h264);
416
417 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
418 WINPR_ASSERT(sys);
419
420 if (!libavcodec_create_encoder_context(h264))
421 return -1;
422
423#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
424 sys->packet = &sys->bufferpacket;
425 av_packet_unref(sys->packet);
426 av_init_packet(sys->packet);
427#else
428 av_packet_free(&sys->packet);
429 sys->packet = av_packet_alloc();
430#endif
431 if (!sys->packet)
432 {
433 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
434 goto fail;
435 }
436
437 WINPR_ASSERT(sys->packet);
438 sys->packet->data = nullptr;
439 sys->packet->size = 0;
440
441 WINPR_ASSERT(sys->videoFrame);
442 WINPR_ASSERT(sys->codecEncoderContext);
443 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
444 sys->videoFrame->width = sys->codecEncoderContext->width;
445 sys->videoFrame->height = sys->codecEncoderContext->height;
446#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
447 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
448#endif
449#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
450 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
451#endif
452 cnv.cpv = pSrcYuv[0];
453 sys->videoFrame->data[0] = cnv.pv;
454
455 cnv.cpv = pSrcYuv[1];
456 sys->videoFrame->data[1] = cnv.pv;
457
458 cnv.cpv = pSrcYuv[2];
459 sys->videoFrame->data[2] = cnv.pv;
460
461 sys->videoFrame->linesize[0] = (int)pStride[0];
462 sys->videoFrame->linesize[1] = (int)pStride[1];
463 sys->videoFrame->linesize[2] = (int)pStride[2];
464 sys->videoFrame->pts++;
465
466#ifdef WITH_VAAPI_H264_ENCODING
467 if (sys->hwctx)
468 {
469 av_frame_unref(sys->hwVideoFrame);
470 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
471 sys->hwVideoFrame, 0)) < 0 ||
472 !sys->hwVideoFrame->hw_frames_ctx)
473 {
474 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
475 av_err2str(status), status);
476 goto fail;
477 }
478 sys->videoFrame->format = AV_PIX_FMT_NV12;
479 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
480 {
481 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
482 av_err2str(status), status);
483 goto fail;
484 }
485 }
486#endif
487
488 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
489#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
490#ifdef WITH_VAAPI_H264_ENCODING
491 status = avcodec_send_frame(sys->codecEncoderContext,
492 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
493#else
494 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
495#endif
496
497 if (status < 0)
498 {
499 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
500 av_err2str(status), status);
501 goto fail;
502 }
503
504 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
505
506 if (status < 0)
507 {
508 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
509 av_err2str(status), status);
510 goto fail;
511 }
512
513 gotFrame = (status == 0);
514#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
515
516 do
517 {
518 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
519 &gotFrame);
520 } while ((status >= 0) && (gotFrame == 0));
521
522#else
523 sys->packet->size =
524 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
525 sys->codecDecoderContext->height);
526 sys->packet->data = av_malloc(sys->packet->size);
527
528 if (!sys->packet->data)
529 status = -1;
530 else
531 {
532 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
533 sys->packet->size, sys->videoFrame);
534 }
535
536#endif
537
538 if (status < 0)
539 {
540 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
541 av_err2str(status), status);
542 goto fail;
543 }
544
545 WINPR_ASSERT(sys->packet);
546 *ppDstData = sys->packet->data;
547 *pDstSize = (UINT32)MAX(0, sys->packet->size);
548
549 if (!gotFrame)
550 {
551 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
552 status);
553 rc = -2;
554 }
555 else
556 rc = 1;
557fail:
558 return rc;
559}
560
561static void libavcodec_uninit(H264_CONTEXT* h264)
562{
563 WINPR_ASSERT(h264);
564
565 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
566
567 if (!sys)
568 return;
569
570 if (sys->packet)
571 {
572#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
573 av_packet_unref(sys->packet);
574#else
575 av_packet_free(&sys->packet);
576#endif
577 }
578
579 if (sys->videoFrame)
580 {
581#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
582 av_frame_free(&sys->videoFrame);
583#else
584 av_free(sys->videoFrame);
585#endif
586 }
587
588#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
589 if (sys->hwVideoFrame)
590 {
591#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
592 av_frame_free(&sys->hwVideoFrame);
593#else
594 av_free(sys->hwVideoFrame);
595#endif
596 }
597
598 if (sys->hwctx)
599 av_buffer_unref(&sys->hwctx);
600
601#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
602
603 if (sys->hw_frames_ctx)
604 av_buffer_unref(&sys->hw_frames_ctx);
605
606#endif
607
608#endif
609
610 if (sys->codecParser)
611 av_parser_close(sys->codecParser);
612
613 if (sys->codecDecoderContext)
614 {
615#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
616 avcodec_free_context(&sys->codecDecoderContext);
617#else
618 avcodec_close(sys->codecDecoderContext);
619 av_free(sys->codecDecoderContext);
620#endif
621 }
622
623 libavcodec_destroy_encoder_context(h264);
624 free(sys);
625 h264->pSystemData = nullptr;
626}
627
628#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
629static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
630 const enum AVPixelFormat* fmts)
631{
632 WINPR_ASSERT(ctx);
633
634 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
635 WINPR_ASSERT(h264);
636
637 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
638 WINPR_ASSERT(sys);
639
640 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
641 {
642 if (*p == sys->hw_pix_fmt)
643 {
644#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
645 if (sys->hw_frames_ctx)
646 av_buffer_unref(&sys->hw_frames_ctx);
647
648 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
649
650 if (!sys->hw_frames_ctx)
651 {
652 return AV_PIX_FMT_NONE;
653 }
654
655 sys->codecDecoderContext->pix_fmt = *p;
656 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
657 frames->format = *p;
658 frames->height = sys->codecDecoderContext->coded_height;
659 frames->width = sys->codecDecoderContext->coded_width;
660#ifdef WITH_VIDEOTOOLBOX
661 frames->sw_format = AV_PIX_FMT_YUV420P;
662#else
663 frames->sw_format =
664 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
665 : AV_PIX_FMT_NV12);
666#endif
667 frames->initial_pool_size = 20;
668
669 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
670 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
671
672 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
673
674 if (err < 0)
675 {
676 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
677 av_err2str(err));
678 return AV_PIX_FMT_NONE;
679 }
680
681 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
682#endif
683 return *p;
684 }
685 }
686
687 return AV_PIX_FMT_NONE;
688}
689#endif
690
691static BOOL libavcodec_init(H264_CONTEXT* h264)
692{
693 WINPR_ASSERT(h264);
694 H264_CONTEXT_LIBAVCODEC* sys =
695 (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
696
697 if (!sys)
698 {
699 goto EXCEPTION;
700 }
701
702 h264->pSystemData = (void*)sys;
703
704#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
705 avcodec_register_all();
706#endif
707
708 if (!h264->Compressor)
709 {
710 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
711
712 if (!sys->codecDecoder)
713 {
714 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
715 goto EXCEPTION;
716 }
717
718 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
719
720 if (!sys->codecDecoderContext)
721 {
722 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
723 goto EXCEPTION;
724 }
725
726#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
727 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
728 {
729 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
730 }
731#endif
732
733#ifdef WITH_VAAPI
734
735 if (!sys->hwctx)
736 {
737 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI,
738 get_vaapi_device(), nullptr, 0);
739
740 if (ret < 0)
741 {
742 WLog_Print(h264->log, WLOG_ERROR,
743 "Could not initialize hardware decoder, falling back to software: %s",
744 av_err2str(ret));
745 sys->hwctx = nullptr;
746 goto fail_hwdevice_create;
747 }
748 }
749 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 decoding");
750
751 sys->codecDecoderContext->get_format = libavcodec_get_format;
752 sys->hw_pix_fmt = AV_PIX_FMT_VAAPI;
753#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
754 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
755#endif
756 sys->codecDecoderContext->opaque = (void*)h264;
757 fail_hwdevice_create:
758#endif
759
760#ifdef WITH_VIDEOTOOLBOX
761
762 if (!sys->hwctx)
763 {
764 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr,
765 nullptr, 0);
766
767 if (ret < 0)
768 {
769 WLog_Print(
770 h264->log, WLOG_ERROR,
771 "Could not initialize VideoToolbox decoder, falling back to software: %s",
772 av_err2str(ret));
773 sys->hwctx = nullptr;
774 goto fail_vt_create;
775 }
776 }
777 WLog_Print(h264->log, WLOG_INFO, "Using VideoToolbox for accelerated H264 decoding");
778
779 sys->codecDecoderContext->get_format = libavcodec_get_format;
780 sys->hw_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
781 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
782 sys->codecDecoderContext->opaque = (void*)h264;
783 fail_vt_create:
784#endif
785
786 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
787 {
788 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
789 goto EXCEPTION;
790 }
791
792 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
793
794 if (!sys->codecParser)
795 {
796 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
797 goto EXCEPTION;
798 }
799 }
800 else
801 {
802#ifdef WITH_VAAPI_H264_ENCODING
803 if (h264->hwAccel) /* user requested hw accel */
804 {
805 sys->codecEncoder = avcodec_find_encoder_by_name("h264_vaapi");
806 if (!sys->codecEncoder)
807 {
808 WLog_Print(h264->log, WLOG_ERROR, "H264 VAAPI encoder not found");
809 }
810 else if (av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, get_vaapi_device(),
811 nullptr, 0) < 0)
812 {
813 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create failed");
814 sys->codecEncoder = nullptr;
815 sys->hwctx = nullptr;
816 }
817 else
818 {
819 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 encoding");
820 }
821 }
822#endif
823 if (!sys->codecEncoder)
824 {
825 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
826 h264->hwAccel = FALSE; /* not supported */
827 }
828
829 if (!sys->codecEncoder)
830 {
831 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
832 goto EXCEPTION;
833 }
834 }
835
836#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
837 sys->videoFrame = av_frame_alloc();
838#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
839 sys->hwVideoFrame = av_frame_alloc();
840#endif
841#else
842 sys->videoFrame = avcodec_alloc_frame();
843#endif
844
845 if (!sys->videoFrame)
846 {
847 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
848 goto EXCEPTION;
849 }
850
851#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
852 if (!sys->hwVideoFrame)
853 {
854 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
855 goto EXCEPTION;
856 }
857
858#endif
859 sys->videoFrame->pts = 0;
860 return TRUE;
861EXCEPTION:
862 libavcodec_uninit(h264);
863 return FALSE;
864}
865
866const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
867 libavcodec_uninit, libavcodec_decompress,
868 libavcodec_compress };