FreeRDP
Loading...
Searching...
No Matches
av1_videotoolbox.c
1
19
20/* Apple silicon from M3 and A17 Pro on decodes AV1 in hardware; VideoToolbox has no
21 * software AV1 decoder, so older machines keep dav1d (av1_vt_new() returns nullptr).
22 *
23 * VideoToolbox takes AV1 the way ISO BMFF carries it: an av1C record (AV1 codec ISO media
24 * file format binding, 2.3) with the sequence header in the format description, and each
25 * sample a temporal unit without its temporal delimiter. The decoded YUV planes go back to
26 * av1.c, which converts them to RGB exactly as it does dav1d's pictures. */
27
28#include <freerdp/config.h>
29
30#include <string.h>
31
32#include <winpr/assert.h>
33#include <winpr/cast.h>
34#include <winpr/crt.h>
35
36#include <CoreFoundation/CoreFoundation.h>
37#include <CoreMedia/CoreMedia.h>
38#include <CoreVideo/CoreVideo.h>
39#include <VideoToolbox/VideoToolbox.h>
40
41#include "av1_videotoolbox.h"
42
43/* kCMVideoCodecType_AV1, spelled out for SDKs older than macOS 13 */
44#define AV1_VT_CODEC_TYPE ((CMVideoCodecType)0x61763031) /* 'av01' */
45
46enum
47{
48 OBU_SEQUENCE_HEADER = 1,
49 OBU_TEMPORAL_DELIMITER = 2,
50 OBU_FRAME_HEADER = 3,
51 OBU_TILE_GROUP = 4,
52 OBU_FRAME = 6,
53 OBU_PADDING = 15
54};
55
56struct S_AV1_VT
57{
58 wLog* log;
59 VTDecompressionSessionRef session;
60 CMVideoFormatDescriptionRef format;
61 BYTE* seq; /* the sequence header OBU the session was made for */
62 size_t seqSize;
63 BOOL i444;
64 BOOL decoded; /* a frame of the session's stream came out */
65
66 BYTE* sample; /* the temporal unit without temporal delimiters and padding */
67 size_t sampleCapacity;
68
69 OSStatus outStatus; /* of the last output callback */
70 CVImageBufferRef pending; /* the output of the last decode */
71 CVPixelBufferRef handedOut; /* locked, its planes are in the caller's AV1_VT_PICTURE */
72 BYTE* chroma[2]; /* Cb and Cr of a bi-planar picture, apart */
73 size_t chromaSize;
74};
75
76/* --- sequence header (AV1 specification 5.5) --- */
77
78typedef struct
79{
80 const BYTE* p;
81 size_t size;
82 size_t bit;
83 BOOL overrun;
84} AV1_VT_BITS;
85
86static UINT32 av1_vt_bits(AV1_VT_BITS* b, unsigned count)
87{
88 UINT32 v = 0;
89 for (unsigned i = 0; i < count; i++)
90 {
91 if ((b->bit >> 3) >= b->size)
92 {
93 b->overrun = TRUE;
94 return 0;
95 }
96 v = (v << 1) | ((b->p[b->bit >> 3] >> (7 - (b->bit & 7))) & 1);
97 b->bit++;
98 }
99 return v;
100}
101
102static void av1_vt_uvlc(AV1_VT_BITS* b)
103{
104 unsigned leadingZeros = 0;
105 while (!b->overrun && (av1_vt_bits(b, 1) == 0))
106 {
107 if (++leadingZeros >= 32)
108 {
109 b->overrun = TRUE;
110 return;
111 }
112 }
113 (void)av1_vt_bits(b, leadingZeros);
114}
115
116typedef struct
117{
118 UINT32 profile;
119 UINT32 level0;
120 UINT32 tier0;
121 UINT32 highBitdepth;
122 UINT32 twelveBit;
123 UINT32 mono;
124 UINT32 subsamplingX;
125 UINT32 subsamplingY;
126 UINT32 chromaSamplePosition;
127 UINT32 fullRange;
128 UINT32 width;
129 UINT32 height;
130} AV1_VT_SEQ;
131
132static BOOL av1_vt_parse_sequence_header(const BYTE* p, size_t size, AV1_VT_SEQ* s)
133{
134 AV1_VT_BITS b = { .p = p, .size = size };
135 memset(s, 0, sizeof(*s));
136
137 s->profile = av1_vt_bits(&b, 3);
138 (void)av1_vt_bits(&b, 1); /* still_picture */
139 const UINT32 reduced = av1_vt_bits(&b, 1);
140 if (reduced)
141 s->level0 = av1_vt_bits(&b, 5);
142 else
143 {
144 UINT32 decoderModelInfo = 0;
145 UINT32 bufferDelayLength = 0;
146 if (av1_vt_bits(&b, 1)) /* timing_info_present_flag */
147 {
148 (void)av1_vt_bits(&b, 32); /* num_units_in_display_tick */
149 (void)av1_vt_bits(&b, 32); /* time_scale */
150 if (av1_vt_bits(&b, 1)) /* equal_picture_interval */
151 av1_vt_uvlc(&b);
152 decoderModelInfo = av1_vt_bits(&b, 1);
153 if (decoderModelInfo)
154 {
155 bufferDelayLength = av1_vt_bits(&b, 5) + 1;
156 (void)av1_vt_bits(&b, 32); /* num_units_in_decoding_tick */
157 (void)av1_vt_bits(&b, 10); /* buffer_removal_time_length, frame_presentation */
158 }
159 }
160 const UINT32 initialDisplayDelay = av1_vt_bits(&b, 1);
161 const UINT32 operatingPoints = av1_vt_bits(&b, 5) + 1;
162 for (UINT32 i = 0; i < operatingPoints; i++)
163 {
164 (void)av1_vt_bits(&b, 12); /* operating_point_idc */
165 const UINT32 level = av1_vt_bits(&b, 5);
166 const UINT32 tier = (level > 7) ? av1_vt_bits(&b, 1) : 0;
167 if (i == 0)
168 {
169 s->level0 = level;
170 s->tier0 = tier;
171 }
172 if (decoderModelInfo && av1_vt_bits(&b, 1))
173 {
174 (void)av1_vt_bits(&b, bufferDelayLength); /* decoder_buffer_delay */
175 (void)av1_vt_bits(&b, bufferDelayLength); /* encoder_buffer_delay */
176 (void)av1_vt_bits(&b, 1); /* low_delay_mode_flag */
177 }
178 if (initialDisplayDelay && av1_vt_bits(&b, 1))
179 (void)av1_vt_bits(&b, 4);
180 }
181 }
182
183 const UINT32 widthBits = av1_vt_bits(&b, 4) + 1;
184 const UINT32 heightBits = av1_vt_bits(&b, 4) + 1;
185 s->width = av1_vt_bits(&b, widthBits) + 1;
186 s->height = av1_vt_bits(&b, heightBits) + 1;
187 if (!reduced && av1_vt_bits(&b, 1)) /* frame_id_numbers_present_flag */
188 (void)av1_vt_bits(&b, 7);
189 (void)av1_vt_bits(&b, 3); /* 128x128 superblock, filter intra, intra edge filter */
190 if (!reduced)
191 {
192 (void)av1_vt_bits(&b, 4); /* interintra, masked compound, warped motion, dual filter */
193 const UINT32 orderHint = av1_vt_bits(&b, 1);
194 if (orderHint)
195 (void)av1_vt_bits(&b, 2); /* jnt_comp, ref_frame_mvs */
196 UINT32 forceScreenContentTools = 2;
197 if (!av1_vt_bits(&b, 1)) /* seq_choose_screen_content_tools */
198 forceScreenContentTools = av1_vt_bits(&b, 1);
199 if (forceScreenContentTools > 0)
200 {
201 if (!av1_vt_bits(&b, 1)) /* seq_choose_integer_mv */
202 (void)av1_vt_bits(&b, 1);
203 }
204 if (orderHint)
205 (void)av1_vt_bits(&b, 3);
206 }
207 (void)av1_vt_bits(&b, 3); /* superres, cdef, restoration */
208
209 /* color_config() */
210 s->highBitdepth = av1_vt_bits(&b, 1);
211 if ((s->profile == 2) && s->highBitdepth)
212 s->twelveBit = av1_vt_bits(&b, 1);
213 s->mono = (s->profile == 1) ? 0 : av1_vt_bits(&b, 1);
214 UINT32 primaries = 2;
215 UINT32 transfer = 2;
216 UINT32 matrix = 2;
217 if (av1_vt_bits(&b, 1)) /* color_description_present_flag */
218 {
219 primaries = av1_vt_bits(&b, 8);
220 transfer = av1_vt_bits(&b, 8);
221 matrix = av1_vt_bits(&b, 8);
222 }
223 if (s->mono)
224 {
225 s->fullRange = av1_vt_bits(&b, 1);
226 s->subsamplingX = s->subsamplingY = 1;
227 }
228 else if ((primaries == 1) && (transfer == 13) && (matrix == 0)) /* sRGB */
229 s->fullRange = 1;
230 else
231 {
232 s->fullRange = av1_vt_bits(&b, 1);
233 if (s->profile == 0)
234 s->subsamplingX = s->subsamplingY = 1;
235 else if (s->profile == 2)
236 {
237 if (s->twelveBit)
238 {
239 s->subsamplingX = av1_vt_bits(&b, 1);
240 s->subsamplingY = s->subsamplingX ? av1_vt_bits(&b, 1) : 0;
241 }
242 else
243 s->subsamplingX = 1;
244 }
245 if (s->subsamplingX && s->subsamplingY)
246 s->chromaSamplePosition = av1_vt_bits(&b, 2);
247 }
248 return !b.overrun;
249}
250
251/* --- the session --- */
252
253static void av1_vt_release_picture(AV1_VT* vt)
254{
255 if (vt->handedOut)
256 {
257 CVPixelBufferUnlockBaseAddress(vt->handedOut, kCVPixelBufferLock_ReadOnly);
258 CVPixelBufferRelease(vt->handedOut);
259 vt->handedOut = nullptr;
260 }
261 if (vt->pending)
262 {
263 CVPixelBufferRelease(vt->pending);
264 vt->pending = nullptr;
265 }
266}
267
268static void av1_vt_close(AV1_VT* vt)
269{
270 av1_vt_release_picture(vt);
271 if (vt->session)
272 {
273 VTDecompressionSessionInvalidate(vt->session);
274 CFRelease(vt->session);
275 vt->session = nullptr;
276 }
277 if (vt->format)
278 {
279 CFRelease(vt->format);
280 vt->format = nullptr;
281 }
282 free(vt->seq);
283 vt->seq = nullptr;
284 vt->seqSize = 0;
285 vt->decoded = FALSE;
286}
287
288static void av1_vt_output(void* decompressionOutputRefCon,
289 WINPR_ATTR_UNUSED void* sourceFrameRefCon, OSStatus status,
290 WINPR_ATTR_UNUSED VTDecodeInfoFlags infoFlags,
291 CVImageBufferRef imageBuffer,
292 WINPR_ATTR_UNUSED CMTime presentationTimeStamp,
293 WINPR_ATTR_UNUSED CMTime presentationDuration)
294{
295 AV1_VT* vt = decompressionOutputRefCon;
296 WINPR_ASSERT(vt);
297 vt->outStatus = status;
298 if ((status != noErr) || !imageBuffer)
299 return;
300 if (vt->pending)
301 CVPixelBufferRelease(vt->pending);
302 vt->pending = CVPixelBufferRetain(imageBuffer);
303}
304
305static CFDictionaryRef av1_vt_dictionary(const void* key, const void* value)
306{
307 const void* keys[] = { key };
308 const void* values[] = { value };
309 return CFDictionaryCreate(kCFAllocatorDefault, keys, values, 1, &kCFTypeDictionaryKeyCallBacks,
310 &kCFTypeDictionaryValueCallBacks);
311}
312
313/* Return: TRUE, AV1_VT_UNSUPPORTED, AV1_VT_INVALID (nothing changed) or -1 */
314static INT32 av1_vt_open(AV1_VT* vt, const BYTE* obu, size_t obuSize, const BYTE* payload,
315 size_t payloadSize)
316{
317 AV1_VT_SEQ s = WINPR_C_ARRAY_INIT;
318 if (!av1_vt_parse_sequence_header(payload, payloadSize, &s))
319 {
320 WLog_Print(vt->log, WLOG_WARN, "AV1 sequence header too short");
321 return AV1_VT_INVALID;
322 }
323
324 /* a new stream: the old session is done */
325 av1_vt_close(vt);
326
327 /* What av1.c converts: 8 bit 4:2:0 and 4:4:4 */
328 const BOOL i420 = (s.subsamplingX == 1) && (s.subsamplingY == 1);
329 const BOOL i444 = (s.subsamplingX == 0) && (s.subsamplingY == 0);
330 if (s.highBitdepth || s.mono || (!i420 && !i444))
331 {
332 WLog_Print(vt->log, WLOG_DEBUG,
333 "AV1 profile %" PRIu32 " high bit depth %" PRIu32 " mono %" PRIu32
334 " subsampling %" PRIu32 "x%" PRIu32 ": not for VideoToolbox",
335 s.profile, s.highBitdepth, s.mono, s.subsamplingX, s.subsamplingY);
336 return AV1_VT_UNSUPPORTED;
337 }
338
339 /* av1C: marker 1, version 1; seq_profile, seq_level_idx_0; seq_tier_0, high_bitdepth,
340 * twelve_bit, monochrome, chroma_subsampling_x/y, chroma_sample_position;
341 * no initial_presentation_delay; then the sequence header OBU. */
342 BYTE* av1c = malloc(4 + obuSize);
343 if (!av1c)
344 return -1;
345 av1c[0] = 0x81;
346 av1c[1] = (BYTE)((s.profile << 5) | (s.level0 & 0x1f));
347 av1c[2] = (BYTE)((s.tier0 << 7) | (s.highBitdepth << 6) | (s.twelveBit << 5) | (s.mono << 4) |
348 (s.subsamplingX << 3) | (s.subsamplingY << 2) | (s.chromaSamplePosition & 3));
349 av1c[3] = 0;
350 memcpy(&av1c[4], obu, obuSize);
351
352 INT32 rc = AV1_VT_UNSUPPORTED;
353 CFDataRef record =
354 CFDataCreate(kCFAllocatorDefault, av1c, WINPR_ASSERTING_INT_CAST(CFIndex, 4 + obuSize));
355 free(av1c);
356 CFDictionaryRef atoms = record ? av1_vt_dictionary(CFSTR("av1C"), record) : nullptr;
357 CFDictionaryRef extensions =
358 atoms ? av1_vt_dictionary(kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
359 atoms)
360 : nullptr;
361
362 /* The decoder's own bi-planar layout and the stream's range: no conversion in
363 * VideoToolbox, the samples come out as decoded. */
364 OSType pixelFormat = 0;
365 if (i444)
366 pixelFormat = s.fullRange ? kCVPixelFormatType_444YpCbCr8BiPlanarFullRange
367 : kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange;
368 else
369 pixelFormat = s.fullRange ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
370 : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
371 SInt32 pf = WINPR_ASSERTING_INT_CAST(SInt32, pixelFormat);
372 CFNumberRef pfNumber = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pf);
373 CFDictionaryRef attributes =
374 pfNumber ? av1_vt_dictionary(kCVPixelBufferPixelFormatTypeKey, pfNumber) : nullptr;
375 CFDictionaryRef specification = av1_vt_dictionary(
376 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, kCFBooleanTrue);
377
378 if (!extensions || !attributes || !specification)
379 {
380 rc = -1;
381 goto out;
382 }
383
384 OSStatus status = CMVideoFormatDescriptionCreate(
385 kCFAllocatorDefault, AV1_VT_CODEC_TYPE, WINPR_ASSERTING_INT_CAST(int32_t, s.width),
386 WINPR_ASSERTING_INT_CAST(int32_t, s.height), extensions, &vt->format);
387 if (status != noErr)
388 {
389 WLog_Print(vt->log, WLOG_WARN, "CMVideoFormatDescriptionCreate: %d", (int)status);
390 goto out;
391 }
392
393 const VTDecompressionOutputCallbackRecord callback = { .decompressionOutputCallback =
394 av1_vt_output,
395 .decompressionOutputRefCon = vt };
396 status = VTDecompressionSessionCreate(kCFAllocatorDefault, vt->format, specification,
397 attributes, &callback, &vt->session);
398 if (status != noErr)
399 {
400 WLog_Print(vt->log, WLOG_INFO,
401 "VideoToolbox can't decode AV1 profile %" PRIu32 " %s %" PRIu32 "x%" PRIu32
402 " (%d): software decoder",
403 s.profile, i444 ? "4:4:4" : "4:2:0", s.width, s.height, (int)status);
404 goto out;
405 }
406 (void)VTSessionSetProperty(vt->session, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
407
408 vt->seq = malloc(obuSize);
409 if (!vt->seq)
410 {
411 rc = -1;
412 goto out;
413 }
414 memcpy(vt->seq, obu, obuSize);
415 vt->seqSize = obuSize;
416 vt->i444 = i444;
417 WLog_Print(vt->log, WLOG_DEBUG,
418 "VideoToolbox AV1 session: profile %" PRIu32 " level %" PRIu32 " %s %" PRIu32
419 "x%" PRIu32 " %s range",
420 s.profile, s.level0, i444 ? "4:4:4" : "4:2:0", s.width, s.height,
421 s.fullRange ? "full" : "video");
422 rc = TRUE;
423
424out:
425 if (specification)
426 CFRelease(specification);
427 if (attributes)
428 CFRelease(attributes);
429 if (pfNumber)
430 CFRelease(pfNumber);
431 if (extensions)
432 CFRelease(extensions);
433 if (atoms)
434 CFRelease(atoms);
435 if (record)
436 CFRelease(record);
437 if (rc != TRUE)
438 av1_vt_close(vt);
439 return rc;
440}
441
442static BOOL av1_vt_leb128(const BYTE* p, size_t size, size_t* pos, UINT64* value)
443{
444 *value = 0;
445 for (unsigned i = 0; i < 8; i++)
446 {
447 if (*pos >= size)
448 return FALSE;
449 const BYTE c = p[(*pos)++];
450 *value |= (UINT64)(c & 0x7f) << (7 * i);
451 if ((c & 0x80) == 0)
452 return TRUE;
453 }
454 return FALSE;
455}
456
457static BOOL av1_vt_reserve(AV1_VT* vt, size_t size)
458{
459 if (size <= vt->sampleCapacity)
460 return TRUE;
461 BYTE* p = realloc(vt->sample, size);
462 if (!p)
463 return FALSE;
464 vt->sample = p;
465 vt->sampleCapacity = size;
466 return TRUE;
467}
468
469/* Hand the decoded picture to the caller: lock it, split a bi-planar CbCr plane. */
470static INT32 av1_vt_picture(AV1_VT* vt, AV1_VT_PICTURE* pic)
471{
472 CVPixelBufferRef buffer = vt->pending;
473 vt->pending = nullptr;
474 if (CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess)
475 {
476 CVPixelBufferRelease(buffer);
477 return -1;
478 }
479 vt->handedOut = buffer;
480
481 const size_t planes = CVPixelBufferGetPlaneCount(buffer);
482 pic->width = WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetWidth(buffer));
483 pic->height = WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetHeight(buffer));
484 pic->i444 = vt->i444;
485 pic->planes[0] = CVPixelBufferGetBaseAddressOfPlane(buffer, 0);
486 pic->strides[0] =
487 WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetBytesPerRowOfPlane(buffer, 0));
488
489 if (planes == 3)
490 {
491 for (size_t i = 1; i < 3; i++)
492 {
493 pic->planes[i] = CVPixelBufferGetBaseAddressOfPlane(buffer, i);
494 pic->strides[i] =
495 WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetBytesPerRowOfPlane(buffer, i));
496 }
497 return TRUE;
498 }
499 if (planes != 2)
500 {
501 WLog_Print(vt->log, WLOG_ERROR, "VideoToolbox picture with %" PRIuz " planes", planes);
502 return -1;
503 }
504
505 const size_t cw = CVPixelBufferGetWidthOfPlane(buffer, 1);
506 const size_t ch = CVPixelBufferGetHeightOfPlane(buffer, 1);
507 const size_t stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1);
508 const BYTE* cbcr = CVPixelBufferGetBaseAddressOfPlane(buffer, 1);
509 if (!cbcr || (stride < 2 * cw))
510 return -1;
511 if (cw * ch > vt->chromaSize)
512 {
513 for (size_t i = 0; i < 2; i++)
514 {
515 winpr_aligned_free(vt->chroma[i]);
516 vt->chroma[i] = winpr_aligned_malloc(cw * ch, 64);
517 }
518 vt->chromaSize = (vt->chroma[0] && vt->chroma[1]) ? cw * ch : 0;
519 if (vt->chromaSize == 0)
520 return -1;
521 }
522 for (size_t y = 0; y < ch; y++)
523 {
524 const BYTE* src = &cbcr[y * stride];
525 BYTE* cb = &vt->chroma[0][y * cw];
526 BYTE* cr = &vt->chroma[1][y * cw];
527 for (size_t x = 0; x < cw; x++)
528 {
529 cb[x] = src[2 * x];
530 cr[x] = src[2 * x + 1];
531 }
532 }
533 pic->planes[1] = vt->chroma[0];
534 pic->planes[2] = vt->chroma[1];
535 pic->strides[1] = pic->strides[2] = WINPR_ASSERTING_INT_CAST(UINT32, cw);
536 return TRUE;
537}
538
539INT32 av1_vt_decode(AV1_VT* vt, const BYTE* pSrcData, UINT32 SrcSize, AV1_VT_PICTURE* pic)
540{
541 WINPR_ASSERT(vt);
542 WINPR_ASSERT(pSrcData || (SrcSize == 0));
543 WINPR_ASSERT(pic);
544
545 av1_vt_release_picture(vt);
546
547 /* First check the whole temporal unit, so that bad input changes nothing, and see whether
548 * it carries a frame at all (a temporal unit may hold a sequence header only). */
549 BOOL hasFrame = FALSE;
550 for (size_t pos = 0; pos < SrcSize;)
551 {
552 const size_t start = pos;
553 const BYTE header = pSrcData[pos++];
554 const unsigned type = (header >> 3) & 0x0f;
555 if (header & 0x04) /* obu_extension_flag */
556 pos++;
557 UINT64 size = 0;
558 if (((header & 0x02) == 0) || !av1_vt_leb128(pSrcData, SrcSize, &pos, &size) ||
559 (size > SrcSize - pos))
560 {
561 WLog_Print(vt->log, WLOG_WARN, "AV1 OBU at %" PRIuz " of %" PRIu32 " is malformed",
562 start, SrcSize);
563 return AV1_VT_INVALID;
564 }
565 pos += WINPR_ASSERTING_INT_CAST(size_t, size);
566 if ((type == OBU_FRAME_HEADER) || (type == OBU_TILE_GROUP) || (type == OBU_FRAME))
567 hasFrame = TRUE;
568 }
569
570 if (!av1_vt_reserve(vt, SrcSize))
571 return -1;
572
573 /* Keep every OBU but temporal delimiters and padding; a changed sequence header starts a
574 * new stream, so a new session. */
575 size_t sampleSize = 0;
576 size_t pos = 0;
577 while (pos < SrcSize)
578 {
579 const size_t start = pos;
580 const BYTE header = pSrcData[pos++];
581 const unsigned type = (header >> 3) & 0x0f;
582 if (header & 0x04) /* obu_extension_flag */
583 pos++;
584 UINT64 size = 0;
585 if (!av1_vt_leb128(pSrcData, SrcSize, &pos, &size)) /* checked above */
586 return AV1_VT_INVALID;
587 const size_t payload = pos;
588 pos += WINPR_ASSERTING_INT_CAST(size_t, size);
589
590 if ((type == OBU_TEMPORAL_DELIMITER) || (type == OBU_PADDING))
591 continue;
592 if (type == OBU_SEQUENCE_HEADER)
593 {
594 const size_t obuSize = pos - start;
595 if (!vt->session || (obuSize != vt->seqSize) ||
596 (memcmp(vt->seq, &pSrcData[start], obuSize) != 0))
597 {
598 const INT32 rc = av1_vt_open(vt, &pSrcData[start], obuSize, &pSrcData[payload],
599 WINPR_ASSERTING_INT_CAST(size_t, size));
600 if (rc != TRUE)
601 return rc;
602 }
603 }
604 memcpy(&vt->sample[sampleSize], &pSrcData[start], pos - start);
605 sampleSize += pos - start;
606 }
607
608 if (!vt->session)
609 {
610 WLog_Print(vt->log, WLOG_WARN, "AV1 stream without a sequence header");
611 return AV1_VT_INVALID;
612 }
613 if (!hasFrame)
614 return FALSE;
615
616 CMBlockBufferRef block = nullptr;
617 CMSampleBufferRef sample = nullptr;
618 OSStatus status =
619 CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, vt->sample, sampleSize,
620 kCFAllocatorNull, nullptr, 0, sampleSize, 0, &block);
621 if (status == noErr)
622 {
623 const size_t sizes[] = { sampleSize };
624 status = CMSampleBufferCreateReady(kCFAllocatorDefault, block, vt->format, 1, 0, nullptr, 1,
625 sizes, &sample);
626 }
627 if (status != noErr)
628 {
629 if (block)
630 CFRelease(block);
631 WLog_Print(vt->log, WLOG_ERROR, "AV1 sample buffer: %d", (int)status);
632 return -1;
633 }
634
635 /* Synchronous: the output callback runs before the call returns (and the wait makes sure). */
636 vt->outStatus = noErr;
637 VTDecodeInfoFlags info = 0;
638 status = VTDecompressionSessionDecodeFrame(vt->session, sample, 0, nullptr, &info);
639 if (status == noErr)
640 status = VTDecompressionSessionWaitForAsynchronousFrames(vt->session);
641 CFRelease(sample);
642 CFRelease(block);
643 if (status == noErr)
644 status = vt->outStatus;
645
646 if (status != noErr)
647 {
648 av1_vt_release_picture(vt);
649 if (!vt->decoded)
650 {
651 /* The first frame of the stream: a software decoder can take over from here. */
652 WLog_Print(vt->log, WLOG_INFO,
653 "VideoToolbox can't decode this AV1 stream (%d): software decoder",
654 (int)status);
655 av1_vt_close(vt);
656 return AV1_VT_UNSUPPORTED;
657 }
658 WLog_Print(vt->log, WLOG_WARN, "VideoToolbox AV1 decode: %d", (int)status);
659 return -1;
660 }
661
662 if (!vt->pending)
663 return FALSE;
664 vt->decoded = TRUE;
665 return av1_vt_picture(vt, pic);
666}
667
668void av1_vt_reset(AV1_VT* vt)
669{
670 if (vt)
671 av1_vt_close(vt);
672}
673
674AV1_VT* av1_vt_new(wLog* log)
675{
676 if (__builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, *))
677 {
678 if (!VTIsHardwareDecodeSupported(AV1_VT_CODEC_TYPE))
679 return nullptr;
680 }
681 else
682 return nullptr;
683
684 AV1_VT* vt = calloc(1, sizeof(AV1_VT));
685 if (!vt)
686 return nullptr;
687 vt->log = log;
688 return vt;
689}
690
691void av1_vt_free(AV1_VT* vt)
692{
693 if (!vt)
694 return;
695 av1_vt_close(vt);
696 for (size_t i = 0; i < 2; i++)
697 winpr_aligned_free(vt->chroma[i]);
698 free(vt->sample);
699 free(vt);
700}