28#include <freerdp/config.h>
32#include <winpr/assert.h>
33#include <winpr/cast.h>
36#include <CoreFoundation/CoreFoundation.h>
37#include <CoreMedia/CoreMedia.h>
38#include <CoreVideo/CoreVideo.h>
39#include <VideoToolbox/VideoToolbox.h>
41#include "av1_videotoolbox.h"
44#define AV1_VT_CODEC_TYPE ((CMVideoCodecType)0x61763031)
48 OBU_SEQUENCE_HEADER = 1,
49 OBU_TEMPORAL_DELIMITER = 2,
59 VTDecompressionSessionRef session;
60 CMVideoFormatDescriptionRef format;
67 size_t sampleCapacity;
70 CVImageBufferRef pending;
71 CVPixelBufferRef handedOut;
86static UINT32 av1_vt_bits(AV1_VT_BITS* b,
unsigned count)
89 for (
unsigned i = 0; i < count; i++)
91 if ((b->bit >> 3) >= b->size)
96 v = (v << 1) | ((b->p[b->bit >> 3] >> (7 - (b->bit & 7))) & 1);
102static void av1_vt_uvlc(AV1_VT_BITS* b)
104 unsigned leadingZeros = 0;
105 while (!b->overrun && (av1_vt_bits(b, 1) == 0))
107 if (++leadingZeros >= 32)
113 (void)av1_vt_bits(b, leadingZeros);
126 UINT32 chromaSamplePosition;
132static BOOL av1_vt_parse_sequence_header(
const BYTE* p,
size_t size, AV1_VT_SEQ* s)
134 AV1_VT_BITS b = { .p = p, .size = size };
135 memset(s, 0,
sizeof(*s));
137 s->profile = av1_vt_bits(&b, 3);
138 (void)av1_vt_bits(&b, 1);
139 const UINT32 reduced = av1_vt_bits(&b, 1);
141 s->level0 = av1_vt_bits(&b, 5);
144 UINT32 decoderModelInfo = 0;
145 UINT32 bufferDelayLength = 0;
146 if (av1_vt_bits(&b, 1))
148 (void)av1_vt_bits(&b, 32);
149 (void)av1_vt_bits(&b, 32);
150 if (av1_vt_bits(&b, 1))
152 decoderModelInfo = av1_vt_bits(&b, 1);
153 if (decoderModelInfo)
155 bufferDelayLength = av1_vt_bits(&b, 5) + 1;
156 (void)av1_vt_bits(&b, 32);
157 (void)av1_vt_bits(&b, 10);
160 const UINT32 initialDisplayDelay = av1_vt_bits(&b, 1);
161 const UINT32 operatingPoints = av1_vt_bits(&b, 5) + 1;
162 for (UINT32 i = 0; i < operatingPoints; i++)
164 (void)av1_vt_bits(&b, 12);
165 const UINT32 level = av1_vt_bits(&b, 5);
166 const UINT32 tier = (level > 7) ? av1_vt_bits(&b, 1) : 0;
172 if (decoderModelInfo && av1_vt_bits(&b, 1))
174 (void)av1_vt_bits(&b, bufferDelayLength);
175 (void)av1_vt_bits(&b, bufferDelayLength);
176 (void)av1_vt_bits(&b, 1);
178 if (initialDisplayDelay && av1_vt_bits(&b, 1))
179 (void)av1_vt_bits(&b, 4);
183 const UINT32 widthBits = av1_vt_bits(&b, 4) + 1;
184 const UINT32 heightBits = av1_vt_bits(&b, 4) + 1;
185 s->width = av1_vt_bits(&b, widthBits) + 1;
186 s->height = av1_vt_bits(&b, heightBits) + 1;
187 if (!reduced && av1_vt_bits(&b, 1))
188 (void)av1_vt_bits(&b, 7);
189 (void)av1_vt_bits(&b, 3);
192 (void)av1_vt_bits(&b, 4);
193 const UINT32 orderHint = av1_vt_bits(&b, 1);
195 (void)av1_vt_bits(&b, 2);
196 UINT32 forceScreenContentTools = 2;
197 if (!av1_vt_bits(&b, 1))
198 forceScreenContentTools = av1_vt_bits(&b, 1);
199 if (forceScreenContentTools > 0)
201 if (!av1_vt_bits(&b, 1))
202 (void)av1_vt_bits(&b, 1);
205 (void)av1_vt_bits(&b, 3);
207 (void)av1_vt_bits(&b, 3);
210 s->highBitdepth = av1_vt_bits(&b, 1);
211 if ((s->profile == 2) && s->highBitdepth)
212 s->twelveBit = av1_vt_bits(&b, 1);
213 s->mono = (s->profile == 1) ? 0 : av1_vt_bits(&b, 1);
214 UINT32 primaries = 2;
217 if (av1_vt_bits(&b, 1))
219 primaries = av1_vt_bits(&b, 8);
220 transfer = av1_vt_bits(&b, 8);
221 matrix = av1_vt_bits(&b, 8);
225 s->fullRange = av1_vt_bits(&b, 1);
226 s->subsamplingX = s->subsamplingY = 1;
228 else if ((primaries == 1) && (transfer == 13) && (matrix == 0))
232 s->fullRange = av1_vt_bits(&b, 1);
234 s->subsamplingX = s->subsamplingY = 1;
235 else if (s->profile == 2)
239 s->subsamplingX = av1_vt_bits(&b, 1);
240 s->subsamplingY = s->subsamplingX ? av1_vt_bits(&b, 1) : 0;
245 if (s->subsamplingX && s->subsamplingY)
246 s->chromaSamplePosition = av1_vt_bits(&b, 2);
253static void av1_vt_release_picture(AV1_VT* vt)
257 CVPixelBufferUnlockBaseAddress(vt->handedOut, kCVPixelBufferLock_ReadOnly);
258 CVPixelBufferRelease(vt->handedOut);
259 vt->handedOut =
nullptr;
263 CVPixelBufferRelease(vt->pending);
264 vt->pending =
nullptr;
268static void av1_vt_close(AV1_VT* vt)
270 av1_vt_release_picture(vt);
273 VTDecompressionSessionInvalidate(vt->session);
274 CFRelease(vt->session);
275 vt->session =
nullptr;
279 CFRelease(vt->format);
280 vt->format =
nullptr;
288static void av1_vt_output(
void* decompressionOutputRefCon,
289 WINPR_ATTR_UNUSED
void* sourceFrameRefCon, OSStatus status,
290 WINPR_ATTR_UNUSED VTDecodeInfoFlags infoFlags,
291 CVImageBufferRef imageBuffer,
292 WINPR_ATTR_UNUSED CMTime presentationTimeStamp,
293 WINPR_ATTR_UNUSED CMTime presentationDuration)
295 AV1_VT* vt = decompressionOutputRefCon;
297 vt->outStatus = status;
298 if ((status != noErr) || !imageBuffer)
301 CVPixelBufferRelease(vt->pending);
302 vt->pending = CVPixelBufferRetain(imageBuffer);
305static CFDictionaryRef av1_vt_dictionary(
const void* key,
const void* value)
307 const void* keys[] = { key };
308 const void* values[] = { value };
309 return CFDictionaryCreate(kCFAllocatorDefault, keys, values, 1, &kCFTypeDictionaryKeyCallBacks,
310 &kCFTypeDictionaryValueCallBacks);
314static INT32 av1_vt_open(AV1_VT* vt,
const BYTE* obu,
size_t obuSize,
const BYTE* payload,
317 AV1_VT_SEQ s = WINPR_C_ARRAY_INIT;
318 if (!av1_vt_parse_sequence_header(payload, payloadSize, &s))
320 WLog_Print(vt->log, WLOG_WARN,
"AV1 sequence header too short");
321 return AV1_VT_INVALID;
328 const BOOL i420 = (s.subsamplingX == 1) && (s.subsamplingY == 1);
329 const BOOL i444 = (s.subsamplingX == 0) && (s.subsamplingY == 0);
330 if (s.highBitdepth || s.mono || (!i420 && !i444))
332 WLog_Print(vt->log, WLOG_DEBUG,
333 "AV1 profile %" PRIu32
" high bit depth %" PRIu32
" mono %" PRIu32
334 " subsampling %" PRIu32
"x%" PRIu32
": not for VideoToolbox",
335 s.profile, s.highBitdepth, s.mono, s.subsamplingX, s.subsamplingY);
336 return AV1_VT_UNSUPPORTED;
342 BYTE* av1c = malloc(4 + obuSize);
346 av1c[1] = (BYTE)((s.profile << 5) | (s.level0 & 0x1f));
347 av1c[2] = (BYTE)((s.tier0 << 7) | (s.highBitdepth << 6) | (s.twelveBit << 5) | (s.mono << 4) |
348 (s.subsamplingX << 3) | (s.subsamplingY << 2) | (s.chromaSamplePosition & 3));
350 memcpy(&av1c[4], obu, obuSize);
352 INT32 rc = AV1_VT_UNSUPPORTED;
354 CFDataCreate(kCFAllocatorDefault, av1c, WINPR_ASSERTING_INT_CAST(CFIndex, 4 + obuSize));
356 CFDictionaryRef atoms = record ? av1_vt_dictionary(CFSTR(
"av1C"), record) : nullptr;
357 CFDictionaryRef extensions =
358 atoms ? av1_vt_dictionary(kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
364 OSType pixelFormat = 0;
366 pixelFormat = s.fullRange ? kCVPixelFormatType_444YpCbCr8BiPlanarFullRange
367 : kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange;
369 pixelFormat = s.fullRange ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
370 : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
371 SInt32 pf = WINPR_ASSERTING_INT_CAST(SInt32, pixelFormat);
372 CFNumberRef pfNumber = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pf);
373 CFDictionaryRef attributes =
374 pfNumber ? av1_vt_dictionary(kCVPixelBufferPixelFormatTypeKey, pfNumber) : nullptr;
375 CFDictionaryRef specification = av1_vt_dictionary(
376 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, kCFBooleanTrue);
378 if (!extensions || !attributes || !specification)
384 OSStatus status = CMVideoFormatDescriptionCreate(
385 kCFAllocatorDefault, AV1_VT_CODEC_TYPE, WINPR_ASSERTING_INT_CAST(int32_t, s.width),
386 WINPR_ASSERTING_INT_CAST(int32_t, s.height), extensions, &vt->format);
389 WLog_Print(vt->log, WLOG_WARN,
"CMVideoFormatDescriptionCreate: %d", (
int)status);
393 const VTDecompressionOutputCallbackRecord callback = { .decompressionOutputCallback =
395 .decompressionOutputRefCon = vt };
396 status = VTDecompressionSessionCreate(kCFAllocatorDefault, vt->format, specification,
397 attributes, &callback, &vt->session);
400 WLog_Print(vt->log, WLOG_INFO,
401 "VideoToolbox can't decode AV1 profile %" PRIu32
" %s %" PRIu32
"x%" PRIu32
402 " (%d): software decoder",
403 s.profile, i444 ?
"4:4:4" :
"4:2:0", s.width, s.height, (
int)status);
406 (void)VTSessionSetProperty(vt->session, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
408 vt->seq = malloc(obuSize);
414 memcpy(vt->seq, obu, obuSize);
415 vt->seqSize = obuSize;
417 WLog_Print(vt->log, WLOG_DEBUG,
418 "VideoToolbox AV1 session: profile %" PRIu32
" level %" PRIu32
" %s %" PRIu32
419 "x%" PRIu32
" %s range",
420 s.profile, s.level0, i444 ?
"4:4:4" :
"4:2:0", s.width, s.height,
421 s.fullRange ?
"full" :
"video");
426 CFRelease(specification);
428 CFRelease(attributes);
432 CFRelease(extensions);
442static BOOL av1_vt_leb128(
const BYTE* p,
size_t size,
size_t* pos, UINT64* value)
445 for (
unsigned i = 0; i < 8; i++)
449 const BYTE c = p[(*pos)++];
450 *value |= (UINT64)(c & 0x7f) << (7 * i);
457static BOOL av1_vt_reserve(AV1_VT* vt,
size_t size)
459 if (size <= vt->sampleCapacity)
461 BYTE* p = realloc(vt->sample, size);
465 vt->sampleCapacity = size;
472 CVPixelBufferRef buffer = vt->pending;
473 vt->pending =
nullptr;
474 if (CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess)
476 CVPixelBufferRelease(buffer);
479 vt->handedOut = buffer;
481 const size_t planes = CVPixelBufferGetPlaneCount(buffer);
482 pic->width = WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetWidth(buffer));
483 pic->height = WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetHeight(buffer));
484 pic->
i444 = vt->i444;
485 pic->planes[0] = CVPixelBufferGetBaseAddressOfPlane(buffer, 0);
487 WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetBytesPerRowOfPlane(buffer, 0));
491 for (
size_t i = 1; i < 3; i++)
493 pic->planes[i] = CVPixelBufferGetBaseAddressOfPlane(buffer, i);
495 WINPR_ASSERTING_INT_CAST(UINT32, CVPixelBufferGetBytesPerRowOfPlane(buffer, i));
501 WLog_Print(vt->log, WLOG_ERROR,
"VideoToolbox picture with %" PRIuz
" planes", planes);
505 const size_t cw = CVPixelBufferGetWidthOfPlane(buffer, 1);
506 const size_t ch = CVPixelBufferGetHeightOfPlane(buffer, 1);
507 const size_t stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1);
508 const BYTE* cbcr = CVPixelBufferGetBaseAddressOfPlane(buffer, 1);
509 if (!cbcr || (stride < 2 * cw))
511 if (cw * ch > vt->chromaSize)
513 for (
size_t i = 0; i < 2; i++)
515 winpr_aligned_free(vt->chroma[i]);
516 vt->chroma[i] = winpr_aligned_malloc(cw * ch, 64);
518 vt->chromaSize = (vt->chroma[0] && vt->chroma[1]) ? cw * ch : 0;
519 if (vt->chromaSize == 0)
522 for (
size_t y = 0; y < ch; y++)
524 const BYTE* src = &cbcr[y * stride];
525 BYTE* cb = &vt->chroma[0][y * cw];
526 BYTE* cr = &vt->chroma[1][y * cw];
527 for (
size_t x = 0; x < cw; x++)
530 cr[x] = src[2 * x + 1];
533 pic->planes[1] = vt->chroma[0];
534 pic->planes[2] = vt->chroma[1];
535 pic->strides[1] = pic->strides[2] = WINPR_ASSERTING_INT_CAST(UINT32, cw);
539INT32 av1_vt_decode(AV1_VT* vt,
const BYTE* pSrcData, UINT32 SrcSize,
AV1_VT_PICTURE* pic)
542 WINPR_ASSERT(pSrcData || (SrcSize == 0));
545 av1_vt_release_picture(vt);
549 BOOL hasFrame = FALSE;
550 for (
size_t pos = 0; pos < SrcSize;)
552 const size_t start = pos;
553 const BYTE header = pSrcData[pos++];
554 const unsigned type = (header >> 3) & 0x0f;
558 if (((header & 0x02) == 0) || !av1_vt_leb128(pSrcData, SrcSize, &pos, &size) ||
559 (size > SrcSize - pos))
561 WLog_Print(vt->log, WLOG_WARN,
"AV1 OBU at %" PRIuz
" of %" PRIu32
" is malformed",
563 return AV1_VT_INVALID;
565 pos += WINPR_ASSERTING_INT_CAST(
size_t, size);
566 if ((type == OBU_FRAME_HEADER) || (type == OBU_TILE_GROUP) || (type == OBU_FRAME))
570 if (!av1_vt_reserve(vt, SrcSize))
575 size_t sampleSize = 0;
577 while (pos < SrcSize)
579 const size_t start = pos;
580 const BYTE header = pSrcData[pos++];
581 const unsigned type = (header >> 3) & 0x0f;
585 if (!av1_vt_leb128(pSrcData, SrcSize, &pos, &size))
586 return AV1_VT_INVALID;
587 const size_t payload = pos;
588 pos += WINPR_ASSERTING_INT_CAST(
size_t, size);
590 if ((type == OBU_TEMPORAL_DELIMITER) || (type == OBU_PADDING))
592 if (type == OBU_SEQUENCE_HEADER)
594 const size_t obuSize = pos - start;
595 if (!vt->session || (obuSize != vt->seqSize) ||
596 (memcmp(vt->seq, &pSrcData[start], obuSize) != 0))
598 const INT32 rc = av1_vt_open(vt, &pSrcData[start], obuSize, &pSrcData[payload],
599 WINPR_ASSERTING_INT_CAST(
size_t, size));
604 memcpy(&vt->sample[sampleSize], &pSrcData[start], pos - start);
605 sampleSize += pos - start;
610 WLog_Print(vt->log, WLOG_WARN,
"AV1 stream without a sequence header");
611 return AV1_VT_INVALID;
616 CMBlockBufferRef block =
nullptr;
617 CMSampleBufferRef sample =
nullptr;
619 CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, vt->sample, sampleSize,
620 kCFAllocatorNull,
nullptr, 0, sampleSize, 0, &block);
623 const size_t sizes[] = { sampleSize };
624 status = CMSampleBufferCreateReady(kCFAllocatorDefault, block, vt->format, 1, 0,
nullptr, 1,
631 WLog_Print(vt->log, WLOG_ERROR,
"AV1 sample buffer: %d", (
int)status);
636 vt->outStatus = noErr;
637 VTDecodeInfoFlags info = 0;
638 status = VTDecompressionSessionDecodeFrame(vt->session, sample, 0,
nullptr, &info);
640 status = VTDecompressionSessionWaitForAsynchronousFrames(vt->session);
644 status = vt->outStatus;
648 av1_vt_release_picture(vt);
652 WLog_Print(vt->log, WLOG_INFO,
653 "VideoToolbox can't decode this AV1 stream (%d): software decoder",
656 return AV1_VT_UNSUPPORTED;
658 WLog_Print(vt->log, WLOG_WARN,
"VideoToolbox AV1 decode: %d", (
int)status);
665 return av1_vt_picture(vt, pic);
668void av1_vt_reset(AV1_VT* vt)
674AV1_VT* av1_vt_new(wLog* log)
676 if (__builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, *))
678 if (!VTIsHardwareDecodeSupported(AV1_VT_CODEC_TYPE))
684 AV1_VT* vt = calloc(1,
sizeof(AV1_VT));
691void av1_vt_free(AV1_VT* vt)
696 for (
size_t i = 0; i < 2; i++)
697 winpr_aligned_free(vt->chroma[i]);