FreeRDP
Loading...
Searching...
No Matches
mac_capture.m
1
19#import <Foundation/Foundation.h>
20#import <CoreMedia/CoreMedia.h>
21#import <ScreenCaptureKit/ScreenCaptureKit.h>
22
23#include <stdatomic.h>
24
25#include <winpr/assert.h>
26
27#include <freerdp/log.h>
28
29#include "mac_capture.h"
30
31#define TAG SERVER_TAG("shadow.mac")
32
33/* ScreenCaptureKit answers through completion handlers; the shadow server
34 * runs no run loop, so wait for them, but never forever. */
35static const int64_t mac_capture_timeout_ns = 10LL * NSEC_PER_SEC;
36
37@interface MacCaptureOutput : NSObject <SCStreamOutput, SCStreamDelegate>
38{
39 @public
40 macCaptureConfig config;
41 float *audioBuffer;
42 size_t audioBufferFrames;
43 /* the stream ended by itself: it must not be stopped again */
44 atomic_bool stopped;
45}
46@end
47
48struct mac_capture
49{
50 macCaptureConfig config;
51 MacCaptureOutput *output;
52 SCStream *stream;
53 dispatch_queue_t videoQueue;
54 dispatch_queue_t audioQueue;
55 BOOL running;
56};
57
58static void mac_capture_log_error(const char *what, NSError *error)
59{
60 if (error)
61 WLog_ERR(TAG, "%s: %s (%s %ld)", what, error.localizedDescription.UTF8String,
62 error.domain.UTF8String, (long)error.code);
63 else
64 WLog_ERR(TAG, "%s: timed out", what);
65}
66
67@implementation MacCaptureOutput
68
69- (void)dealloc
70{
71 free(audioBuffer);
72}
73
74- (void)onScreen:(CMSampleBufferRef)sampleBuffer
75{
76 CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
77 if (!attachments || (CFArrayGetCount(attachments) < 1))
78 return;
79
80 NSDictionary *info = (__bridge NSDictionary *)CFArrayGetValueAtIndex(attachments, 0);
81 NSNumber *status = info[SCStreamFrameInfoStatus];
82
83 /* Idle frames (nothing changed) carry no picture */
84 if (!status || (status.integerValue != SCFrameStatusComplete))
85 return;
86
87 CVPixelBufferRef frame = CMSampleBufferGetImageBuffer(sampleBuffer);
88 if (!frame)
89 return;
90
91 NSArray *dirty = info[SCStreamFrameInfoDirtyRects];
92 const size_t count = dirty.count;
93 CGRect stackRects[32];
94 CGRect *rects = stackRects;
95
96 if (count > ARRAYSIZE(stackRects))
97 {
98 rects = calloc(count, sizeof(CGRect));
99 if (!rects)
100 return;
101 }
102
103 size_t used = 0;
104 for (NSDictionary *value in dirty)
105 {
106 CGRect rect = CGRectNull;
107 if (CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)value, &rect))
108 rects[used++] = rect;
109 }
110
111 /* No rectangles at all: take the whole frame */
112 if (used == 0)
113 {
114 rects[0] = CGRectMake(0, 0, (CGFloat)CVPixelBufferGetWidth(frame),
115 (CGFloat)CVPixelBufferGetHeight(frame));
116 used = 1;
117 }
118
119 config.onFrame(config.context, frame, rects, used);
120
121 if (rects != stackRects)
122 free(rects);
123}
124
125- (void)onAudio:(CMSampleBufferRef)sampleBuffer
126{
127 if (!config.onAudio)
128 return;
129
130 CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
131 const AudioStreamBasicDescription *asbd =
132 format ? CMAudioFormatDescriptionGetStreamBasicDescription(format) : NULL;
133 if (!asbd || (asbd->mFormatID != kAudioFormatLinearPCM) ||
134 !(asbd->mFormatFlags & kAudioFormatFlagIsFloat) || (asbd->mBitsPerChannel != 32))
135 return;
136
137 const UINT32 channels = asbd->mChannelsPerFrame;
138 const size_t frames = (size_t)CMSampleBufferGetNumSamples(sampleBuffer);
139 if ((channels == 0) || (frames == 0))
140 return;
141
142 /* ScreenCaptureKit hands out one buffer per channel (non-interleaved) */
143 size_t listSize = 0;
144 if (CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, &listSize, NULL, 0,
145 NULL, NULL, 0, NULL) != noErr)
146 return;
147
148 AudioBufferList *list = calloc(1, listSize);
149 if (!list)
150 return;
151
152 CMBlockBufferRef block = NULL;
153 if (CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
154 sampleBuffer, NULL, list, listSize, NULL, NULL,
155 kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &block) != noErr)
156 {
157 free(list);
158 return;
159 }
160
161 if (audioBufferFrames < frames)
162 {
163 float *tmp = realloc(audioBuffer, frames * channels * sizeof(float));
164 if (!tmp)
165 goto out;
166 audioBuffer = tmp;
167 audioBufferFrames = frames;
168 }
169
170 const BOOL interleaved = !(asbd->mFormatFlags & kAudioFormatFlagIsNonInterleaved);
171 if (interleaved)
172 {
173 if ((list->mNumberBuffers < 1) ||
174 (list->mBuffers[0].mDataByteSize < frames * channels * sizeof(float)))
175 goto out;
176 memcpy(audioBuffer, list->mBuffers[0].mData, frames * channels * sizeof(float));
177 }
178 else
179 {
180 if (list->mNumberBuffers < channels)
181 goto out;
182 for (UINT32 c = 0; c < channels; c++)
183 {
184 const float *src = list->mBuffers[c].mData;
185 if (list->mBuffers[c].mDataByteSize < frames * sizeof(float))
186 goto out;
187 for (size_t f = 0; f < frames; f++)
188 audioBuffer[f * channels + c] = src[f];
189 }
190 }
191
192 config.onAudio(config.context, audioBuffer, frames, channels, (UINT32)asbd->mSampleRate);
193
194out:
195 if (block)
196 CFRelease(block);
197 free(list);
198}
199
200- (void)stream:(SCStream *)stream
201 didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
202 ofType:(SCStreamOutputType)type
203{
204 WINPR_UNUSED(stream);
205
206 if (!CMSampleBufferIsValid(sampleBuffer))
207 return;
208
209 switch (type)
210 {
211 case SCStreamOutputTypeScreen:
212 [self onScreen:sampleBuffer];
213 break;
214 case SCStreamOutputTypeAudio:
215 [self onAudio:sampleBuffer];
216 break;
217 default:
218 break;
219 }
220}
221
222- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error
223{
224 WINPR_UNUSED(stream);
225 atomic_store(&stopped, true);
226 WLog_WARN(TAG, "screen capture stopped: %s (%s %ld)",
227 error ? error.localizedDescription.UTF8String : "no reason given",
228 error ? error.domain.UTF8String : "-", error ? (long)error.code : 0L);
229 if (config.onStopped)
230 config.onStopped(config.context);
231}
232
233@end
234
235BOOL mac_capture_check_access(void)
236{
237 if (CGPreflightScreenCaptureAccess())
238 return TRUE;
239
240 WLog_ERR(TAG, "no permission to record the screen: allow this program in System Settings > "
241 "Privacy & Security > Screen & System Audio Recording, then start it again");
242 (void)CGRequestScreenCaptureAccess();
243 return FALSE;
244}
245
246static SCDisplay *mac_capture_find_display(CGDirectDisplayID displayID)
247{
248 __block SCShareableContent *content = nil;
249 __block NSError *error = nil;
250 dispatch_semaphore_t done = dispatch_semaphore_create(0);
251
252 [SCShareableContent
253 getShareableContentExcludingDesktopWindows:NO
254 onScreenWindowsOnly:YES
255 completionHandler:^(SCShareableContent *c, NSError *e) {
256 content = c;
257 error = e;
258 dispatch_semaphore_signal(done);
259 }];
260
261 if (dispatch_semaphore_wait(done, dispatch_time(DISPATCH_TIME_NOW, mac_capture_timeout_ns)) !=
262 0)
263 error = nil;
264
265 if (!content)
266 {
267 mac_capture_log_error("the list of displays", error);
268 return nil;
269 }
270
271 for (SCDisplay *display in content.displays)
272 {
273 if (display.displayID == displayID)
274 return display;
275 }
276
277 WLog_ERR(TAG, "display %" PRIu32 " is not among the ones that can be captured", displayID);
278 return nil;
279}
280
281macCapture *mac_capture_new(const macCaptureConfig *config)
282{
283 WINPR_ASSERT(config);
284 WINPR_ASSERT(config->onFrame);
285
286 macCapture *capture = calloc(1, sizeof(macCapture));
287 if (!capture)
288 return NULL;
289
290 capture->config = *config;
291
292 @autoreleasepool
293 {
294 SCDisplay *display = mac_capture_find_display(config->display);
295 if (!display)
296 goto fail;
297
298 SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:display
299 excludingWindows:@[]];
300
301 SCStreamConfiguration *streamConfig = [[SCStreamConfiguration alloc] init];
302 streamConfig.width = config->width;
303 streamConfig.height = config->height;
304 streamConfig.pixelFormat = kCVPixelFormatType_32BGRA;
305 streamConfig.colorSpaceName = kCGColorSpaceSRGB;
306 streamConfig.minimumFrameInterval = CMTimeMake(1, (int32_t)MAX(config->fps, 1));
307 streamConfig.showsCursor = config->showCursor;
308 /* the subsystem keeps the newest frame until it was sent: leave
309 * ScreenCaptureKit enough buffers to go on */
310 streamConfig.queueDepth = 5;
311
312 if (config->audio)
313 {
314 if (@available(macOS 13.0, *))
315 {
316 streamConfig.capturesAudio = YES;
317 streamConfig.excludesCurrentProcessAudio = YES;
318 streamConfig.sampleRate = (NSInteger)config->audioRate;
319 streamConfig.channelCount = (NSInteger)config->audioChannels;
320 }
321 else
322 {
323 WLog_WARN(TAG, "sound needs macOS 13 or later: no sound");
324 capture->config.audio = FALSE;
325 }
326 }
327
328 capture->output = [[MacCaptureOutput alloc] init];
329 capture->output->config = capture->config;
330 capture->stream = [[SCStream alloc] initWithFilter:filter
331 configuration:streamConfig
332 delegate:capture->output];
333 if (!capture->stream)
334 goto fail;
335
336 capture->videoQueue = dispatch_queue_create("freerdp.shadow.mac.video", NULL);
337 NSError *error = nil;
338 if (![capture->stream addStreamOutput:capture->output
339 type:SCStreamOutputTypeScreen
340 sampleHandlerQueue:capture->videoQueue
341 error:&error])
342 {
343 mac_capture_log_error("screen output", error);
344 goto fail;
345 }
346
347 if (capture->config.audio)
348 {
349 if (@available(macOS 13.0, *))
350 {
351 capture->audioQueue = dispatch_queue_create("freerdp.shadow.mac.audio", NULL);
352 if (![capture->stream addStreamOutput:capture->output
353 type:SCStreamOutputTypeAudio
354 sampleHandlerQueue:capture->audioQueue
355 error:&error])
356 {
357 mac_capture_log_error("sound output: no sound", error);
358 capture->output->config.onAudio = NULL;
359 }
360 }
361 }
362 }
363
364 return capture;
365
366fail:
367 mac_capture_free(capture);
368 return NULL;
369}
370
371static BOOL mac_capture_wait(const char *what, void (^call)(void (^)(NSError *)))
372{
373 __block NSError *error = nil;
374 __block BOOL answered = NO;
375 dispatch_semaphore_t done = dispatch_semaphore_create(0);
376
377 call(^(NSError *e) {
378 error = e;
379 answered = YES;
380 dispatch_semaphore_signal(done);
381 });
382
383 if (dispatch_semaphore_wait(done, dispatch_time(DISPATCH_TIME_NOW, mac_capture_timeout_ns)) !=
384 0 ||
385 !answered || error)
386 {
387 mac_capture_log_error(what, answered ? error : nil);
388 return FALSE;
389 }
390 return TRUE;
391}
392
393BOOL mac_capture_start(macCapture *capture)
394{
395 if (!capture || !capture->stream)
396 return FALSE;
397 if (capture->running)
398 return TRUE;
399
400 SCStream *stream = capture->stream;
401 capture->running = mac_capture_wait("starting the screen capture", ^(void (^done)(NSError *)) {
402 [stream startCaptureWithCompletionHandler:done];
403 });
404 return capture->running;
405}
406
407void mac_capture_stop(macCapture *capture)
408{
409 if (!capture || !capture->stream || !capture->running)
410 return;
411 if (atomic_load(&capture->output->stopped))
412 {
413 capture->running = FALSE;
414 return;
415 }
416
417 SCStream *stream = capture->stream;
418 (void)mac_capture_wait("stopping the screen capture", ^(void (^done)(NSError *)) {
419 [stream stopCaptureWithCompletionHandler:done];
420 });
421 capture->running = FALSE;
422}
423
424void mac_capture_free(macCapture *capture)
425{
426 if (!capture)
427 return;
428
429 mac_capture_stop(capture);
430
431 @autoreleasepool
432 {
433 capture->stream = nil;
434 capture->output = nil;
435 capture->videoQueue = nil;
436 capture->audioQueue = nil;
437 }
438
439 free(capture);
440}