FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/addin.h>
38#include <freerdp/primitives.h>
39#include <freerdp/client/channels.h>
40#include <freerdp/client/geometry.h>
41#include <freerdp/client/video.h>
42#include <freerdp/channels/log.h>
43#include <freerdp/codec/h264.h>
44#include <freerdp/codec/yuv.h>
45#include <freerdp/timer.h>
46
47#define TAG CHANNELS_TAG("video.client")
48
49#include "video_main.h"
50
51typedef struct
52{
53 IWTSPlugin wtsPlugin;
54
55 IWTSListener* controlListener;
56 IWTSListener* dataListener;
57 GENERIC_LISTENER_CALLBACK* control_callback;
58 GENERIC_LISTENER_CALLBACK* data_callback;
59
60 VideoClientContext* context;
61 BOOL initialized;
62 rdpContext* rdpcontext;
63} VIDEO_PLUGIN;
64
65#define XF_VIDEO_UNLIMITED_RATE 31
66
67static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
68 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
69
70typedef struct
71{
72 BYTE PresentationId;
73 UINT32 ScaledWidth;
74 UINT32 ScaledHeight;
75
76 UINT64 startTimeStamp;
77 UINT64 publishOffset;
78 wStream* currentSample;
79 UINT64 lastPublishTime;
80 UINT64 nextPublishTime;
81 volatile LONG refCounter;
82 H264_CONTEXT* h264;
83 VideoSurface* surface;
84 MAPPED_GEOMETRY* geometry;
85 VideoClientContext* video;
86} PresentationContext;
87
88typedef struct
89{
90 BYTE PresentationId;
91 UINT64 publishTime;
92 UINT64 hnsDuration;
93 MAPPED_GEOMETRY* geometry;
94 UINT32 w, h;
95 UINT32 scanline;
96 BYTE* surfaceData;
97} VideoFrame;
98
100struct s_VideoClientContextPriv
101{
102 VideoClientContext* video;
103 GeometryClientContext* geometry;
104 wQueue* frames;
105 CRITICAL_SECTION framesLock;
106 wBufferPool* surfacePool;
107 UINT32 publishedFrames;
108 UINT32 droppedFrames;
109 UINT32 lastSentRate;
110 UINT64 nextFeedbackTime;
111 PresentationContext* currentPresentation;
112 FreeRDP_TimerID timerID;
113};
114
115static void PresentationContext_unref(PresentationContext** presentation);
116static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
117
118WINPR_ATTR_NODISCARD
119static const char* video_command_name(BYTE cmd)
120{
121 switch (cmd)
122 {
123 case TSMM_START_PRESENTATION:
124 return "start";
125 case TSMM_STOP_PRESENTATION:
126 return "stop";
127 default:
128 return "<unknown>";
129 }
130}
131
132static void video_client_context_set_geometry(VideoClientContext* video,
133 GeometryClientContext* geometry)
134{
135 WINPR_ASSERT(video);
136 WINPR_ASSERT(video->priv);
137 video->priv->geometry = geometry;
138}
139
140WINPR_ATTR_MALLOC(VideoClientContextPriv_free, 1)
141static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
142{
143 VideoClientContextPriv* ret = nullptr;
144
145 WINPR_ASSERT(video);
146 ret = calloc(1, sizeof(*ret));
147 if (!ret)
148 return nullptr;
149
150 ret->frames = Queue_New(TRUE, 10, 2);
151 if (!ret->frames)
152 {
153 WLog_ERR(TAG, "unable to allocate frames queue");
154 goto fail;
155 }
156
157 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
158 if (!ret->surfacePool)
159 {
160 WLog_ERR(TAG, "unable to create surface pool");
161 goto fail;
162 }
163
164 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
165 {
166 WLog_ERR(TAG, "unable to initialize frames lock");
167 goto fail;
168 }
169
170 ret->video = video;
171
172 /* don't set to unlimited so that we have the chance to send a feedback in
173 * the first second (for servers that want feedback directly)
174 */
175 ret->lastSentRate = 30;
176 return ret;
177
178fail:
179 VideoClientContextPriv_free(ret);
180 return nullptr;
181}
182
183WINPR_ATTR_NODISCARD
184static BOOL PresentationContext_ref(PresentationContext* presentation)
185{
186 WINPR_ASSERT(presentation);
187
188 const LONG val = InterlockedIncrement(&presentation->refCounter);
189 return val > 0;
190}
191
192static void PresentationContext_free(PresentationContext* presentation)
193{
194 if (!presentation)
195 return;
196
197 MAPPED_GEOMETRY* geometry = presentation->geometry;
198 if (geometry)
199 {
200 geometry->MappedGeometryUpdate = nullptr;
201 geometry->MappedGeometryClear = nullptr;
202 geometry->custom = nullptr;
203 mappedGeometryUnref(geometry);
204 }
205
206 h264_context_free(presentation->h264);
207 Stream_Free(presentation->currentSample, TRUE);
208 presentation->video->deleteSurface(presentation->video, presentation->surface);
209 free(presentation);
210}
211
212WINPR_ATTR_MALLOC(PresentationContext_free, 1)
213static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
214 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
215{
216 size_t s = 4ULL * width * height;
217 PresentationContext* ret = nullptr;
218
219 WINPR_ASSERT(video);
220
221 if (s > INT32_MAX)
222 return nullptr;
223
224 ret = calloc(1, sizeof(*ret));
225 if (!ret)
226 return nullptr;
227
228 ret->video = video;
229 ret->PresentationId = PresentationId;
230
231 ret->h264 = h264_context_new(FALSE);
232 if (!ret->h264)
233 {
234 WLog_ERR(TAG, "unable to create a h264 context");
235 goto fail;
236 }
237 if (!h264_context_reset(ret->h264, width, height))
238 goto fail;
239
240 ret->currentSample = Stream_New(nullptr, 4096);
241 if (!ret->currentSample)
242 {
243 WLog_ERR(TAG, "unable to create current packet stream");
244 goto fail;
245 }
246
247 ret->surface = video->createSurface(video, x, y, width, height);
248 if (!ret->surface)
249 {
250 WLog_ERR(TAG, "unable to create surface");
251 goto fail;
252 }
253
254 if (!PresentationContext_ref(ret))
255 goto fail;
256
257 return ret;
258
259fail:
260 PresentationContext_free(ret);
261 return nullptr;
262}
263
264static void PresentationContext_unref(PresentationContext** ppresentation)
265{
266 WINPR_ASSERT(ppresentation);
267
268 PresentationContext* presentation = *ppresentation;
269 if (!presentation)
270 return;
271
272 if (InterlockedDecrement(&presentation->refCounter) > 0)
273 return;
274 *ppresentation = nullptr;
275
276 PresentationContext_free(presentation);
277}
278
279static void VideoFrame_free(VideoClientContextPriv* priv, VideoFrame* frame)
280{
281 WINPR_ASSERT(priv);
282 if (!frame)
283 return;
284
285 mappedGeometryUnref(frame->geometry);
286
287 BufferPool_Return(priv->surfacePool, frame->surfaceData);
288 free(frame);
289}
290
291WINPR_ATTR_MALLOC(VideoFrame_free, 1)
292static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
293 MAPPED_GEOMETRY* geom)
294{
295 WINPR_ASSERT(priv);
296 WINPR_ASSERT(presentation);
297 WINPR_ASSERT(geom);
298
299 const VideoSurface* surface = presentation->surface;
300 WINPR_ASSERT(surface);
301
302 VideoFrame* frame = calloc(1, sizeof(VideoFrame));
303 if (!frame)
304 goto fail;
305 frame->PresentationId = presentation->PresentationId;
306
307 mappedGeometryRef(geom);
308
309 frame->publishTime = presentation->lastPublishTime;
310 frame->geometry = geom;
311 frame->w = surface->alignedWidth;
312 frame->h = surface->alignedHeight;
313 frame->scanline = surface->scanline;
314
315 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
316 if (!frame->surfaceData)
317 goto fail;
318
319 return frame;
320
321fail:
322 VideoFrame_free(priv, frame);
323 return nullptr;
324}
325
326void VideoClientContextPriv_free(VideoClientContextPriv* priv)
327{
328 if (!priv)
329 return;
330
331 EnterCriticalSection(&priv->framesLock);
332
333 if (priv->frames)
334 {
335 while (Queue_Count(priv->frames))
336 {
337 VideoFrame* frame = Queue_Dequeue(priv->frames);
338 if (frame)
339 VideoFrame_free(priv, frame);
340 }
341 }
342
343 Queue_Free(priv->frames);
344 LeaveCriticalSection(&priv->framesLock);
345
346 DeleteCriticalSection(&priv->framesLock);
347
348 if (priv->currentPresentation)
349 PresentationContext_unref(&priv->currentPresentation);
350
351 BufferPool_Free(priv->surfacePool);
352 free(priv);
353}
354
355WINPR_ATTR_NODISCARD
356static UINT video_channel_write(VIDEO_PLUGIN* video, const BYTE* data, UINT32 length)
357{
358 WINPR_ASSERT(video);
359
360 if (!video->control_callback || !video->control_callback->channel_callback)
361 return ERROR_BAD_CONFIGURATION;
362 IWTSVirtualChannel* channel = video->control_callback->channel_callback->channel;
363 if (!channel || !channel->Write)
364 return ERROR_BAD_CONFIGURATION;
365 return channel->Write(channel, length, data, nullptr);
366}
367
368WINPR_ATTR_NODISCARD
369static UINT video_control_send_presentation_response(VideoClientContext* context,
370 TSMM_PRESENTATION_RESPONSE* resp)
371{
372 BYTE buf[12] = WINPR_C_ARRAY_INIT;
373
374 WINPR_ASSERT(context);
375 WINPR_ASSERT(resp);
376
377 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
378 WINPR_ASSERT(video);
379
380 wStream* s = Stream_New(buf, 12);
381 if (!s)
382 return CHANNEL_RC_NO_MEMORY;
383
384 Stream_Write_UINT32(s, 12); /* cbSize */
385 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
386 Stream_Write_UINT8(s, resp->PresentationId);
387 Stream_Zero(s, 3);
388 Stream_SealLength(s);
389 Stream_Free(s, FALSE);
390
391 return video_channel_write(video, buf, sizeof(buf));
392}
393
394WINPR_ATTR_NODISCARD
395static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
396{
397 WINPR_ASSERT(geometry);
398
399 PresentationContext* presentation = (PresentationContext*)geometry->custom;
400 WINPR_ASSERT(presentation);
401
402 RDP_RECT* r = &geometry->geometry.boundingRect;
403 WLog_DBG(TAG,
404 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
405 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
406 "-%" PRId16 "x%" PRId16 ")",
407 geometry->topLevelLeft, geometry->topLevelTop,
408 geometry->topLevelRight - geometry->topLevelLeft,
409 geometry->topLevelBottom - geometry->topLevelTop,
410
411 geometry->left, geometry->top, geometry->right - geometry->left,
412 geometry->bottom - geometry->top,
413
414 r->x, r->y, r->width, r->height);
415
416 WINPR_ASSERT(presentation->surface);
417 presentation->surface->x =
418 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
419 presentation->surface->y =
420 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
421
422 return TRUE;
423}
424
425WINPR_ATTR_NODISCARD
426static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
427{
428 WINPR_ASSERT(geometry);
429
430 PresentationContext* presentation = (PresentationContext*)geometry->custom;
431 WINPR_ASSERT(presentation);
432
433 mappedGeometryUnref(presentation->geometry);
434 presentation->geometry = nullptr;
435 return TRUE;
436}
437
438WINPR_ATTR_NODISCARD
439static UINT video_PresentationRequest(VideoClientContext* video,
440 const TSMM_PRESENTATION_REQUEST* req)
441{
442 UINT ret = CHANNEL_RC_OK;
443
444 WINPR_ASSERT(video);
445 WINPR_ASSERT(req);
446
447 VideoClientContextPriv* priv = video->priv;
448 WINPR_ASSERT(priv);
449
450 EnterCriticalSection(&priv->framesLock);
451 if (req->Command == TSMM_START_PRESENTATION)
452 {
453 MAPPED_GEOMETRY* geom = nullptr;
454 TSMM_PRESENTATION_RESPONSE resp = WINPR_C_ARRAY_INIT;
455
456 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
457 {
458 WLog_ERR(TAG, "not a H264 video, ignoring request");
459 goto fail;
460 }
461
462 if (priv->currentPresentation)
463 {
464 if (priv->currentPresentation->PresentationId == req->PresentationId)
465 {
466 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
467 req->PresentationId);
468 goto fail;
469 }
470
471 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
472 PresentationContext_unref(&priv->currentPresentation);
473 }
474
475 if (!priv->geometry)
476 {
477 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
478 goto fail;
479 }
480
481 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
482 if (!geom)
483 {
484 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
485 goto fail;
486 }
487
488 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
489 priv->currentPresentation = PresentationContext_new(
490 video, req->PresentationId,
491 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
492 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
493 req->SourceHeight);
494 if (!priv->currentPresentation)
495 {
496 WLog_ERR(TAG, "unable to create presentation video");
497 ret = CHANNEL_RC_NO_MEMORY;
498 goto fail;
499 }
500
501 mappedGeometryRef(geom);
502 priv->currentPresentation->geometry = geom;
503
504 priv->currentPresentation->video = video;
505 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
506 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
507
508 geom->custom = priv->currentPresentation;
509 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
510 geom->MappedGeometryClear = video_onMappedGeometryClear;
511
512 /* send back response */
513 resp.PresentationId = req->PresentationId;
514 ret = video_control_send_presentation_response(video, &resp);
515 }
516 else if (req->Command == TSMM_STOP_PRESENTATION)
517 {
518 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
519 if (!priv->currentPresentation)
520 {
521 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
522 goto fail;
523 }
524
525 priv->droppedFrames = 0;
526 priv->publishedFrames = 0;
527 PresentationContext_unref(&priv->currentPresentation);
528 }
529
530fail:
531 LeaveCriticalSection(&priv->framesLock);
532 return ret;
533}
534
535WINPR_ATTR_NODISCARD
536static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
537{
538 TSMM_PRESENTATION_REQUEST req = WINPR_C_ARRAY_INIT;
539
540 WINPR_ASSERT(context);
541 WINPR_ASSERT(s);
542
543 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
544 return ERROR_INVALID_DATA;
545
546 Stream_Read_UINT8(s, req.PresentationId);
547 Stream_Read_UINT8(s, req.Version);
548 Stream_Read_UINT8(s, req.Command);
549 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
550
551 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
552 Stream_Seek_UINT16(s); /* reserved */
553
554 Stream_Read_UINT32(s, req.SourceWidth);
555 Stream_Read_UINT32(s, req.SourceHeight);
556 Stream_Read_UINT32(s, req.ScaledWidth);
557 Stream_Read_UINT32(s, req.ScaledHeight);
558 Stream_Read_UINT64(s, req.hnsTimestampOffset);
559 Stream_Read_UINT64(s, req.GeometryMappingId);
560 Stream_Read(s, req.VideoSubtypeId, 16);
561
562 Stream_Read_UINT32(s, req.cbExtra);
563
564 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
565 return ERROR_INVALID_DATA;
566
567 req.pExtraData = Stream_Pointer(s);
568
569 WLog_DBG(TAG,
570 "presentationReq: id:%" PRIu8 " version:%" PRIu8
571 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
572 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
573 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
574 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
575 req.GeometryMappingId);
576
577 return video_PresentationRequest(context, &req);
578}
579
585WINPR_ATTR_NODISCARD
586static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
587{
588 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
589 UINT ret = CHANNEL_RC_OK;
590 UINT32 cbSize = 0;
591 UINT32 packetType = 0;
592
593 WINPR_ASSERT(callback);
594 WINPR_ASSERT(s);
595
596 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
597 WINPR_ASSERT(video);
598
599 VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
600 WINPR_ASSERT(context);
601
602 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
603 return ERROR_INVALID_DATA;
604
605 Stream_Read_UINT32(s, cbSize);
606 if (cbSize < 8)
607 {
608 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
609 return ERROR_INVALID_DATA;
610 }
611 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
612 return ERROR_INVALID_DATA;
613
614 Stream_Read_UINT32(s, packetType);
615 switch (packetType)
616 {
617 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
618 ret = video_read_tsmm_presentation_req(context, s);
619 break;
620 default:
621 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
622 ret = ERROR_UNSUPPORTED_TYPE;
623 break;
624 }
625
626 return ret;
627}
628
629static UINT video_control_send_client_notification(VideoClientContext* context,
630 const TSMM_CLIENT_NOTIFICATION* notif)
631{
632 BYTE buf[100] = WINPR_C_ARRAY_INIT;
633
634 WINPR_ASSERT(context);
635 WINPR_ASSERT(notif);
636
637 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
638 WINPR_ASSERT(video);
639
640 wStream* s = Stream_New(buf, 32);
641 if (!s)
642 return CHANNEL_RC_NO_MEMORY;
643
644 UINT32 cbSize = 16;
645 Stream_Seek_UINT32(s); /* cbSize */
646 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
647 Stream_Write_UINT8(s, notif->PresentationId);
648 Stream_Write_UINT8(s, notif->NotificationType);
649 Stream_Zero(s, 2);
650 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
651 {
652 Stream_Write_UINT32(s, 16); /* cbData */
653
654 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
655 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
656 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
657 Stream_Zero(s, 4ULL * 2ULL);
658
659 cbSize += 4UL * 4UL;
660 }
661 else
662 {
663 Stream_Write_UINT32(s, 0); /* cbData */
664 }
665
666 Stream_SealLength(s);
667 Stream_ResetPosition(s);
668 Stream_Write_UINT32(s, cbSize);
669 Stream_Free(s, FALSE);
670
671 return video_channel_write(video, buf, cbSize);
672}
673
674static void video_timer(VideoClientContext* video, UINT64 now)
675{
676 VideoFrame* frame = nullptr;
677
678 WINPR_ASSERT(video);
679
680 VideoClientContextPriv* priv = video->priv;
681 WINPR_ASSERT(priv);
682
683 EnterCriticalSection(&priv->framesLock);
684 PresentationContext* presentation = video->priv->currentPresentation;
685 do
686 {
687 const VideoFrame* peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
688 if (!peekFrame)
689 break;
690
691 if (peekFrame->publishTime > now)
692 break;
693
694 if (frame)
695 {
696 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
697 priv->droppedFrames++;
698 VideoFrame_free(priv, frame);
699 }
700 frame = Queue_Dequeue(priv->frames);
701 } while (1);
702
703 if (frame)
704 {
705 if (presentation && (presentation->PresentationId == frame->PresentationId))
706 {
707 VideoSurface* surface = presentation->surface;
708 const size_t frameSize = 1ull * frame->scanline * frame->h;
709 const size_t surfaceSize = 1ull * surface->scanline * surface->alignedHeight;
710
711 /* the presentation id is reused by the server across presentations of different
712 * sizes, so a frame queued for a previous, larger presentation can outlive it in
713 * the queue. copying it would write past the smaller surface->data buffer. */
714 if (frameSize > surfaceSize)
715 WLog_WARN(TAG, "dropping stale frame of %" PRIuz " bytes, surface holds %" PRIuz,
716 frameSize, surfaceSize);
717 else
718 {
719 priv->publishedFrames++;
720 memcpy(surface->data, frame->surfaceData, frameSize);
721
722 WINPR_ASSERT(video->showSurface);
723 if (!video->showSurface(video, surface, presentation->ScaledWidth,
724 presentation->ScaledHeight))
725 WLog_WARN(TAG, "showSurface failed");
726 }
727 }
728 VideoFrame_free(priv, frame);
729 }
730
731 if (priv->nextFeedbackTime < now)
732 {
733 /* we can compute some feedback only if we have some published frames and
734 * a current presentation
735 */
736 if (priv->publishedFrames && priv->currentPresentation)
737 {
738 UINT32 computedRate = 0;
739
740 if (!PresentationContext_ref(priv->currentPresentation))
741 WLog_WARN(TAG, "PresentationContext_ref(priv->currentPresentation) failed");
742
743 if (priv->droppedFrames)
744 {
750 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
751 computedRate = 24;
752 else
753 {
754 computedRate = priv->lastSentRate - 2;
755 if (!computedRate)
756 computedRate = 2;
757 }
758 }
759 else
760 {
765 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
766 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
767 else
768 {
769 computedRate = priv->lastSentRate + 2;
770 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
771 computedRate = XF_VIDEO_UNLIMITED_RATE;
772 }
773 }
774
775 if (computedRate != priv->lastSentRate)
776 {
777 TSMM_CLIENT_NOTIFICATION notif = WINPR_C_ARRAY_INIT;
778
779 WINPR_ASSERT(priv->currentPresentation);
780 notif.PresentationId = priv->currentPresentation->PresentationId;
781 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
782 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
783 {
784 notif.FramerateOverride.Flags = 0x01;
785 notif.FramerateOverride.DesiredFrameRate = 0x00;
786 }
787 else
788 {
789 notif.FramerateOverride.Flags = 0x02;
790 notif.FramerateOverride.DesiredFrameRate = computedRate;
791 }
792
793 video_control_send_client_notification(video, &notif);
794 priv->lastSentRate = computedRate;
795
796 WLog_VRB(TAG,
797 "server notified with rate %" PRIu32 " published=%" PRIu32
798 " dropped=%" PRIu32,
799 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
800 }
801
802 PresentationContext_unref(&priv->currentPresentation);
803 }
804
805 priv->droppedFrames = 0;
806 priv->publishedFrames = 0;
807 priv->nextFeedbackTime = now + 1000;
808 }
809 LeaveCriticalSection(&priv->framesLock);
810}
811
812WINPR_ATTR_NODISCARD
813static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
814{
815 int status = 0;
816 UINT res = CHANNEL_RC_OK;
817
818 WINPR_ASSERT(context);
819 WINPR_ASSERT(data);
820
821 VideoClientContextPriv* priv = context->priv;
822 WINPR_ASSERT(priv);
823
824 PresentationContext* presentation = priv->currentPresentation;
825 if (!presentation)
826 {
827 WLog_ERR(TAG, "no current presentation");
828 return CHANNEL_RC_OK;
829 }
830
831 if (!PresentationContext_ref(presentation))
832 return ERROR_INTERNAL_ERROR;
833
834 EnterCriticalSection(&priv->framesLock);
835 if (presentation->PresentationId != data->PresentationId)
836 {
837 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
838 presentation->PresentationId, data->PresentationId);
839 goto out;
840 }
841
842 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
843 {
844 WLog_ERR(TAG, "unable to expand the current packet");
845 res = CHANNEL_RC_NO_MEMORY;
846 goto out;
847 }
848
849 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
850
851 if (data->CurrentPacketIndex == data->PacketsInSample)
852 {
853 VideoSurface* surface = presentation->surface;
854 H264_CONTEXT* h264 = presentation->h264;
855 const UINT64 startTime = winpr_GetTickCount64NS();
856 MAPPED_GEOMETRY* geom = presentation->geometry;
857
858 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
859 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
860 Stream_SealLength(presentation->currentSample);
861 Stream_ResetPosition(presentation->currentSample);
862
863 if (data->SampleNumber == 1)
864 {
865 presentation->lastPublishTime = startTime;
866 }
867
868 presentation->lastPublishTime += 100ull * data->hnsDuration;
869 const size_t len = Stream_Length(presentation->currentSample);
870 if (len > UINT32_MAX)
871 goto out;
872
873 BOOL enqueueResult = 0;
874 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
875 if (!frame)
876 {
877 WLog_ERR(TAG, "unable to create frame");
878 res = CHANNEL_RC_NO_MEMORY;
879 goto out;
880 }
881
882 status = avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
883 frame->surfaceData, surface->format, surface->scanline,
884 surface->alignedWidth, surface->alignedHeight, &rect, 1);
885 if (status < 0)
886 {
887 VideoFrame_free(priv, frame);
888 goto out;
889 }
890
891 enqueueResult = Queue_Enqueue(priv->frames, frame);
892
893 if (!enqueueResult)
894 {
895 WLog_ERR(TAG, "unable to enqueue frame");
896 VideoFrame_free(priv, frame);
897 res = CHANNEL_RC_NO_MEMORY;
898 goto out;
899 }
900
901 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
902 WLog_DBG(TAG, "scheduling frame in %" PRIu64 " ns", (frame->publishTime - startTime));
903 }
904
905out:
906 PresentationContext_unref(&priv->currentPresentation);
907 LeaveCriticalSection(&priv->framesLock);
908
909 return res;
910}
911
912WINPR_ATTR_NODISCARD
913static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
914{
915 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
916 UINT32 cbSize = 0;
917 UINT32 packetType = 0;
918 TSMM_VIDEO_DATA data;
919
920 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
921 WINPR_ASSERT(video);
922
923 VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
924
925 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
926 return ERROR_INVALID_DATA;
927
928 Stream_Read_UINT32(s, cbSize);
929 if (cbSize < 8)
930 {
931 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
932 return ERROR_INVALID_DATA;
933 }
934
935 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
936 return ERROR_INVALID_DATA;
937
938 Stream_Read_UINT32(s, packetType);
939 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
940 {
941 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
942 return ERROR_INVALID_DATA;
943 }
944
945 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
946 return ERROR_INVALID_DATA;
947
948 Stream_Read_UINT8(s, data.PresentationId);
949 Stream_Read_UINT8(s, data.Version);
950 Stream_Read_UINT8(s, data.Flags);
951 Stream_Seek_UINT8(s); /* reserved */
952 Stream_Read_UINT64(s, data.hnsTimestamp);
953 Stream_Read_UINT64(s, data.hnsDuration);
954 Stream_Read_UINT16(s, data.CurrentPacketIndex);
955 Stream_Read_UINT16(s, data.PacketsInSample);
956 Stream_Read_UINT32(s, data.SampleNumber);
957 Stream_Read_UINT32(s, data.cbSample);
958 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
959 return ERROR_INVALID_DATA;
960 data.pSample = Stream_Pointer(s);
961
962 /*
963 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
964 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
965 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
966 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
967 data.cbSample);
968 */
969
970 return video_VideoData(context, &data);
971}
972
978WINPR_ATTR_NODISCARD
979static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
980{
981 if (pChannelCallback)
982 {
983 GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
984 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
985 if (video && video->control_callback)
986 {
987 video->control_callback->channel_callback = nullptr;
988 }
989 }
990 free(pChannelCallback);
991 return CHANNEL_RC_OK;
992}
993
994WINPR_ATTR_NODISCARD
995static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
996{
997 if (pChannelCallback)
998 {
999 GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
1000 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
1001 if (video && video->data_callback)
1002 {
1003 video->data_callback->channel_callback = nullptr;
1004 }
1005 }
1006 free(pChannelCallback);
1007 return CHANNEL_RC_OK;
1008}
1009
1015// NOLINTBEGIN(readability-non-const-parameter)
1016WINPR_ATTR_NODISCARD
1017static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1018 IWTSVirtualChannel* channel,
1019 WINPR_ATTR_UNUSED BYTE* Data,
1020 WINPR_ATTR_UNUSED BOOL* pbAccept,
1021 IWTSVirtualChannelCallback** ppCallback)
1022// NOLINTEND(readability-non-const-parameter)
1023{
1024 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1025
1026 GENERIC_CHANNEL_CALLBACK* callback =
1027 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1028 if (!callback)
1029 {
1030 WLog_ERR(TAG, "calloc failed!");
1031 return CHANNEL_RC_NO_MEMORY;
1032 }
1033
1034 callback->iface.OnDataReceived = video_control_on_data_received;
1035 callback->iface.OnClose = video_control_on_close;
1036 callback->plugin = listener_callback->plugin;
1037 callback->channel_mgr = listener_callback->channel_mgr;
1038 callback->channel = channel;
1039 listener_callback->channel_callback = callback;
1040
1041 *ppCallback = &callback->iface;
1042
1043 return CHANNEL_RC_OK;
1044}
1045
1046// NOLINTBEGIN(readability-non-const-parameter)
1047WINPR_ATTR_NODISCARD
1048static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1049 IWTSVirtualChannel* pChannel,
1050 WINPR_ATTR_UNUSED BYTE* Data,
1051 WINPR_ATTR_UNUSED BOOL* pbAccept,
1052 IWTSVirtualChannelCallback** ppCallback)
1053// NOLINTEND(readability-non-const-parameter)
1054{
1055 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1056
1057 GENERIC_CHANNEL_CALLBACK* callback =
1058 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1059 if (!callback)
1060 {
1061 WLog_ERR(TAG, "calloc failed!");
1062 return CHANNEL_RC_NO_MEMORY;
1063 }
1064
1065 callback->iface.OnDataReceived = video_data_on_data_received;
1066 callback->iface.OnClose = video_data_on_close;
1067 callback->plugin = listener_callback->plugin;
1068 callback->channel_mgr = listener_callback->channel_mgr;
1069 callback->channel = pChannel;
1070 listener_callback->channel_callback = callback;
1071
1072 *ppCallback = &callback->iface;
1073
1074 return CHANNEL_RC_OK;
1075}
1076
1077WINPR_ATTR_NODISCARD
1078static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
1079 WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
1080 uint64_t interval)
1081{
1082 VideoClientContext* video = userdata;
1083 if (!video)
1084 return 0;
1085 if (!video->timer)
1086 return 0;
1087
1088 video->timer(video, timestamp);
1089
1090 return interval;
1091}
1092
1098WINPR_ATTR_NODISCARD
1099static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1100{
1101 UINT status = 0;
1102 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1103
1104 if (video->initialized)
1105 {
1106 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1107 return ERROR_INVALID_DATA;
1108 }
1109
1110 {
1111 GENERIC_LISTENER_CALLBACK* callback =
1112 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1113 if (!callback)
1114 {
1115 WLog_ERR(TAG, "calloc for control callback failed!");
1116 return CHANNEL_RC_NO_MEMORY;
1117 }
1118
1119 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1120 callback->plugin = plugin;
1121 callback->channel_mgr = channelMgr;
1122
1123 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1124 &callback->iface, &(video->controlListener));
1125 video->control_callback = callback;
1126 if (status != CHANNEL_RC_OK)
1127 return status;
1128 }
1129 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1130
1131 {
1132 GENERIC_LISTENER_CALLBACK* callback =
1133 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1134 if (!callback)
1135 {
1136 WLog_ERR(TAG, "calloc for data callback failed!");
1137 return CHANNEL_RC_NO_MEMORY;
1138 }
1139
1140 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1141 callback->plugin = plugin;
1142 callback->channel_mgr = channelMgr;
1143
1144 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1145 &callback->iface, &(video->dataListener));
1146 video->data_callback = callback;
1147 if (status == CHANNEL_RC_OK)
1148 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1149 }
1150
1151 if (status == CHANNEL_RC_OK)
1152 video->context->priv->timerID =
1153 freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
1154 video->initialized = video->context->priv->timerID != 0;
1155 if (!video->initialized)
1156 status = ERROR_INTERNAL_ERROR;
1157 return status;
1158}
1159
1165WINPR_ATTR_NODISCARD
1166static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1167{
1168 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1169 if (!video)
1170 return CHANNEL_RC_INVALID_INSTANCE;
1171
1172 if (video->context && video->context->priv)
1173 freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);
1174
1175 if (video->control_callback)
1176 {
1177 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1178 if (mgr)
1179 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1180 }
1181 if (video->data_callback)
1182 {
1183 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1184 if (mgr)
1185 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1186 }
1187
1188 if (video->context)
1189 VideoClientContextPriv_free(video->context->priv);
1190
1191 free(video->control_callback);
1192 free(video->data_callback);
1193 free(video->wtsPlugin.pInterface);
1194 free(pPlugin);
1195 return CHANNEL_RC_OK;
1196}
1197
1206WINPR_ATTR_NODISCARD
1207FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1208{
1209 UINT error = ERROR_INTERNAL_ERROR;
1210
1211 VIDEO_PLUGIN* videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1212 if (!videoPlugin)
1213 {
1214 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1215 if (!videoPlugin)
1216 {
1217 WLog_ERR(TAG, "calloc failed!");
1218 return CHANNEL_RC_NO_MEMORY;
1219 }
1220
1221 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1222 videoPlugin->wtsPlugin.Connected = nullptr;
1223 videoPlugin->wtsPlugin.Disconnected = nullptr;
1224 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1225
1226 VideoClientContext* videoContext =
1227 (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1228 if (!videoContext)
1229 {
1230 WLog_ERR(TAG, "calloc failed!");
1231 free(videoPlugin);
1232 return CHANNEL_RC_NO_MEMORY;
1233 }
1234
1235 VideoClientContextPriv* priv = VideoClientContextPriv_new(videoContext);
1236 if (!priv)
1237 {
1238 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1239 free(videoContext);
1240 free(videoPlugin);
1241 return CHANNEL_RC_NO_MEMORY;
1242 }
1243
1244 videoContext->handle = (void*)videoPlugin;
1245 videoContext->priv = priv;
1246 videoContext->timer = video_timer;
1247 videoContext->setGeometry = video_client_context_set_geometry;
1248
1249 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1250 videoPlugin->context = videoContext;
1251 videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
1252 if (videoPlugin->rdpcontext)
1253 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1254 }
1255 else
1256 {
1257 WLog_ERR(TAG, "could not get video Plugin.");
1258 return CHANNEL_RC_BAD_CHANNEL;
1259 }
1260
1261 return error;
1262}
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:160
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36