FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/addin.h>
38#include <freerdp/primitives.h>
39#include <freerdp/client/channels.h>
40#include <freerdp/client/geometry.h>
41#include <freerdp/client/video.h>
42#include <freerdp/channels/log.h>
43#include <freerdp/codec/h264.h>
44#include <freerdp/codec/yuv.h>
45#include <freerdp/timer.h>
46
47#define TAG CHANNELS_TAG("video")
48
49#include "video_main.h"
50
51typedef struct
52{
53 IWTSPlugin wtsPlugin;
54
55 IWTSListener* controlListener;
56 IWTSListener* dataListener;
57 GENERIC_LISTENER_CALLBACK* control_callback;
58 GENERIC_LISTENER_CALLBACK* data_callback;
59
60 VideoClientContext* context;
61 BOOL initialized;
62 rdpContext* rdpcontext;
63} VIDEO_PLUGIN;
64
65#define XF_VIDEO_UNLIMITED_RATE 31
66
67static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
68 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
69
70typedef struct
71{
72 VideoClientContext* video;
73 BYTE PresentationId;
74 UINT32 ScaledWidth, ScaledHeight;
75 MAPPED_GEOMETRY* geometry;
76
77 UINT64 startTimeStamp;
78 UINT64 publishOffset;
79 H264_CONTEXT* h264;
80 wStream* currentSample;
81 UINT64 lastPublishTime, nextPublishTime;
82 volatile LONG refCounter;
83 VideoSurface* surface;
84} PresentationContext;
85
86typedef struct
87{
88 UINT64 publishTime;
89 UINT64 hnsDuration;
90 MAPPED_GEOMETRY* geometry;
91 UINT32 w, h;
92 UINT32 scanline;
93 BYTE* surfaceData;
94 PresentationContext* presentation;
95} VideoFrame;
96
98struct s_VideoClientContextPriv
99{
100 VideoClientContext* video;
101 GeometryClientContext* geometry;
102 wQueue* frames;
103 CRITICAL_SECTION framesLock;
104 wBufferPool* surfacePool;
105 UINT32 publishedFrames;
106 UINT32 droppedFrames;
107 UINT32 lastSentRate;
108 UINT64 nextFeedbackTime;
109 PresentationContext* currentPresentation;
110 FreeRDP_TimerID timerID;
111};
112
113static void PresentationContext_unref(PresentationContext** presentation);
114static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
115
116static const char* video_command_name(BYTE cmd)
117{
118 switch (cmd)
119 {
120 case TSMM_START_PRESENTATION:
121 return "start";
122 case TSMM_STOP_PRESENTATION:
123 return "stop";
124 default:
125 return "<unknown>";
126 }
127}
128
129static void video_client_context_set_geometry(VideoClientContext* video,
130 GeometryClientContext* geometry)
131{
132 WINPR_ASSERT(video);
133 WINPR_ASSERT(video->priv);
134 video->priv->geometry = geometry;
135}
136
137static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
138{
139 VideoClientContextPriv* ret = NULL;
140
141 WINPR_ASSERT(video);
142 ret = calloc(1, sizeof(*ret));
143 if (!ret)
144 return NULL;
145
146 ret->frames = Queue_New(TRUE, 10, 2);
147 if (!ret->frames)
148 {
149 WLog_ERR(TAG, "unable to allocate frames queue");
150 goto fail;
151 }
152
153 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
154 if (!ret->surfacePool)
155 {
156 WLog_ERR(TAG, "unable to create surface pool");
157 goto fail;
158 }
159
160 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
161 {
162 WLog_ERR(TAG, "unable to initialize frames lock");
163 goto fail;
164 }
165
166 ret->video = video;
167
168 /* don't set to unlimited so that we have the chance to send a feedback in
169 * the first second (for servers that want feedback directly)
170 */
171 ret->lastSentRate = 30;
172 return ret;
173
174fail:
175 VideoClientContextPriv_free(ret);
176 return NULL;
177}
178
179static BOOL PresentationContext_ref(PresentationContext* presentation)
180{
181 WINPR_ASSERT(presentation);
182
183 InterlockedIncrement(&presentation->refCounter);
184 return TRUE;
185}
186
187static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
188 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
189{
190 size_t s = 4ULL * width * height;
191 PresentationContext* ret = NULL;
192
193 WINPR_ASSERT(video);
194
195 if (s > INT32_MAX)
196 return NULL;
197
198 ret = calloc(1, sizeof(*ret));
199 if (!ret)
200 return NULL;
201
202 ret->video = video;
203 ret->PresentationId = PresentationId;
204
205 ret->h264 = h264_context_new(FALSE);
206 if (!ret->h264)
207 {
208 WLog_ERR(TAG, "unable to create a h264 context");
209 goto fail;
210 }
211 if (!h264_context_reset(ret->h264, width, height))
212 goto fail;
213
214 ret->currentSample = Stream_New(NULL, 4096);
215 if (!ret->currentSample)
216 {
217 WLog_ERR(TAG, "unable to create current packet stream");
218 goto fail;
219 }
220
221 ret->surface = video->createSurface(video, x, y, width, height);
222 if (!ret->surface)
223 {
224 WLog_ERR(TAG, "unable to create surface");
225 goto fail;
226 }
227
228 if (!PresentationContext_ref(ret))
229 goto fail;
230
231 return ret;
232
233fail:
234 PresentationContext_unref(&ret);
235 return NULL;
236}
237
238static void PresentationContext_unref(PresentationContext** ppresentation)
239{
240 PresentationContext* presentation = NULL;
241 MAPPED_GEOMETRY* geometry = NULL;
242
243 WINPR_ASSERT(ppresentation);
244
245 presentation = *ppresentation;
246 if (!presentation)
247 return;
248
249 if (InterlockedDecrement(&presentation->refCounter) > 0)
250 return;
251
252 geometry = presentation->geometry;
253 if (geometry)
254 {
255 geometry->MappedGeometryUpdate = NULL;
256 geometry->MappedGeometryClear = NULL;
257 geometry->custom = NULL;
258 mappedGeometryUnref(geometry);
259 }
260
261 h264_context_free(presentation->h264);
262 Stream_Free(presentation->currentSample, TRUE);
263 presentation->video->deleteSurface(presentation->video, presentation->surface);
264 free(presentation);
265 *ppresentation = NULL;
266}
267
268static void VideoFrame_free(VideoFrame** pframe)
269{
270 VideoFrame* frame = NULL;
271
272 WINPR_ASSERT(pframe);
273 frame = *pframe;
274 if (!frame)
275 return;
276
277 mappedGeometryUnref(frame->geometry);
278
279 WINPR_ASSERT(frame->presentation);
280 WINPR_ASSERT(frame->presentation->video);
281 WINPR_ASSERT(frame->presentation->video->priv);
282 BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
283 PresentationContext_unref(&frame->presentation);
284 free(frame);
285 *pframe = NULL;
286}
287
288static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
289 MAPPED_GEOMETRY* geom)
290{
291 VideoFrame* frame = NULL;
292 const VideoSurface* surface = NULL;
293
294 WINPR_ASSERT(priv);
295 WINPR_ASSERT(presentation);
296 WINPR_ASSERT(geom);
297
298 surface = presentation->surface;
299 WINPR_ASSERT(surface);
300
301 frame = calloc(1, sizeof(VideoFrame));
302 if (!frame)
303 goto fail;
304
305 mappedGeometryRef(geom);
306
307 frame->publishTime = presentation->lastPublishTime;
308 frame->geometry = geom;
309 frame->w = surface->alignedWidth;
310 frame->h = surface->alignedHeight;
311 frame->scanline = surface->scanline;
312
313 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
314 if (!frame->surfaceData)
315 goto fail;
316
317 frame->presentation = presentation;
318 if (!PresentationContext_ref(frame->presentation))
319 goto fail;
320
321 return frame;
322
323fail:
324 VideoFrame_free(&frame);
325 return NULL;
326}
327
328void VideoClientContextPriv_free(VideoClientContextPriv* priv)
329{
330 if (!priv)
331 return;
332
333 EnterCriticalSection(&priv->framesLock);
334
335 if (priv->frames)
336 {
337 while (Queue_Count(priv->frames))
338 {
339 VideoFrame* frame = Queue_Dequeue(priv->frames);
340 if (frame)
341 VideoFrame_free(&frame);
342 }
343 }
344
345 Queue_Free(priv->frames);
346 LeaveCriticalSection(&priv->framesLock);
347
348 DeleteCriticalSection(&priv->framesLock);
349
350 if (priv->currentPresentation)
351 PresentationContext_unref(&priv->currentPresentation);
352
353 BufferPool_Free(priv->surfacePool);
354 free(priv);
355}
356
357static UINT video_control_send_presentation_response(VideoClientContext* context,
358 TSMM_PRESENTATION_RESPONSE* resp)
359{
360 BYTE buf[12] = { 0 };
361 wStream* s = NULL;
362 VIDEO_PLUGIN* video = NULL;
363 IWTSVirtualChannel* channel = NULL;
364 UINT ret = 0;
365
366 WINPR_ASSERT(context);
367 WINPR_ASSERT(resp);
368
369 video = (VIDEO_PLUGIN*)context->handle;
370 WINPR_ASSERT(video);
371
372 s = Stream_New(buf, 12);
373 if (!s)
374 return CHANNEL_RC_NO_MEMORY;
375
376 Stream_Write_UINT32(s, 12); /* cbSize */
377 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
378 Stream_Write_UINT8(s, resp->PresentationId);
379 Stream_Zero(s, 3);
380 Stream_SealLength(s);
381
382 channel = video->control_callback->channel_callback->channel;
383 ret = channel->Write(channel, 12, buf, NULL);
384 Stream_Free(s, FALSE);
385
386 return ret;
387}
388
389static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
390{
391 PresentationContext* presentation = NULL;
392 RDP_RECT* r = NULL;
393
394 WINPR_ASSERT(geometry);
395
396 presentation = (PresentationContext*)geometry->custom;
397 WINPR_ASSERT(presentation);
398
399 r = &geometry->geometry.boundingRect;
400 WLog_DBG(TAG,
401 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
402 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
403 "-%" PRId16 "x%" PRId16 ")",
404 geometry->topLevelLeft, geometry->topLevelTop,
405 geometry->topLevelRight - geometry->topLevelLeft,
406 geometry->topLevelBottom - geometry->topLevelTop,
407
408 geometry->left, geometry->top, geometry->right - geometry->left,
409 geometry->bottom - geometry->top,
410
411 r->x, r->y, r->width, r->height);
412
413 presentation->surface->x =
414 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
415 presentation->surface->y =
416 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
417
418 return TRUE;
419}
420
421static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
422{
423 PresentationContext* presentation = NULL;
424
425 WINPR_ASSERT(geometry);
426
427 presentation = (PresentationContext*)geometry->custom;
428 WINPR_ASSERT(presentation);
429
430 mappedGeometryUnref(presentation->geometry);
431 presentation->geometry = NULL;
432 return TRUE;
433}
434
435static UINT video_PresentationRequest(VideoClientContext* video,
436 const TSMM_PRESENTATION_REQUEST* req)
437{
438 UINT ret = CHANNEL_RC_OK;
439
440 WINPR_ASSERT(video);
441 WINPR_ASSERT(req);
442
443 VideoClientContextPriv* priv = video->priv;
444 WINPR_ASSERT(priv);
445
446 if (req->Command == TSMM_START_PRESENTATION)
447 {
448 MAPPED_GEOMETRY* geom = NULL;
449 TSMM_PRESENTATION_RESPONSE resp;
450
451 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
452 {
453 WLog_ERR(TAG, "not a H264 video, ignoring request");
454 return CHANNEL_RC_OK;
455 }
456
457 if (priv->currentPresentation)
458 {
459 if (priv->currentPresentation->PresentationId == req->PresentationId)
460 {
461 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
462 req->PresentationId);
463 return CHANNEL_RC_OK;
464 }
465
466 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
467 PresentationContext_unref(&priv->currentPresentation);
468 }
469
470 if (!priv->geometry)
471 {
472 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
473 return CHANNEL_RC_OK;
474 }
475
476 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
477 if (!geom)
478 {
479 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
480 return CHANNEL_RC_OK;
481 }
482
483 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
484 priv->currentPresentation = PresentationContext_new(
485 video, req->PresentationId,
486 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
487 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
488 req->SourceHeight);
489 if (!priv->currentPresentation)
490 {
491 WLog_ERR(TAG, "unable to create presentation video");
492 return CHANNEL_RC_NO_MEMORY;
493 }
494
495 mappedGeometryRef(geom);
496 priv->currentPresentation->geometry = geom;
497
498 priv->currentPresentation->video = video;
499 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
500 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
501
502 geom->custom = priv->currentPresentation;
503 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
504 geom->MappedGeometryClear = video_onMappedGeometryClear;
505
506 /* send back response */
507 resp.PresentationId = req->PresentationId;
508 ret = video_control_send_presentation_response(video, &resp);
509 }
510 else if (req->Command == TSMM_STOP_PRESENTATION)
511 {
512 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
513 if (!priv->currentPresentation)
514 {
515 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
516 return CHANNEL_RC_OK;
517 }
518
519 priv->droppedFrames = 0;
520 priv->publishedFrames = 0;
521 PresentationContext_unref(&priv->currentPresentation);
522 }
523
524 return ret;
525}
526
527static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
528{
529 TSMM_PRESENTATION_REQUEST req = { 0 };
530
531 WINPR_ASSERT(context);
532 WINPR_ASSERT(s);
533
534 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
535 return ERROR_INVALID_DATA;
536
537 Stream_Read_UINT8(s, req.PresentationId);
538 Stream_Read_UINT8(s, req.Version);
539 Stream_Read_UINT8(s, req.Command);
540 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
541
542 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
543 Stream_Seek_UINT16(s); /* reserved */
544
545 Stream_Read_UINT32(s, req.SourceWidth);
546 Stream_Read_UINT32(s, req.SourceHeight);
547 Stream_Read_UINT32(s, req.ScaledWidth);
548 Stream_Read_UINT32(s, req.ScaledHeight);
549 Stream_Read_UINT64(s, req.hnsTimestampOffset);
550 Stream_Read_UINT64(s, req.GeometryMappingId);
551 Stream_Read(s, req.VideoSubtypeId, 16);
552
553 Stream_Read_UINT32(s, req.cbExtra);
554
555 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
556 return ERROR_INVALID_DATA;
557
558 req.pExtraData = Stream_Pointer(s);
559
560 WLog_DBG(TAG,
561 "presentationReq: id:%" PRIu8 " version:%" PRIu8
562 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
563 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
564 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
565 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
566 req.GeometryMappingId);
567
568 return video_PresentationRequest(context, &req);
569}
570
576static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
577{
578 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
579 VIDEO_PLUGIN* video = NULL;
580 VideoClientContext* context = NULL;
581 UINT ret = CHANNEL_RC_OK;
582 UINT32 cbSize = 0;
583 UINT32 packetType = 0;
584
585 WINPR_ASSERT(callback);
586 WINPR_ASSERT(s);
587
588 video = (VIDEO_PLUGIN*)callback->plugin;
589 WINPR_ASSERT(video);
590
591 context = (VideoClientContext*)video->wtsPlugin.pInterface;
592 WINPR_ASSERT(context);
593
594 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
595 return ERROR_INVALID_DATA;
596
597 Stream_Read_UINT32(s, cbSize);
598 if (cbSize < 8)
599 {
600 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
601 return ERROR_INVALID_DATA;
602 }
603 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
604 return ERROR_INVALID_DATA;
605
606 Stream_Read_UINT32(s, packetType);
607 switch (packetType)
608 {
609 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
610 ret = video_read_tsmm_presentation_req(context, s);
611 break;
612 default:
613 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
614 ret = ERROR_UNSUPPORTED_TYPE;
615 break;
616 }
617
618 return ret;
619}
620
621static UINT video_control_send_client_notification(VideoClientContext* context,
622 const TSMM_CLIENT_NOTIFICATION* notif)
623{
624 BYTE buf[100];
625 wStream* s = NULL;
626 VIDEO_PLUGIN* video = NULL;
627 IWTSVirtualChannel* channel = NULL;
628 UINT ret = 0;
629 UINT32 cbSize = 0;
630
631 WINPR_ASSERT(context);
632 WINPR_ASSERT(notif);
633
634 video = (VIDEO_PLUGIN*)context->handle;
635 WINPR_ASSERT(video);
636
637 s = Stream_New(buf, 32);
638 if (!s)
639 return CHANNEL_RC_NO_MEMORY;
640
641 cbSize = 16;
642 Stream_Seek_UINT32(s); /* cbSize */
643 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
644 Stream_Write_UINT8(s, notif->PresentationId);
645 Stream_Write_UINT8(s, notif->NotificationType);
646 Stream_Zero(s, 2);
647 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
648 {
649 Stream_Write_UINT32(s, 16); /* cbData */
650
651 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
652 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
653 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
654 Stream_Zero(s, 4ULL * 2ULL);
655
656 cbSize += 4UL * 4UL;
657 }
658 else
659 {
660 Stream_Write_UINT32(s, 0); /* cbData */
661 }
662
663 Stream_SealLength(s);
664 Stream_SetPosition(s, 0);
665 Stream_Write_UINT32(s, cbSize);
666 Stream_Free(s, FALSE);
667
668 WINPR_ASSERT(video->control_callback);
669 WINPR_ASSERT(video->control_callback->channel_callback);
670
671 channel = video->control_callback->channel_callback->channel;
672 WINPR_ASSERT(channel);
673 WINPR_ASSERT(channel->Write);
674
675 ret = channel->Write(channel, cbSize, buf, NULL);
676
677 return ret;
678}
679
680static void video_timer(VideoClientContext* video, UINT64 now)
681{
682 PresentationContext* presentation = NULL;
683 VideoFrame* frame = NULL;
684
685 WINPR_ASSERT(video);
686
687 VideoClientContextPriv* priv = video->priv;
688 WINPR_ASSERT(priv);
689
690 EnterCriticalSection(&priv->framesLock);
691 do
692 {
693 VideoFrame* peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
694 if (!peekFrame)
695 break;
696
697 if (peekFrame->publishTime > now)
698 break;
699
700 if (frame)
701 {
702 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
703 priv->droppedFrames++;
704 VideoFrame_free(&frame);
705 }
706 frame = peekFrame;
707 Queue_Dequeue(priv->frames);
708 } while (1);
709 LeaveCriticalSection(&priv->framesLock);
710
711 if (frame)
712 {
713 presentation = frame->presentation;
714
715 priv->publishedFrames++;
716 memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
717
718 WINPR_ASSERT(video->showSurface);
719 video->showSurface(video, presentation->surface, presentation->ScaledWidth,
720 presentation->ScaledHeight);
721
722 VideoFrame_free(&frame);
723 }
724
725 if (priv->nextFeedbackTime < now)
726 {
727 /* we can compute some feedback only if we have some published frames and
728 * a current presentation
729 */
730 if (priv->publishedFrames && priv->currentPresentation)
731 {
732 UINT32 computedRate = 0;
733
734 PresentationContext_ref(priv->currentPresentation);
735
736 if (priv->droppedFrames)
737 {
743 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
744 computedRate = 24;
745 else
746 {
747 computedRate = priv->lastSentRate - 2;
748 if (!computedRate)
749 computedRate = 2;
750 }
751 }
752 else
753 {
758 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
759 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
760 else
761 {
762 computedRate = priv->lastSentRate + 2;
763 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
764 computedRate = XF_VIDEO_UNLIMITED_RATE;
765 }
766 }
767
768 if (computedRate != priv->lastSentRate)
769 {
770 TSMM_CLIENT_NOTIFICATION notif;
771
772 WINPR_ASSERT(priv->currentPresentation);
773 notif.PresentationId = priv->currentPresentation->PresentationId;
774 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
775 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
776 {
777 notif.FramerateOverride.Flags = 0x01;
778 notif.FramerateOverride.DesiredFrameRate = 0x00;
779 }
780 else
781 {
782 notif.FramerateOverride.Flags = 0x02;
783 notif.FramerateOverride.DesiredFrameRate = computedRate;
784 }
785
786 video_control_send_client_notification(video, &notif);
787 priv->lastSentRate = computedRate;
788
789 WLog_VRB(TAG,
790 "server notified with rate %" PRIu32 " published=%" PRIu32
791 " dropped=%" PRIu32,
792 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
793 }
794
795 PresentationContext_unref(&priv->currentPresentation);
796 }
797
798 priv->droppedFrames = 0;
799 priv->publishedFrames = 0;
800 priv->nextFeedbackTime = now + 1000;
801 }
802}
803
804static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
805{
806 VideoClientContextPriv* priv = NULL;
807 PresentationContext* presentation = NULL;
808 int status = 0;
809
810 WINPR_ASSERT(context);
811 WINPR_ASSERT(data);
812
813 priv = context->priv;
814 WINPR_ASSERT(priv);
815
816 presentation = priv->currentPresentation;
817 if (!presentation)
818 {
819 WLog_ERR(TAG, "no current presentation");
820 return CHANNEL_RC_OK;
821 }
822
823 if (presentation->PresentationId != data->PresentationId)
824 {
825 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
826 presentation->PresentationId, data->PresentationId);
827 return CHANNEL_RC_OK;
828 }
829
830 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
831 {
832 WLog_ERR(TAG, "unable to expand the current packet");
833 return CHANNEL_RC_NO_MEMORY;
834 }
835
836 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
837
838 if (data->CurrentPacketIndex == data->PacketsInSample)
839 {
840 VideoSurface* surface = presentation->surface;
841 H264_CONTEXT* h264 = presentation->h264;
842 const UINT64 startTime = winpr_GetTickCount64NS();
843 MAPPED_GEOMETRY* geom = presentation->geometry;
844
845 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
846 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
847 Stream_SealLength(presentation->currentSample);
848 Stream_SetPosition(presentation->currentSample, 0);
849
850 const UINT64 timeAfterH264 = winpr_GetTickCount64NS();
851 if (data->SampleNumber == 1)
852 {
853 presentation->lastPublishTime = startTime;
854 }
855
856 presentation->lastPublishTime += 100ull * data->hnsDuration;
857 if (presentation->lastPublishTime <= (10000000ull + timeAfterH264))
858 {
859 int dropped = 0;
860
861 const size_t len = Stream_Length(presentation->currentSample);
862 if (len > UINT32_MAX)
863 return CHANNEL_RC_OK;
864
865 /* if the frame is to be published in less than 10 ms, let's consider it's now */
866 status =
867 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
868 surface->data, surface->format, surface->scanline,
869 surface->alignedWidth, surface->alignedHeight, &rect, 1);
870
871 if (status < 0)
872 return CHANNEL_RC_OK;
873
874 WINPR_ASSERT(context->showSurface);
875 context->showSurface(context, presentation->surface, presentation->ScaledWidth,
876 presentation->ScaledHeight);
877
878 priv->publishedFrames++;
879
880 /* cleanup previously scheduled frames */
881 EnterCriticalSection(&priv->framesLock);
882 while (Queue_Count(priv->frames) > 0)
883 {
884 VideoFrame* frame = Queue_Dequeue(priv->frames);
885 if (frame)
886 {
887 priv->droppedFrames++;
888 VideoFrame_free(&frame);
889 dropped++;
890 }
891 }
892 LeaveCriticalSection(&priv->framesLock);
893
894 if (dropped)
895 WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
896 }
897 else
898 {
899 const size_t len = Stream_Length(presentation->currentSample);
900 if (len > UINT32_MAX)
901 return CHANNEL_RC_OK;
902
903 BOOL enqueueResult = 0;
904 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
905 if (!frame)
906 {
907 WLog_ERR(TAG, "unable to create frame");
908 return CHANNEL_RC_NO_MEMORY;
909 }
910
911 status =
912 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
913 frame->surfaceData, surface->format, surface->scanline,
914 surface->alignedWidth, surface->alignedHeight, &rect, 1);
915 if (status < 0)
916 {
917 VideoFrame_free(&frame);
918 return CHANNEL_RC_OK;
919 }
920
921 EnterCriticalSection(&priv->framesLock);
922 enqueueResult = Queue_Enqueue(priv->frames, frame);
923 LeaveCriticalSection(&priv->framesLock);
924
925 if (!enqueueResult)
926 {
927 WLog_ERR(TAG, "unable to enqueue frame");
928 VideoFrame_free(&frame);
929 return CHANNEL_RC_NO_MEMORY;
930 }
931
932 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
933 WLog_DBG(TAG, "scheduling frame in %" PRIu64 " ms", (frame->publishTime - startTime));
934 }
935 }
936
937 return CHANNEL_RC_OK;
938}
939
940static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
941{
942 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
943 VIDEO_PLUGIN* video = NULL;
944 VideoClientContext* context = NULL;
945 UINT32 cbSize = 0;
946 UINT32 packetType = 0;
947 TSMM_VIDEO_DATA data;
948
949 video = (VIDEO_PLUGIN*)callback->plugin;
950 context = (VideoClientContext*)video->wtsPlugin.pInterface;
951
952 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
953 return ERROR_INVALID_DATA;
954
955 Stream_Read_UINT32(s, cbSize);
956 if (cbSize < 8)
957 {
958 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
959 return ERROR_INVALID_DATA;
960 }
961
962 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
963 return ERROR_INVALID_DATA;
964
965 Stream_Read_UINT32(s, packetType);
966 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
967 {
968 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
969 return ERROR_INVALID_DATA;
970 }
971
972 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
973 return ERROR_INVALID_DATA;
974
975 Stream_Read_UINT8(s, data.PresentationId);
976 Stream_Read_UINT8(s, data.Version);
977 Stream_Read_UINT8(s, data.Flags);
978 Stream_Seek_UINT8(s); /* reserved */
979 Stream_Read_UINT64(s, data.hnsTimestamp);
980 Stream_Read_UINT64(s, data.hnsDuration);
981 Stream_Read_UINT16(s, data.CurrentPacketIndex);
982 Stream_Read_UINT16(s, data.PacketsInSample);
983 Stream_Read_UINT32(s, data.SampleNumber);
984 Stream_Read_UINT32(s, data.cbSample);
985 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
986 return ERROR_INVALID_DATA;
987 data.pSample = Stream_Pointer(s);
988
989 /*
990 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
991 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
992 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
993 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
994 data.cbSample);
995 */
996
997 return video_VideoData(context, &data);
998}
999
1005static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1006{
1007 free(pChannelCallback);
1008 return CHANNEL_RC_OK;
1009}
1010
1011static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1012{
1013 free(pChannelCallback);
1014 return CHANNEL_RC_OK;
1015}
1016
1022// NOLINTBEGIN(readability-non-const-parameter)
1023static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1024 IWTSVirtualChannel* channel, BYTE* Data,
1025 BOOL* pbAccept,
1026 IWTSVirtualChannelCallback** ppCallback)
1027// NOLINTEND(readability-non-const-parameter)
1028{
1029 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1030
1031 WINPR_UNUSED(Data);
1032 WINPR_UNUSED(pbAccept);
1033
1034 GENERIC_CHANNEL_CALLBACK* callback =
1035 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1036 if (!callback)
1037 {
1038 WLog_ERR(TAG, "calloc failed!");
1039 return CHANNEL_RC_NO_MEMORY;
1040 }
1041
1042 callback->iface.OnDataReceived = video_control_on_data_received;
1043 callback->iface.OnClose = video_control_on_close;
1044 callback->plugin = listener_callback->plugin;
1045 callback->channel_mgr = listener_callback->channel_mgr;
1046 callback->channel = channel;
1047 listener_callback->channel_callback = callback;
1048
1049 *ppCallback = &callback->iface;
1050
1051 return CHANNEL_RC_OK;
1052}
1053
1054// NOLINTBEGIN(readability-non-const-parameter)
1055static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1056 IWTSVirtualChannel* pChannel, BYTE* Data,
1057 BOOL* pbAccept,
1058 IWTSVirtualChannelCallback** ppCallback)
1059// NOLINTEND(readability-non-const-parameter)
1060{
1061 GENERIC_CHANNEL_CALLBACK* callback = NULL;
1062 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1063
1064 WINPR_UNUSED(Data);
1065 WINPR_UNUSED(pbAccept);
1066
1067 callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1068 if (!callback)
1069 {
1070 WLog_ERR(TAG, "calloc failed!");
1071 return CHANNEL_RC_NO_MEMORY;
1072 }
1073
1074 callback->iface.OnDataReceived = video_data_on_data_received;
1075 callback->iface.OnClose = video_data_on_close;
1076 callback->plugin = listener_callback->plugin;
1077 callback->channel_mgr = listener_callback->channel_mgr;
1078 callback->channel = pChannel;
1079 listener_callback->channel_callback = callback;
1080
1081 *ppCallback = &callback->iface;
1082
1083 return CHANNEL_RC_OK;
1084}
1085
1086static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
1087 WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
1088 uint64_t interval)
1089{
1090 VideoClientContext* video = userdata;
1091 if (!video)
1092 return 0;
1093 if (!video->timer)
1094 return 0;
1095
1096 video->timer(video, timestamp);
1097
1098 return interval;
1099}
1100
1106static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1107{
1108 UINT status = 0;
1109 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1110 GENERIC_LISTENER_CALLBACK* callback = NULL;
1111
1112 if (video->initialized)
1113 {
1114 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1115 return ERROR_INVALID_DATA;
1116 }
1117 video->control_callback = callback =
1118 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1119 if (!callback)
1120 {
1121 WLog_ERR(TAG, "calloc for control callback failed!");
1122 return CHANNEL_RC_NO_MEMORY;
1123 }
1124
1125 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1126 callback->plugin = plugin;
1127 callback->channel_mgr = channelMgr;
1128
1129 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1130 &callback->iface, &(video->controlListener));
1131
1132 if (status != CHANNEL_RC_OK)
1133 return status;
1134 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1135
1136 video->data_callback = callback =
1137 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1138 if (!callback)
1139 {
1140 WLog_ERR(TAG, "calloc for data callback failed!");
1141 return CHANNEL_RC_NO_MEMORY;
1142 }
1143
1144 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1145 callback->plugin = plugin;
1146 callback->channel_mgr = channelMgr;
1147
1148 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1149 &callback->iface, &(video->dataListener));
1150
1151 if (status == CHANNEL_RC_OK)
1152 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1153
1154 if (status == CHANNEL_RC_OK)
1155 video->context->priv->timerID =
1156 freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
1157 video->initialized = video->context->priv->timerID != 0;
1158 if (!video->initialized)
1159 status = ERROR_INTERNAL_ERROR;
1160 return status;
1161}
1162
1168static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1169{
1170 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1171 if (!video)
1172 return CHANNEL_RC_INVALID_INSTANCE;
1173
1174 if (video->context && video->context->priv)
1175 freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);
1176
1177 if (video->control_callback)
1178 {
1179 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1180 if (mgr)
1181 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1182 }
1183 if (video->data_callback)
1184 {
1185 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1186 if (mgr)
1187 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1188 }
1189
1190 if (video->context)
1191 VideoClientContextPriv_free(video->context->priv);
1192
1193 free(video->control_callback);
1194 free(video->data_callback);
1195 free(video->wtsPlugin.pInterface);
1196 free(pPlugin);
1197 return CHANNEL_RC_OK;
1198}
1199
1208FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1209{
1210 UINT error = ERROR_INTERNAL_ERROR;
1211 VIDEO_PLUGIN* videoPlugin = NULL;
1212 VideoClientContext* videoContext = NULL;
1213 VideoClientContextPriv* priv = NULL;
1214
1215 videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1216 if (!videoPlugin)
1217 {
1218 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1219 if (!videoPlugin)
1220 {
1221 WLog_ERR(TAG, "calloc failed!");
1222 return CHANNEL_RC_NO_MEMORY;
1223 }
1224
1225 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1226 videoPlugin->wtsPlugin.Connected = NULL;
1227 videoPlugin->wtsPlugin.Disconnected = NULL;
1228 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1229
1230 videoContext = (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1231 if (!videoContext)
1232 {
1233 WLog_ERR(TAG, "calloc failed!");
1234 free(videoPlugin);
1235 return CHANNEL_RC_NO_MEMORY;
1236 }
1237
1238 priv = VideoClientContextPriv_new(videoContext);
1239 if (!priv)
1240 {
1241 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1242 free(videoContext);
1243 free(videoPlugin);
1244 return CHANNEL_RC_NO_MEMORY;
1245 }
1246
1247 videoContext->handle = (void*)videoPlugin;
1248 videoContext->priv = priv;
1249 videoContext->timer = video_timer;
1250 videoContext->setGeometry = video_client_context_set_geometry;
1251
1252 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1253 videoPlugin->context = videoContext;
1254 videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
1255 if (videoPlugin->rdpcontext)
1256 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1257 }
1258 else
1259 {
1260 WLog_ERR(TAG, "could not get video Plugin.");
1261 return CHANNEL_RC_BAD_CHANNEL;
1262 }
1263
1264 return error;
1265}
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:174
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36