FreeRDP
video_main.c
1 
20 #include <freerdp/config.h>
21 
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 
26 #include <winpr/crt.h>
27 #include <winpr/assert.h>
28 #include <winpr/synch.h>
29 #include <winpr/print.h>
30 #include <winpr/stream.h>
31 #include <winpr/cmdline.h>
32 #include <winpr/collections.h>
33 #include <winpr/interlocked.h>
34 #include <winpr/sysinfo.h>
35 
36 #include <freerdp/addin.h>
37 #include <freerdp/primitives.h>
38 #include <freerdp/client/channels.h>
39 #include <freerdp/client/geometry.h>
40 #include <freerdp/client/video.h>
41 #include <freerdp/channels/log.h>
42 #include <freerdp/codec/h264.h>
43 #include <freerdp/codec/yuv.h>
44 
45 #define TAG CHANNELS_TAG("video")
46 
47 #include "video_main.h"
48 
49 typedef struct
50 {
51  IWTSPlugin wtsPlugin;
52 
53  IWTSListener* controlListener;
54  IWTSListener* dataListener;
55  GENERIC_LISTENER_CALLBACK* control_callback;
56  GENERIC_LISTENER_CALLBACK* data_callback;
57 
58  VideoClientContext* context;
59  BOOL initialized;
60 } VIDEO_PLUGIN;
61 
62 #define XF_VIDEO_UNLIMITED_RATE 31
63 
64 static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
65  0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
66 
67 typedef struct
68 {
69  VideoClientContext* video;
70  BYTE PresentationId;
71  UINT32 ScaledWidth, ScaledHeight;
72  MAPPED_GEOMETRY* geometry;
73 
74  UINT64 startTimeStamp;
75  UINT64 publishOffset;
76  H264_CONTEXT* h264;
77  wStream* currentSample;
78  UINT64 lastPublishTime, nextPublishTime;
79  volatile LONG refCounter;
80  VideoSurface* surface;
81 } PresentationContext;
82 
83 typedef struct
84 {
85  UINT64 publishTime;
86  UINT64 hnsDuration;
87  MAPPED_GEOMETRY* geometry;
88  UINT32 w, h;
89  UINT32 scanline;
90  BYTE* surfaceData;
91  PresentationContext* presentation;
92 } VideoFrame;
93 
95 struct s_VideoClientContextPriv
96 {
97  VideoClientContext* video;
98  GeometryClientContext* geometry;
99  wQueue* frames;
100  CRITICAL_SECTION framesLock;
101  wBufferPool* surfacePool;
102  UINT32 publishedFrames;
103  UINT32 droppedFrames;
104  UINT32 lastSentRate;
105  UINT64 nextFeedbackTime;
106  PresentationContext* currentPresentation;
107 };
108 
109 static void PresentationContext_unref(PresentationContext** presentation);
110 static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
111 
112 static const char* video_command_name(BYTE cmd)
113 {
114  switch (cmd)
115  {
116  case TSMM_START_PRESENTATION:
117  return "start";
118  case TSMM_STOP_PRESENTATION:
119  return "stop";
120  default:
121  return "<unknown>";
122  }
123 }
124 
125 static void video_client_context_set_geometry(VideoClientContext* video,
126  GeometryClientContext* geometry)
127 {
128  WINPR_ASSERT(video);
129  WINPR_ASSERT(video->priv);
130  video->priv->geometry = geometry;
131 }
132 
133 static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
134 {
135  VideoClientContextPriv* ret = NULL;
136 
137  WINPR_ASSERT(video);
138  ret = calloc(1, sizeof(*ret));
139  if (!ret)
140  return NULL;
141 
142  ret->frames = Queue_New(TRUE, 10, 2);
143  if (!ret->frames)
144  {
145  WLog_ERR(TAG, "unable to allocate frames queue");
146  goto fail;
147  }
148 
149  ret->surfacePool = BufferPool_New(FALSE, 0, 16);
150  if (!ret->surfacePool)
151  {
152  WLog_ERR(TAG, "unable to create surface pool");
153  goto fail;
154  }
155 
156  if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
157  {
158  WLog_ERR(TAG, "unable to initialize frames lock");
159  goto fail;
160  }
161 
162  ret->video = video;
163 
164  /* don't set to unlimited so that we have the chance to send a feedback in
165  * the first second (for servers that want feedback directly)
166  */
167  ret->lastSentRate = 30;
168  return ret;
169 
170 fail:
171  VideoClientContextPriv_free(ret);
172  return NULL;
173 }
174 
175 static BOOL PresentationContext_ref(PresentationContext* presentation)
176 {
177  WINPR_ASSERT(presentation);
178 
179  InterlockedIncrement(&presentation->refCounter);
180  return TRUE;
181 }
182 
183 static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
184  UINT32 x, UINT32 y, UINT32 width, UINT32 height)
185 {
186  size_t s = 4ULL * width * height;
187  VideoClientContextPriv* priv = NULL;
188  PresentationContext* ret = NULL;
189 
190  WINPR_ASSERT(video);
191 
192  priv = video->priv;
193  WINPR_ASSERT(priv);
194 
195  if (s > INT32_MAX)
196  return NULL;
197 
198  ret = calloc(1, sizeof(*ret));
199  if (!ret)
200  return NULL;
201 
202  ret->video = video;
203  ret->PresentationId = PresentationId;
204 
205  ret->h264 = h264_context_new(FALSE);
206  if (!ret->h264)
207  {
208  WLog_ERR(TAG, "unable to create a h264 context");
209  goto fail;
210  }
211  if (!h264_context_reset(ret->h264, width, height))
212  goto fail;
213 
214  ret->currentSample = Stream_New(NULL, 4096);
215  if (!ret->currentSample)
216  {
217  WLog_ERR(TAG, "unable to create current packet stream");
218  goto fail;
219  }
220 
221  ret->surface = video->createSurface(video, x, y, width, height);
222  if (!ret->surface)
223  {
224  WLog_ERR(TAG, "unable to create surface");
225  goto fail;
226  }
227 
228  if (!PresentationContext_ref(ret))
229  goto fail;
230 
231  return ret;
232 
233 fail:
234  PresentationContext_unref(&ret);
235  return NULL;
236 }
237 
238 static void PresentationContext_unref(PresentationContext** ppresentation)
239 {
240  PresentationContext* presentation = NULL;
241  MAPPED_GEOMETRY* geometry = NULL;
242 
243  WINPR_ASSERT(ppresentation);
244 
245  presentation = *ppresentation;
246  if (!presentation)
247  return;
248 
249  if (InterlockedDecrement(&presentation->refCounter) > 0)
250  return;
251 
252  geometry = presentation->geometry;
253  if (geometry)
254  {
255  geometry->MappedGeometryUpdate = NULL;
256  geometry->MappedGeometryClear = NULL;
257  geometry->custom = NULL;
258  mappedGeometryUnref(geometry);
259  }
260 
261  h264_context_free(presentation->h264);
262  Stream_Free(presentation->currentSample, TRUE);
263  presentation->video->deleteSurface(presentation->video, presentation->surface);
264  free(presentation);
265  *ppresentation = NULL;
266 }
267 
268 static void VideoFrame_free(VideoFrame** pframe)
269 {
270  VideoFrame* frame = NULL;
271 
272  WINPR_ASSERT(pframe);
273  frame = *pframe;
274  if (!frame)
275  return;
276 
277  mappedGeometryUnref(frame->geometry);
278 
279  WINPR_ASSERT(frame->presentation);
280  WINPR_ASSERT(frame->presentation->video);
281  WINPR_ASSERT(frame->presentation->video->priv);
282  BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
283  PresentationContext_unref(&frame->presentation);
284  free(frame);
285  *pframe = NULL;
286 }
287 
288 static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
289  MAPPED_GEOMETRY* geom)
290 {
291  VideoFrame* frame = NULL;
292  const VideoSurface* surface = NULL;
293 
294  WINPR_ASSERT(priv);
295  WINPR_ASSERT(presentation);
296  WINPR_ASSERT(geom);
297 
298  surface = presentation->surface;
299  WINPR_ASSERT(surface);
300 
301  frame = calloc(1, sizeof(VideoFrame));
302  if (!frame)
303  goto fail;
304 
305  mappedGeometryRef(geom);
306 
307  frame->publishTime = presentation->lastPublishTime;
308  frame->geometry = geom;
309  frame->w = surface->alignedWidth;
310  frame->h = surface->alignedHeight;
311  frame->scanline = surface->scanline;
312 
313  frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
314  if (!frame->surfaceData)
315  goto fail;
316 
317  frame->presentation = presentation;
318  if (!PresentationContext_ref(frame->presentation))
319  goto fail;
320 
321  return frame;
322 
323 fail:
324  VideoFrame_free(&frame);
325  return NULL;
326 }
327 
328 void VideoClientContextPriv_free(VideoClientContextPriv* priv)
329 {
330  if (!priv)
331  return;
332 
333  EnterCriticalSection(&priv->framesLock);
334 
335  if (priv->frames)
336  {
337  while (Queue_Count(priv->frames))
338  {
339  VideoFrame* frame = Queue_Dequeue(priv->frames);
340  if (frame)
341  VideoFrame_free(&frame);
342  }
343  }
344 
345  Queue_Free(priv->frames);
346  LeaveCriticalSection(&priv->framesLock);
347 
348  DeleteCriticalSection(&priv->framesLock);
349 
350  if (priv->currentPresentation)
351  PresentationContext_unref(&priv->currentPresentation);
352 
353  BufferPool_Free(priv->surfacePool);
354  free(priv);
355 }
356 
357 static UINT video_control_send_presentation_response(VideoClientContext* context,
358  TSMM_PRESENTATION_RESPONSE* resp)
359 {
360  BYTE buf[12] = { 0 };
361  wStream* s = NULL;
362  VIDEO_PLUGIN* video = NULL;
363  IWTSVirtualChannel* channel = NULL;
364  UINT ret = 0;
365 
366  WINPR_ASSERT(context);
367  WINPR_ASSERT(resp);
368 
369  video = (VIDEO_PLUGIN*)context->handle;
370  WINPR_ASSERT(video);
371 
372  s = Stream_New(buf, 12);
373  if (!s)
374  return CHANNEL_RC_NO_MEMORY;
375 
376  Stream_Write_UINT32(s, 12); /* cbSize */
377  Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
378  Stream_Write_UINT8(s, resp->PresentationId);
379  Stream_Zero(s, 3);
380  Stream_SealLength(s);
381 
382  channel = video->control_callback->channel_callback->channel;
383  ret = channel->Write(channel, 12, buf, NULL);
384  Stream_Free(s, FALSE);
385 
386  return ret;
387 }
388 
389 static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
390 {
391  PresentationContext* presentation = NULL;
392  RDP_RECT* r = NULL;
393 
394  WINPR_ASSERT(geometry);
395 
396  presentation = (PresentationContext*)geometry->custom;
397  WINPR_ASSERT(presentation);
398 
399  r = &geometry->geometry.boundingRect;
400  WLog_DBG(TAG,
401  "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
402  ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
403  "-%" PRId16 "x%" PRId16 ")",
404  geometry->topLevelLeft, geometry->topLevelTop,
405  geometry->topLevelRight - geometry->topLevelLeft,
406  geometry->topLevelBottom - geometry->topLevelTop,
407 
408  geometry->left, geometry->top, geometry->right - geometry->left,
409  geometry->bottom - geometry->top,
410 
411  r->x, r->y, r->width, r->height);
412 
413  presentation->surface->x = geometry->topLevelLeft + geometry->left;
414  presentation->surface->y = geometry->topLevelTop + geometry->top;
415 
416  return TRUE;
417 }
418 
419 static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
420 {
421  PresentationContext* presentation = NULL;
422 
423  WINPR_ASSERT(geometry);
424 
425  presentation = (PresentationContext*)geometry->custom;
426  WINPR_ASSERT(presentation);
427 
428  mappedGeometryUnref(presentation->geometry);
429  presentation->geometry = NULL;
430  return TRUE;
431 }
432 
433 static UINT video_PresentationRequest(VideoClientContext* video,
434  const TSMM_PRESENTATION_REQUEST* req)
435 {
436  UINT ret = CHANNEL_RC_OK;
437 
438  WINPR_ASSERT(video);
439  WINPR_ASSERT(req);
440 
441  VideoClientContextPriv* priv = video->priv;
442  WINPR_ASSERT(priv);
443 
444  if (req->Command == TSMM_START_PRESENTATION)
445  {
446  MAPPED_GEOMETRY* geom = NULL;
447  TSMM_PRESENTATION_RESPONSE resp;
448 
449  if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
450  {
451  WLog_ERR(TAG, "not a H264 video, ignoring request");
452  return CHANNEL_RC_OK;
453  }
454 
455  if (priv->currentPresentation)
456  {
457  if (priv->currentPresentation->PresentationId == req->PresentationId)
458  {
459  WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
460  req->PresentationId);
461  return CHANNEL_RC_OK;
462  }
463 
464  WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
465  PresentationContext_unref(&priv->currentPresentation);
466  }
467 
468  if (!priv->geometry)
469  {
470  WLog_ERR(TAG, "geometry channel not ready, ignoring request");
471  return CHANNEL_RC_OK;
472  }
473 
474  geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
475  if (!geom)
476  {
477  WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
478  return CHANNEL_RC_OK;
479  }
480 
481  WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
482  priv->currentPresentation = PresentationContext_new(
483  video, req->PresentationId, geom->topLevelLeft + geom->left,
484  geom->topLevelTop + geom->top, req->SourceWidth, req->SourceHeight);
485  if (!priv->currentPresentation)
486  {
487  WLog_ERR(TAG, "unable to create presentation video");
488  return CHANNEL_RC_NO_MEMORY;
489  }
490 
491  mappedGeometryRef(geom);
492  priv->currentPresentation->geometry = geom;
493 
494  priv->currentPresentation->video = video;
495  priv->currentPresentation->ScaledWidth = req->ScaledWidth;
496  priv->currentPresentation->ScaledHeight = req->ScaledHeight;
497 
498  geom->custom = priv->currentPresentation;
499  geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
500  geom->MappedGeometryClear = video_onMappedGeometryClear;
501 
502  /* send back response */
503  resp.PresentationId = req->PresentationId;
504  ret = video_control_send_presentation_response(video, &resp);
505  }
506  else if (req->Command == TSMM_STOP_PRESENTATION)
507  {
508  WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
509  if (!priv->currentPresentation)
510  {
511  WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
512  return CHANNEL_RC_OK;
513  }
514 
515  priv->droppedFrames = 0;
516  priv->publishedFrames = 0;
517  PresentationContext_unref(&priv->currentPresentation);
518  }
519 
520  return ret;
521 }
522 
523 static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
524 {
525  TSMM_PRESENTATION_REQUEST req = { 0 };
526 
527  WINPR_ASSERT(context);
528  WINPR_ASSERT(s);
529 
530  if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
531  return ERROR_INVALID_DATA;
532 
533  Stream_Read_UINT8(s, req.PresentationId);
534  Stream_Read_UINT8(s, req.Version);
535  Stream_Read_UINT8(s, req.Command);
536  Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
537 
538  Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
539  Stream_Seek_UINT16(s); /* reserved */
540 
541  Stream_Read_UINT32(s, req.SourceWidth);
542  Stream_Read_UINT32(s, req.SourceHeight);
543  Stream_Read_UINT32(s, req.ScaledWidth);
544  Stream_Read_UINT32(s, req.ScaledHeight);
545  Stream_Read_UINT64(s, req.hnsTimestampOffset);
546  Stream_Read_UINT64(s, req.GeometryMappingId);
547  Stream_Read(s, req.VideoSubtypeId, 16);
548 
549  Stream_Read_UINT32(s, req.cbExtra);
550 
551  if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
552  return ERROR_INVALID_DATA;
553 
554  req.pExtraData = Stream_Pointer(s);
555 
556  WLog_DBG(TAG,
557  "presentationReq: id:%" PRIu8 " version:%" PRIu8
558  " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
559  "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
560  req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
561  req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
562  req.GeometryMappingId);
563 
564  return video_PresentationRequest(context, &req);
565 }
566 
572 static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
573 {
574  GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
575  VIDEO_PLUGIN* video = NULL;
576  VideoClientContext* context = NULL;
577  UINT ret = CHANNEL_RC_OK;
578  UINT32 cbSize = 0;
579  UINT32 packetType = 0;
580 
581  WINPR_ASSERT(callback);
582  WINPR_ASSERT(s);
583 
584  video = (VIDEO_PLUGIN*)callback->plugin;
585  WINPR_ASSERT(video);
586 
587  context = (VideoClientContext*)video->wtsPlugin.pInterface;
588  WINPR_ASSERT(context);
589 
590  if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
591  return ERROR_INVALID_DATA;
592 
593  Stream_Read_UINT32(s, cbSize);
594  if (cbSize < 8)
595  {
596  WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
597  return ERROR_INVALID_DATA;
598  }
599  if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
600  return ERROR_INVALID_DATA;
601 
602  Stream_Read_UINT32(s, packetType);
603  switch (packetType)
604  {
605  case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
606  ret = video_read_tsmm_presentation_req(context, s);
607  break;
608  default:
609  WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
610  ret = ERROR_UNSUPPORTED_TYPE;
611  break;
612  }
613 
614  return ret;
615 }
616 
617 static UINT video_control_send_client_notification(VideoClientContext* context,
618  const TSMM_CLIENT_NOTIFICATION* notif)
619 {
620  BYTE buf[100];
621  wStream* s = NULL;
622  VIDEO_PLUGIN* video = NULL;
623  IWTSVirtualChannel* channel = NULL;
624  UINT ret = 0;
625  UINT32 cbSize = 0;
626 
627  WINPR_ASSERT(context);
628  WINPR_ASSERT(notif);
629 
630  video = (VIDEO_PLUGIN*)context->handle;
631  WINPR_ASSERT(video);
632 
633  s = Stream_New(buf, 32);
634  if (!s)
635  return CHANNEL_RC_NO_MEMORY;
636 
637  cbSize = 16;
638  Stream_Seek_UINT32(s); /* cbSize */
639  Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
640  Stream_Write_UINT8(s, notif->PresentationId);
641  Stream_Write_UINT8(s, notif->NotificationType);
642  Stream_Zero(s, 2);
643  if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
644  {
645  Stream_Write_UINT32(s, 16); /* cbData */
646 
647  /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
648  Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
649  Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
650  Stream_Zero(s, 4ULL * 2ULL);
651 
652  cbSize += 4UL * 4UL;
653  }
654  else
655  {
656  Stream_Write_UINT32(s, 0); /* cbData */
657  }
658 
659  Stream_SealLength(s);
660  Stream_SetPosition(s, 0);
661  Stream_Write_UINT32(s, cbSize);
662  Stream_Free(s, FALSE);
663 
664  WINPR_ASSERT(video->control_callback);
665  WINPR_ASSERT(video->control_callback->channel_callback);
666 
667  channel = video->control_callback->channel_callback->channel;
668  WINPR_ASSERT(channel);
669  WINPR_ASSERT(channel->Write);
670 
671  ret = channel->Write(channel, cbSize, buf, NULL);
672 
673  return ret;
674 }
675 
676 static void video_timer(VideoClientContext* video, UINT64 now)
677 {
678  PresentationContext* presentation = NULL;
679  VideoClientContextPriv* priv = NULL;
680  VideoFrame* peekFrame = NULL;
681  VideoFrame* frame = NULL;
682 
683  WINPR_ASSERT(video);
684 
685  priv = video->priv;
686  WINPR_ASSERT(priv);
687 
688  EnterCriticalSection(&priv->framesLock);
689  do
690  {
691  peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
692  if (!peekFrame)
693  break;
694 
695  if (peekFrame->publishTime > now)
696  break;
697 
698  if (frame)
699  {
700  WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
701  priv->droppedFrames++;
702  VideoFrame_free(&frame);
703  }
704  frame = peekFrame;
705  Queue_Dequeue(priv->frames);
706  } while (1);
707  LeaveCriticalSection(&priv->framesLock);
708 
709  if (!frame)
710  goto treat_feedback;
711 
712  presentation = frame->presentation;
713 
714  priv->publishedFrames++;
715  memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
716 
717  WINPR_ASSERT(video->showSurface);
718  video->showSurface(video, presentation->surface, presentation->ScaledWidth,
719  presentation->ScaledHeight);
720 
721  VideoFrame_free(&frame);
722 
723 treat_feedback:
724  if (priv->nextFeedbackTime < now)
725  {
726  /* we can compute some feedback only if we have some published frames and
727  * a current presentation
728  */
729  if (priv->publishedFrames && priv->currentPresentation)
730  {
731  UINT32 computedRate = 0;
732 
733  PresentationContext_ref(priv->currentPresentation);
734 
735  if (priv->droppedFrames)
736  {
742  if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
743  computedRate = 24;
744  else
745  {
746  computedRate = priv->lastSentRate - 2;
747  if (!computedRate)
748  computedRate = 2;
749  }
750  }
751  else
752  {
757  if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
758  computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
759  else
760  {
761  computedRate = priv->lastSentRate + 2;
762  if (computedRate > XF_VIDEO_UNLIMITED_RATE)
763  computedRate = XF_VIDEO_UNLIMITED_RATE;
764  }
765  }
766 
767  if (computedRate != priv->lastSentRate)
768  {
769  TSMM_CLIENT_NOTIFICATION notif;
770 
771  WINPR_ASSERT(priv->currentPresentation);
772  notif.PresentationId = priv->currentPresentation->PresentationId;
773  notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
774  if (computedRate == XF_VIDEO_UNLIMITED_RATE)
775  {
776  notif.FramerateOverride.Flags = 0x01;
777  notif.FramerateOverride.DesiredFrameRate = 0x00;
778  }
779  else
780  {
781  notif.FramerateOverride.Flags = 0x02;
782  notif.FramerateOverride.DesiredFrameRate = computedRate;
783  }
784 
785  video_control_send_client_notification(video, &notif);
786  priv->lastSentRate = computedRate;
787 
788  WLog_DBG(TAG,
789  "server notified with rate %" PRIu32 " published=%" PRIu32
790  " dropped=%" PRIu32,
791  priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
792  }
793 
794  PresentationContext_unref(&priv->currentPresentation);
795  }
796 
797  WLog_DBG(TAG, "currentRate=%" PRIu32 " published=%" PRIu32 " dropped=%" PRIu32,
798  priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
799 
800  priv->droppedFrames = 0;
801  priv->publishedFrames = 0;
802  priv->nextFeedbackTime = now + 1000;
803  }
804 }
805 
806 static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
807 {
808  VideoClientContextPriv* priv = NULL;
809  PresentationContext* presentation = NULL;
810  int status = 0;
811 
812  WINPR_ASSERT(context);
813  WINPR_ASSERT(data);
814 
815  priv = context->priv;
816  WINPR_ASSERT(priv);
817 
818  presentation = priv->currentPresentation;
819  if (!presentation)
820  {
821  WLog_ERR(TAG, "no current presentation");
822  return CHANNEL_RC_OK;
823  }
824 
825  if (presentation->PresentationId != data->PresentationId)
826  {
827  WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
828  presentation->PresentationId, data->PresentationId);
829  return CHANNEL_RC_OK;
830  }
831 
832  if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
833  {
834  WLog_ERR(TAG, "unable to expand the current packet");
835  return CHANNEL_RC_NO_MEMORY;
836  }
837 
838  Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
839 
840  if (data->CurrentPacketIndex == data->PacketsInSample)
841  {
842  VideoSurface* surface = presentation->surface;
843  H264_CONTEXT* h264 = presentation->h264;
844  UINT64 startTime = GetTickCount64();
845  UINT64 timeAfterH264 = 0;
846  MAPPED_GEOMETRY* geom = presentation->geometry;
847 
848  const RECTANGLE_16 rect = { 0, 0, surface->alignedWidth, surface->alignedHeight };
849  Stream_SealLength(presentation->currentSample);
850  Stream_SetPosition(presentation->currentSample, 0);
851 
852  timeAfterH264 = GetTickCount64();
853  if (data->SampleNumber == 1)
854  {
855  presentation->lastPublishTime = startTime;
856  }
857 
858  presentation->lastPublishTime += (data->hnsDuration / 10000);
859  if (presentation->lastPublishTime <= timeAfterH264 + 10)
860  {
861  int dropped = 0;
862 
863  const size_t len = Stream_Length(presentation->currentSample);
864  if (len > UINT32_MAX)
865  return CHANNEL_RC_OK;
866 
867  /* if the frame is to be published in less than 10 ms, let's consider it's now */
868  status =
869  avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
870  surface->data, surface->format, surface->scanline,
871  surface->alignedWidth, surface->alignedHeight, &rect, 1);
872 
873  if (status < 0)
874  return CHANNEL_RC_OK;
875 
876  WINPR_ASSERT(context->showSurface);
877  context->showSurface(context, presentation->surface, presentation->ScaledWidth,
878  presentation->ScaledHeight);
879 
880  priv->publishedFrames++;
881 
882  /* cleanup previously scheduled frames */
883  EnterCriticalSection(&priv->framesLock);
884  while (Queue_Count(priv->frames) > 0)
885  {
886  VideoFrame* frame = Queue_Dequeue(priv->frames);
887  if (frame)
888  {
889  priv->droppedFrames++;
890  VideoFrame_free(&frame);
891  dropped++;
892  }
893  }
894  LeaveCriticalSection(&priv->framesLock);
895 
896  if (dropped)
897  WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
898  }
899  else
900  {
901  const size_t len = Stream_Length(presentation->currentSample);
902  if (len > UINT32_MAX)
903  return CHANNEL_RC_OK;
904 
905  BOOL enqueueResult = 0;
906  VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
907  if (!frame)
908  {
909  WLog_ERR(TAG, "unable to create frame");
910  return CHANNEL_RC_NO_MEMORY;
911  }
912 
913  status =
914  avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
915  frame->surfaceData, surface->format, surface->scanline,
916  surface->alignedWidth, surface->alignedHeight, &rect, 1);
917  if (status < 0)
918  {
919  VideoFrame_free(&frame);
920  return CHANNEL_RC_OK;
921  }
922 
923  EnterCriticalSection(&priv->framesLock);
924  enqueueResult = Queue_Enqueue(priv->frames, frame);
925  LeaveCriticalSection(&priv->framesLock);
926 
927  if (!enqueueResult)
928  {
929  WLog_ERR(TAG, "unable to enqueue frame");
930  VideoFrame_free(&frame);
931  return CHANNEL_RC_NO_MEMORY;
932  }
933 
934  // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
935  WLog_DBG(TAG, "scheduling frame in %" PRIu32 " ms", (frame->publishTime - startTime));
936  }
937  }
938 
939  return CHANNEL_RC_OK;
940 }
941 
942 static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
943 {
944  GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
945  VIDEO_PLUGIN* video = NULL;
946  VideoClientContext* context = NULL;
947  UINT32 cbSize = 0;
948  UINT32 packetType = 0;
949  TSMM_VIDEO_DATA data;
950 
951  video = (VIDEO_PLUGIN*)callback->plugin;
952  context = (VideoClientContext*)video->wtsPlugin.pInterface;
953 
954  if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
955  return ERROR_INVALID_DATA;
956 
957  Stream_Read_UINT32(s, cbSize);
958  if (cbSize < 8)
959  {
960  WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
961  return ERROR_INVALID_DATA;
962  }
963 
964  if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
965  return ERROR_INVALID_DATA;
966 
967  Stream_Read_UINT32(s, packetType);
968  if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
969  {
970  WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
971  return ERROR_INVALID_DATA;
972  }
973 
974  if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
975  return ERROR_INVALID_DATA;
976 
977  Stream_Read_UINT8(s, data.PresentationId);
978  Stream_Read_UINT8(s, data.Version);
979  Stream_Read_UINT8(s, data.Flags);
980  Stream_Seek_UINT8(s); /* reserved */
981  Stream_Read_UINT64(s, data.hnsTimestamp);
982  Stream_Read_UINT64(s, data.hnsDuration);
983  Stream_Read_UINT16(s, data.CurrentPacketIndex);
984  Stream_Read_UINT16(s, data.PacketsInSample);
985  Stream_Read_UINT32(s, data.SampleNumber);
986  Stream_Read_UINT32(s, data.cbSample);
987  if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
988  return ERROR_INVALID_DATA;
989  data.pSample = Stream_Pointer(s);
990 
991  /*
992  WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
993  duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
994  cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
995  data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
996  data.cbSample);
997  */
998 
999  return video_VideoData(context, &data);
1000 }
1001 
1007 static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1008 {
1009  free(pChannelCallback);
1010  return CHANNEL_RC_OK;
1011 }
1012 
1013 static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1014 {
1015  free(pChannelCallback);
1016  return CHANNEL_RC_OK;
1017 }
1018 
1024 // NOLINTBEGIN(readability-non-const-parameter)
1025 static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1026  IWTSVirtualChannel* channel, BYTE* Data,
1027  BOOL* pbAccept,
1028  IWTSVirtualChannelCallback** ppCallback)
1029 // NOLINTEND(readability-non-const-parameter)
1030 {
1031  GENERIC_CHANNEL_CALLBACK* callback = NULL;
1032  GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1033 
1034  WINPR_UNUSED(Data);
1035  WINPR_UNUSED(pbAccept);
1036 
1037  callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1038  if (!callback)
1039  {
1040  WLog_ERR(TAG, "calloc failed!");
1041  return CHANNEL_RC_NO_MEMORY;
1042  }
1043 
1044  callback->iface.OnDataReceived = video_control_on_data_received;
1045  callback->iface.OnClose = video_control_on_close;
1046  callback->plugin = listener_callback->plugin;
1047  callback->channel_mgr = listener_callback->channel_mgr;
1048  callback->channel = channel;
1049  listener_callback->channel_callback = callback;
1050 
1051  *ppCallback = (IWTSVirtualChannelCallback*)callback;
1052 
1053  return CHANNEL_RC_OK;
1054 }
1055 
1056 // NOLINTBEGIN(readability-non-const-parameter)
1057 static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1058  IWTSVirtualChannel* pChannel, BYTE* Data,
1059  BOOL* pbAccept,
1060  IWTSVirtualChannelCallback** ppCallback)
1061 // NOLINTEND(readability-non-const-parameter)
1062 {
1063  GENERIC_CHANNEL_CALLBACK* callback = NULL;
1064  GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1065 
1066  WINPR_UNUSED(Data);
1067  WINPR_UNUSED(pbAccept);
1068 
1069  callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1070  if (!callback)
1071  {
1072  WLog_ERR(TAG, "calloc failed!");
1073  return CHANNEL_RC_NO_MEMORY;
1074  }
1075 
1076  callback->iface.OnDataReceived = video_data_on_data_received;
1077  callback->iface.OnClose = video_data_on_close;
1078  callback->plugin = listener_callback->plugin;
1079  callback->channel_mgr = listener_callback->channel_mgr;
1080  callback->channel = pChannel;
1081  listener_callback->channel_callback = callback;
1082 
1083  *ppCallback = (IWTSVirtualChannelCallback*)callback;
1084 
1085  return CHANNEL_RC_OK;
1086 }
1087 
1093 static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1094 {
1095  UINT status = 0;
1096  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1097  GENERIC_LISTENER_CALLBACK* callback = NULL;
1098 
1099  if (video->initialized)
1100  {
1101  WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1102  return ERROR_INVALID_DATA;
1103  }
1104  video->control_callback = callback =
1105  (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1106  if (!callback)
1107  {
1108  WLog_ERR(TAG, "calloc for control callback failed!");
1109  return CHANNEL_RC_NO_MEMORY;
1110  }
1111 
1112  callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1113  callback->plugin = plugin;
1114  callback->channel_mgr = channelMgr;
1115 
1116  status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1117  &callback->iface, &(video->controlListener));
1118 
1119  if (status != CHANNEL_RC_OK)
1120  return status;
1121  video->controlListener->pInterface = video->wtsPlugin.pInterface;
1122 
1123  video->data_callback = callback =
1124  (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1125  if (!callback)
1126  {
1127  WLog_ERR(TAG, "calloc for data callback failed!");
1128  return CHANNEL_RC_NO_MEMORY;
1129  }
1130 
1131  callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1132  callback->plugin = plugin;
1133  callback->channel_mgr = channelMgr;
1134 
1135  status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1136  &callback->iface, &(video->dataListener));
1137 
1138  if (status == CHANNEL_RC_OK)
1139  video->dataListener->pInterface = video->wtsPlugin.pInterface;
1140 
1141  video->initialized = status == CHANNEL_RC_OK;
1142  return status;
1143 }
1144 
1150 static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1151 {
1152  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1153 
1154  if (video->control_callback)
1155  {
1156  IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1157  if (mgr)
1158  IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1159  }
1160  if (video->data_callback)
1161  {
1162  IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1163  if (mgr)
1164  IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1165  }
1166 
1167  if (video->context)
1168  VideoClientContextPriv_free(video->context->priv);
1169 
1170  free(video->control_callback);
1171  free(video->data_callback);
1172  free(video->wtsPlugin.pInterface);
1173  free(pPlugin);
1174  return CHANNEL_RC_OK;
1175 }
1176 
1185 FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1186 {
1187  UINT error = CHANNEL_RC_OK;
1188  VIDEO_PLUGIN* videoPlugin = NULL;
1189  VideoClientContext* videoContext = NULL;
1190  VideoClientContextPriv* priv = NULL;
1191 
1192  videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1193  if (!videoPlugin)
1194  {
1195  videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1196  if (!videoPlugin)
1197  {
1198  WLog_ERR(TAG, "calloc failed!");
1199  return CHANNEL_RC_NO_MEMORY;
1200  }
1201 
1202  videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1203  videoPlugin->wtsPlugin.Connected = NULL;
1204  videoPlugin->wtsPlugin.Disconnected = NULL;
1205  videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1206 
1207  videoContext = (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1208  if (!videoContext)
1209  {
1210  WLog_ERR(TAG, "calloc failed!");
1211  free(videoPlugin);
1212  return CHANNEL_RC_NO_MEMORY;
1213  }
1214 
1215  priv = VideoClientContextPriv_new(videoContext);
1216  if (!priv)
1217  {
1218  WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1219  free(videoContext);
1220  free(videoPlugin);
1221  return CHANNEL_RC_NO_MEMORY;
1222  }
1223 
1224  videoContext->handle = (void*)videoPlugin;
1225  videoContext->priv = priv;
1226  videoContext->timer = video_timer;
1227  videoContext->setGeometry = video_client_context_set_geometry;
1228 
1229  videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1230  videoPlugin->context = videoContext;
1231 
1232  error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1233  }
1234  else
1235  {
1236  WLog_ERR(TAG, "could not get video Plugin.");
1237  return CHANNEL_RC_BAD_CHANNEL;
1238  }
1239 
1240  return error;
1241 }
GENERIC_CHANNEL_CALLBACK
Definition: include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition: include/freerdp/client/channels.h:43
RDP_RECT
Definition: types.h:101
RECTANGLE_16
Definition: types.h:109
RTL_CRITICAL_SECTION
Definition: include/winpr/synch.h:158
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition: channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition: channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition: channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition: channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition: client/video.h:36