FreeRDP
video_main.c
1 
20 #include <freerdp/config.h>
21 
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 
26 #include <winpr/crt.h>
27 #include <winpr/assert.h>
28 #include <winpr/cast.h>
29 #include <winpr/synch.h>
30 #include <winpr/print.h>
31 #include <winpr/stream.h>
32 #include <winpr/cmdline.h>
33 #include <winpr/collections.h>
34 #include <winpr/interlocked.h>
35 #include <winpr/sysinfo.h>
36 
37 #include <freerdp/addin.h>
38 #include <freerdp/primitives.h>
39 #include <freerdp/client/channels.h>
40 #include <freerdp/client/geometry.h>
41 #include <freerdp/client/video.h>
42 #include <freerdp/channels/log.h>
43 #include <freerdp/codec/h264.h>
44 #include <freerdp/codec/yuv.h>
45 
46 #define TAG CHANNELS_TAG("video")
47 
48 #include "video_main.h"
49 
50 typedef struct
51 {
52  IWTSPlugin wtsPlugin;
53 
54  IWTSListener* controlListener;
55  IWTSListener* dataListener;
56  GENERIC_LISTENER_CALLBACK* control_callback;
57  GENERIC_LISTENER_CALLBACK* data_callback;
58 
59  VideoClientContext* context;
60  BOOL initialized;
61 } VIDEO_PLUGIN;
62 
63 #define XF_VIDEO_UNLIMITED_RATE 31
64 
65 static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
66  0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
67 
68 typedef struct
69 {
70  VideoClientContext* video;
71  BYTE PresentationId;
72  UINT32 ScaledWidth, ScaledHeight;
73  MAPPED_GEOMETRY* geometry;
74 
75  UINT64 startTimeStamp;
76  UINT64 publishOffset;
77  H264_CONTEXT* h264;
78  wStream* currentSample;
79  UINT64 lastPublishTime, nextPublishTime;
80  volatile LONG refCounter;
81  VideoSurface* surface;
82 } PresentationContext;
83 
84 typedef struct
85 {
86  UINT64 publishTime;
87  UINT64 hnsDuration;
88  MAPPED_GEOMETRY* geometry;
89  UINT32 w, h;
90  UINT32 scanline;
91  BYTE* surfaceData;
92  PresentationContext* presentation;
93 } VideoFrame;
94 
96 struct s_VideoClientContextPriv
97 {
98  VideoClientContext* video;
99  GeometryClientContext* geometry;
100  wQueue* frames;
101  CRITICAL_SECTION framesLock;
102  wBufferPool* surfacePool;
103  UINT32 publishedFrames;
104  UINT32 droppedFrames;
105  UINT32 lastSentRate;
106  UINT64 nextFeedbackTime;
107  PresentationContext* currentPresentation;
108 };
109 
110 static void PresentationContext_unref(PresentationContext** presentation);
111 static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
112 
113 static const char* video_command_name(BYTE cmd)
114 {
115  switch (cmd)
116  {
117  case TSMM_START_PRESENTATION:
118  return "start";
119  case TSMM_STOP_PRESENTATION:
120  return "stop";
121  default:
122  return "<unknown>";
123  }
124 }
125 
126 static void video_client_context_set_geometry(VideoClientContext* video,
127  GeometryClientContext* geometry)
128 {
129  WINPR_ASSERT(video);
130  WINPR_ASSERT(video->priv);
131  video->priv->geometry = geometry;
132 }
133 
134 static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
135 {
136  VideoClientContextPriv* ret = NULL;
137 
138  WINPR_ASSERT(video);
139  ret = calloc(1, sizeof(*ret));
140  if (!ret)
141  return NULL;
142 
143  ret->frames = Queue_New(TRUE, 10, 2);
144  if (!ret->frames)
145  {
146  WLog_ERR(TAG, "unable to allocate frames queue");
147  goto fail;
148  }
149 
150  ret->surfacePool = BufferPool_New(FALSE, 0, 16);
151  if (!ret->surfacePool)
152  {
153  WLog_ERR(TAG, "unable to create surface pool");
154  goto fail;
155  }
156 
157  if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
158  {
159  WLog_ERR(TAG, "unable to initialize frames lock");
160  goto fail;
161  }
162 
163  ret->video = video;
164 
165  /* don't set to unlimited so that we have the chance to send a feedback in
166  * the first second (for servers that want feedback directly)
167  */
168  ret->lastSentRate = 30;
169  return ret;
170 
171 fail:
172  VideoClientContextPriv_free(ret);
173  return NULL;
174 }
175 
176 static BOOL PresentationContext_ref(PresentationContext* presentation)
177 {
178  WINPR_ASSERT(presentation);
179 
180  InterlockedIncrement(&presentation->refCounter);
181  return TRUE;
182 }
183 
184 static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
185  UINT32 x, UINT32 y, UINT32 width, UINT32 height)
186 {
187  size_t s = 4ULL * width * height;
188  VideoClientContextPriv* priv = NULL;
189  PresentationContext* ret = NULL;
190 
191  WINPR_ASSERT(video);
192 
193  priv = video->priv;
194  WINPR_ASSERT(priv);
195 
196  if (s > INT32_MAX)
197  return NULL;
198 
199  ret = calloc(1, sizeof(*ret));
200  if (!ret)
201  return NULL;
202 
203  ret->video = video;
204  ret->PresentationId = PresentationId;
205 
206  ret->h264 = h264_context_new(FALSE);
207  if (!ret->h264)
208  {
209  WLog_ERR(TAG, "unable to create a h264 context");
210  goto fail;
211  }
212  if (!h264_context_reset(ret->h264, width, height))
213  goto fail;
214 
215  ret->currentSample = Stream_New(NULL, 4096);
216  if (!ret->currentSample)
217  {
218  WLog_ERR(TAG, "unable to create current packet stream");
219  goto fail;
220  }
221 
222  ret->surface = video->createSurface(video, x, y, width, height);
223  if (!ret->surface)
224  {
225  WLog_ERR(TAG, "unable to create surface");
226  goto fail;
227  }
228 
229  if (!PresentationContext_ref(ret))
230  goto fail;
231 
232  return ret;
233 
234 fail:
235  PresentationContext_unref(&ret);
236  return NULL;
237 }
238 
239 static void PresentationContext_unref(PresentationContext** ppresentation)
240 {
241  PresentationContext* presentation = NULL;
242  MAPPED_GEOMETRY* geometry = NULL;
243 
244  WINPR_ASSERT(ppresentation);
245 
246  presentation = *ppresentation;
247  if (!presentation)
248  return;
249 
250  if (InterlockedDecrement(&presentation->refCounter) > 0)
251  return;
252 
253  geometry = presentation->geometry;
254  if (geometry)
255  {
256  geometry->MappedGeometryUpdate = NULL;
257  geometry->MappedGeometryClear = NULL;
258  geometry->custom = NULL;
259  mappedGeometryUnref(geometry);
260  }
261 
262  h264_context_free(presentation->h264);
263  Stream_Free(presentation->currentSample, TRUE);
264  presentation->video->deleteSurface(presentation->video, presentation->surface);
265  free(presentation);
266  *ppresentation = NULL;
267 }
268 
269 static void VideoFrame_free(VideoFrame** pframe)
270 {
271  VideoFrame* frame = NULL;
272 
273  WINPR_ASSERT(pframe);
274  frame = *pframe;
275  if (!frame)
276  return;
277 
278  mappedGeometryUnref(frame->geometry);
279 
280  WINPR_ASSERT(frame->presentation);
281  WINPR_ASSERT(frame->presentation->video);
282  WINPR_ASSERT(frame->presentation->video->priv);
283  BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
284  PresentationContext_unref(&frame->presentation);
285  free(frame);
286  *pframe = NULL;
287 }
288 
289 static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
290  MAPPED_GEOMETRY* geom)
291 {
292  VideoFrame* frame = NULL;
293  const VideoSurface* surface = NULL;
294 
295  WINPR_ASSERT(priv);
296  WINPR_ASSERT(presentation);
297  WINPR_ASSERT(geom);
298 
299  surface = presentation->surface;
300  WINPR_ASSERT(surface);
301 
302  frame = calloc(1, sizeof(VideoFrame));
303  if (!frame)
304  goto fail;
305 
306  mappedGeometryRef(geom);
307 
308  frame->publishTime = presentation->lastPublishTime;
309  frame->geometry = geom;
310  frame->w = surface->alignedWidth;
311  frame->h = surface->alignedHeight;
312  frame->scanline = surface->scanline;
313 
314  frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
315  if (!frame->surfaceData)
316  goto fail;
317 
318  frame->presentation = presentation;
319  if (!PresentationContext_ref(frame->presentation))
320  goto fail;
321 
322  return frame;
323 
324 fail:
325  VideoFrame_free(&frame);
326  return NULL;
327 }
328 
329 void VideoClientContextPriv_free(VideoClientContextPriv* priv)
330 {
331  if (!priv)
332  return;
333 
334  EnterCriticalSection(&priv->framesLock);
335 
336  if (priv->frames)
337  {
338  while (Queue_Count(priv->frames))
339  {
340  VideoFrame* frame = Queue_Dequeue(priv->frames);
341  if (frame)
342  VideoFrame_free(&frame);
343  }
344  }
345 
346  Queue_Free(priv->frames);
347  LeaveCriticalSection(&priv->framesLock);
348 
349  DeleteCriticalSection(&priv->framesLock);
350 
351  if (priv->currentPresentation)
352  PresentationContext_unref(&priv->currentPresentation);
353 
354  BufferPool_Free(priv->surfacePool);
355  free(priv);
356 }
357 
358 static UINT video_control_send_presentation_response(VideoClientContext* context,
359  TSMM_PRESENTATION_RESPONSE* resp)
360 {
361  BYTE buf[12] = { 0 };
362  wStream* s = NULL;
363  VIDEO_PLUGIN* video = NULL;
364  IWTSVirtualChannel* channel = NULL;
365  UINT ret = 0;
366 
367  WINPR_ASSERT(context);
368  WINPR_ASSERT(resp);
369 
370  video = (VIDEO_PLUGIN*)context->handle;
371  WINPR_ASSERT(video);
372 
373  s = Stream_New(buf, 12);
374  if (!s)
375  return CHANNEL_RC_NO_MEMORY;
376 
377  Stream_Write_UINT32(s, 12); /* cbSize */
378  Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
379  Stream_Write_UINT8(s, resp->PresentationId);
380  Stream_Zero(s, 3);
381  Stream_SealLength(s);
382 
383  channel = video->control_callback->channel_callback->channel;
384  ret = channel->Write(channel, 12, buf, NULL);
385  Stream_Free(s, FALSE);
386 
387  return ret;
388 }
389 
390 static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
391 {
392  PresentationContext* presentation = NULL;
393  RDP_RECT* r = NULL;
394 
395  WINPR_ASSERT(geometry);
396 
397  presentation = (PresentationContext*)geometry->custom;
398  WINPR_ASSERT(presentation);
399 
400  r = &geometry->geometry.boundingRect;
401  WLog_DBG(TAG,
402  "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
403  ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
404  "-%" PRId16 "x%" PRId16 ")",
405  geometry->topLevelLeft, geometry->topLevelTop,
406  geometry->topLevelRight - geometry->topLevelLeft,
407  geometry->topLevelBottom - geometry->topLevelTop,
408 
409  geometry->left, geometry->top, geometry->right - geometry->left,
410  geometry->bottom - geometry->top,
411 
412  r->x, r->y, r->width, r->height);
413 
414  presentation->surface->x =
415  WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
416  presentation->surface->y =
417  WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
418 
419  return TRUE;
420 }
421 
422 static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
423 {
424  PresentationContext* presentation = NULL;
425 
426  WINPR_ASSERT(geometry);
427 
428  presentation = (PresentationContext*)geometry->custom;
429  WINPR_ASSERT(presentation);
430 
431  mappedGeometryUnref(presentation->geometry);
432  presentation->geometry = NULL;
433  return TRUE;
434 }
435 
436 static UINT video_PresentationRequest(VideoClientContext* video,
437  const TSMM_PRESENTATION_REQUEST* req)
438 {
439  UINT ret = CHANNEL_RC_OK;
440 
441  WINPR_ASSERT(video);
442  WINPR_ASSERT(req);
443 
444  VideoClientContextPriv* priv = video->priv;
445  WINPR_ASSERT(priv);
446 
447  if (req->Command == TSMM_START_PRESENTATION)
448  {
449  MAPPED_GEOMETRY* geom = NULL;
450  TSMM_PRESENTATION_RESPONSE resp;
451 
452  if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
453  {
454  WLog_ERR(TAG, "not a H264 video, ignoring request");
455  return CHANNEL_RC_OK;
456  }
457 
458  if (priv->currentPresentation)
459  {
460  if (priv->currentPresentation->PresentationId == req->PresentationId)
461  {
462  WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
463  req->PresentationId);
464  return CHANNEL_RC_OK;
465  }
466 
467  WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
468  PresentationContext_unref(&priv->currentPresentation);
469  }
470 
471  if (!priv->geometry)
472  {
473  WLog_ERR(TAG, "geometry channel not ready, ignoring request");
474  return CHANNEL_RC_OK;
475  }
476 
477  geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
478  if (!geom)
479  {
480  WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
481  return CHANNEL_RC_OK;
482  }
483 
484  WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
485  priv->currentPresentation = PresentationContext_new(
486  video, req->PresentationId,
487  WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
488  WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
489  req->SourceHeight);
490  if (!priv->currentPresentation)
491  {
492  WLog_ERR(TAG, "unable to create presentation video");
493  return CHANNEL_RC_NO_MEMORY;
494  }
495 
496  mappedGeometryRef(geom);
497  priv->currentPresentation->geometry = geom;
498 
499  priv->currentPresentation->video = video;
500  priv->currentPresentation->ScaledWidth = req->ScaledWidth;
501  priv->currentPresentation->ScaledHeight = req->ScaledHeight;
502 
503  geom->custom = priv->currentPresentation;
504  geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
505  geom->MappedGeometryClear = video_onMappedGeometryClear;
506 
507  /* send back response */
508  resp.PresentationId = req->PresentationId;
509  ret = video_control_send_presentation_response(video, &resp);
510  }
511  else if (req->Command == TSMM_STOP_PRESENTATION)
512  {
513  WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
514  if (!priv->currentPresentation)
515  {
516  WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
517  return CHANNEL_RC_OK;
518  }
519 
520  priv->droppedFrames = 0;
521  priv->publishedFrames = 0;
522  PresentationContext_unref(&priv->currentPresentation);
523  }
524 
525  return ret;
526 }
527 
528 static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
529 {
530  TSMM_PRESENTATION_REQUEST req = { 0 };
531 
532  WINPR_ASSERT(context);
533  WINPR_ASSERT(s);
534 
535  if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
536  return ERROR_INVALID_DATA;
537 
538  Stream_Read_UINT8(s, req.PresentationId);
539  Stream_Read_UINT8(s, req.Version);
540  Stream_Read_UINT8(s, req.Command);
541  Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
542 
543  Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
544  Stream_Seek_UINT16(s); /* reserved */
545 
546  Stream_Read_UINT32(s, req.SourceWidth);
547  Stream_Read_UINT32(s, req.SourceHeight);
548  Stream_Read_UINT32(s, req.ScaledWidth);
549  Stream_Read_UINT32(s, req.ScaledHeight);
550  Stream_Read_UINT64(s, req.hnsTimestampOffset);
551  Stream_Read_UINT64(s, req.GeometryMappingId);
552  Stream_Read(s, req.VideoSubtypeId, 16);
553 
554  Stream_Read_UINT32(s, req.cbExtra);
555 
556  if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
557  return ERROR_INVALID_DATA;
558 
559  req.pExtraData = Stream_Pointer(s);
560 
561  WLog_DBG(TAG,
562  "presentationReq: id:%" PRIu8 " version:%" PRIu8
563  " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
564  "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
565  req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
566  req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
567  req.GeometryMappingId);
568 
569  return video_PresentationRequest(context, &req);
570 }
571 
577 static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
578 {
579  GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
580  VIDEO_PLUGIN* video = NULL;
581  VideoClientContext* context = NULL;
582  UINT ret = CHANNEL_RC_OK;
583  UINT32 cbSize = 0;
584  UINT32 packetType = 0;
585 
586  WINPR_ASSERT(callback);
587  WINPR_ASSERT(s);
588 
589  video = (VIDEO_PLUGIN*)callback->plugin;
590  WINPR_ASSERT(video);
591 
592  context = (VideoClientContext*)video->wtsPlugin.pInterface;
593  WINPR_ASSERT(context);
594 
595  if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
596  return ERROR_INVALID_DATA;
597 
598  Stream_Read_UINT32(s, cbSize);
599  if (cbSize < 8)
600  {
601  WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
602  return ERROR_INVALID_DATA;
603  }
604  if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
605  return ERROR_INVALID_DATA;
606 
607  Stream_Read_UINT32(s, packetType);
608  switch (packetType)
609  {
610  case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
611  ret = video_read_tsmm_presentation_req(context, s);
612  break;
613  default:
614  WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
615  ret = ERROR_UNSUPPORTED_TYPE;
616  break;
617  }
618 
619  return ret;
620 }
621 
622 static UINT video_control_send_client_notification(VideoClientContext* context,
623  const TSMM_CLIENT_NOTIFICATION* notif)
624 {
625  BYTE buf[100];
626  wStream* s = NULL;
627  VIDEO_PLUGIN* video = NULL;
628  IWTSVirtualChannel* channel = NULL;
629  UINT ret = 0;
630  UINT32 cbSize = 0;
631 
632  WINPR_ASSERT(context);
633  WINPR_ASSERT(notif);
634 
635  video = (VIDEO_PLUGIN*)context->handle;
636  WINPR_ASSERT(video);
637 
638  s = Stream_New(buf, 32);
639  if (!s)
640  return CHANNEL_RC_NO_MEMORY;
641 
642  cbSize = 16;
643  Stream_Seek_UINT32(s); /* cbSize */
644  Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
645  Stream_Write_UINT8(s, notif->PresentationId);
646  Stream_Write_UINT8(s, notif->NotificationType);
647  Stream_Zero(s, 2);
648  if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
649  {
650  Stream_Write_UINT32(s, 16); /* cbData */
651 
652  /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
653  Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
654  Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
655  Stream_Zero(s, 4ULL * 2ULL);
656 
657  cbSize += 4UL * 4UL;
658  }
659  else
660  {
661  Stream_Write_UINT32(s, 0); /* cbData */
662  }
663 
664  Stream_SealLength(s);
665  Stream_SetPosition(s, 0);
666  Stream_Write_UINT32(s, cbSize);
667  Stream_Free(s, FALSE);
668 
669  WINPR_ASSERT(video->control_callback);
670  WINPR_ASSERT(video->control_callback->channel_callback);
671 
672  channel = video->control_callback->channel_callback->channel;
673  WINPR_ASSERT(channel);
674  WINPR_ASSERT(channel->Write);
675 
676  ret = channel->Write(channel, cbSize, buf, NULL);
677 
678  return ret;
679 }
680 
681 static void video_timer(VideoClientContext* video, UINT64 now)
682 {
683  PresentationContext* presentation = NULL;
684  VideoClientContextPriv* priv = NULL;
685  VideoFrame* peekFrame = NULL;
686  VideoFrame* frame = NULL;
687 
688  WINPR_ASSERT(video);
689 
690  priv = video->priv;
691  WINPR_ASSERT(priv);
692 
693  EnterCriticalSection(&priv->framesLock);
694  do
695  {
696  peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
697  if (!peekFrame)
698  break;
699 
700  if (peekFrame->publishTime > now)
701  break;
702 
703  if (frame)
704  {
705  WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
706  priv->droppedFrames++;
707  VideoFrame_free(&frame);
708  }
709  frame = peekFrame;
710  Queue_Dequeue(priv->frames);
711  } while (1);
712  LeaveCriticalSection(&priv->framesLock);
713 
714  if (!frame)
715  goto treat_feedback;
716 
717  presentation = frame->presentation;
718 
719  priv->publishedFrames++;
720  memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
721 
722  WINPR_ASSERT(video->showSurface);
723  video->showSurface(video, presentation->surface, presentation->ScaledWidth,
724  presentation->ScaledHeight);
725 
726  VideoFrame_free(&frame);
727 
728 treat_feedback:
729  if (priv->nextFeedbackTime < now)
730  {
731  /* we can compute some feedback only if we have some published frames and
732  * a current presentation
733  */
734  if (priv->publishedFrames && priv->currentPresentation)
735  {
736  UINT32 computedRate = 0;
737 
738  PresentationContext_ref(priv->currentPresentation);
739 
740  if (priv->droppedFrames)
741  {
747  if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
748  computedRate = 24;
749  else
750  {
751  computedRate = priv->lastSentRate - 2;
752  if (!computedRate)
753  computedRate = 2;
754  }
755  }
756  else
757  {
762  if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
763  computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
764  else
765  {
766  computedRate = priv->lastSentRate + 2;
767  if (computedRate > XF_VIDEO_UNLIMITED_RATE)
768  computedRate = XF_VIDEO_UNLIMITED_RATE;
769  }
770  }
771 
772  if (computedRate != priv->lastSentRate)
773  {
774  TSMM_CLIENT_NOTIFICATION notif;
775 
776  WINPR_ASSERT(priv->currentPresentation);
777  notif.PresentationId = priv->currentPresentation->PresentationId;
778  notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
779  if (computedRate == XF_VIDEO_UNLIMITED_RATE)
780  {
781  notif.FramerateOverride.Flags = 0x01;
782  notif.FramerateOverride.DesiredFrameRate = 0x00;
783  }
784  else
785  {
786  notif.FramerateOverride.Flags = 0x02;
787  notif.FramerateOverride.DesiredFrameRate = computedRate;
788  }
789 
790  video_control_send_client_notification(video, &notif);
791  priv->lastSentRate = computedRate;
792 
793  WLog_DBG(TAG,
794  "server notified with rate %" PRIu32 " published=%" PRIu32
795  " dropped=%" PRIu32,
796  priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
797  }
798 
799  PresentationContext_unref(&priv->currentPresentation);
800  }
801 
802  WLog_DBG(TAG, "currentRate=%" PRIu32 " published=%" PRIu32 " dropped=%" PRIu32,
803  priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
804 
805  priv->droppedFrames = 0;
806  priv->publishedFrames = 0;
807  priv->nextFeedbackTime = now + 1000;
808  }
809 }
810 
811 static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
812 {
813  VideoClientContextPriv* priv = NULL;
814  PresentationContext* presentation = NULL;
815  int status = 0;
816 
817  WINPR_ASSERT(context);
818  WINPR_ASSERT(data);
819 
820  priv = context->priv;
821  WINPR_ASSERT(priv);
822 
823  presentation = priv->currentPresentation;
824  if (!presentation)
825  {
826  WLog_ERR(TAG, "no current presentation");
827  return CHANNEL_RC_OK;
828  }
829 
830  if (presentation->PresentationId != data->PresentationId)
831  {
832  WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
833  presentation->PresentationId, data->PresentationId);
834  return CHANNEL_RC_OK;
835  }
836 
837  if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
838  {
839  WLog_ERR(TAG, "unable to expand the current packet");
840  return CHANNEL_RC_NO_MEMORY;
841  }
842 
843  Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
844 
845  if (data->CurrentPacketIndex == data->PacketsInSample)
846  {
847  VideoSurface* surface = presentation->surface;
848  H264_CONTEXT* h264 = presentation->h264;
849  UINT64 startTime = GetTickCount64();
850  UINT64 timeAfterH264 = 0;
851  MAPPED_GEOMETRY* geom = presentation->geometry;
852 
853  const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
854  WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
855  Stream_SealLength(presentation->currentSample);
856  Stream_SetPosition(presentation->currentSample, 0);
857 
858  timeAfterH264 = GetTickCount64();
859  if (data->SampleNumber == 1)
860  {
861  presentation->lastPublishTime = startTime;
862  }
863 
864  presentation->lastPublishTime += (data->hnsDuration / 10000);
865  if (presentation->lastPublishTime <= timeAfterH264 + 10)
866  {
867  int dropped = 0;
868 
869  const size_t len = Stream_Length(presentation->currentSample);
870  if (len > UINT32_MAX)
871  return CHANNEL_RC_OK;
872 
873  /* if the frame is to be published in less than 10 ms, let's consider it's now */
874  status =
875  avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
876  surface->data, surface->format, surface->scanline,
877  surface->alignedWidth, surface->alignedHeight, &rect, 1);
878 
879  if (status < 0)
880  return CHANNEL_RC_OK;
881 
882  WINPR_ASSERT(context->showSurface);
883  context->showSurface(context, presentation->surface, presentation->ScaledWidth,
884  presentation->ScaledHeight);
885 
886  priv->publishedFrames++;
887 
888  /* cleanup previously scheduled frames */
889  EnterCriticalSection(&priv->framesLock);
890  while (Queue_Count(priv->frames) > 0)
891  {
892  VideoFrame* frame = Queue_Dequeue(priv->frames);
893  if (frame)
894  {
895  priv->droppedFrames++;
896  VideoFrame_free(&frame);
897  dropped++;
898  }
899  }
900  LeaveCriticalSection(&priv->framesLock);
901 
902  if (dropped)
903  WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
904  }
905  else
906  {
907  const size_t len = Stream_Length(presentation->currentSample);
908  if (len > UINT32_MAX)
909  return CHANNEL_RC_OK;
910 
911  BOOL enqueueResult = 0;
912  VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
913  if (!frame)
914  {
915  WLog_ERR(TAG, "unable to create frame");
916  return CHANNEL_RC_NO_MEMORY;
917  }
918 
919  status =
920  avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
921  frame->surfaceData, surface->format, surface->scanline,
922  surface->alignedWidth, surface->alignedHeight, &rect, 1);
923  if (status < 0)
924  {
925  VideoFrame_free(&frame);
926  return CHANNEL_RC_OK;
927  }
928 
929  EnterCriticalSection(&priv->framesLock);
930  enqueueResult = Queue_Enqueue(priv->frames, frame);
931  LeaveCriticalSection(&priv->framesLock);
932 
933  if (!enqueueResult)
934  {
935  WLog_ERR(TAG, "unable to enqueue frame");
936  VideoFrame_free(&frame);
937  return CHANNEL_RC_NO_MEMORY;
938  }
939 
940  // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
941  WLog_DBG(TAG, "scheduling frame in %" PRIu32 " ms", (frame->publishTime - startTime));
942  }
943  }
944 
945  return CHANNEL_RC_OK;
946 }
947 
948 static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
949 {
950  GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
951  VIDEO_PLUGIN* video = NULL;
952  VideoClientContext* context = NULL;
953  UINT32 cbSize = 0;
954  UINT32 packetType = 0;
955  TSMM_VIDEO_DATA data;
956 
957  video = (VIDEO_PLUGIN*)callback->plugin;
958  context = (VideoClientContext*)video->wtsPlugin.pInterface;
959 
960  if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
961  return ERROR_INVALID_DATA;
962 
963  Stream_Read_UINT32(s, cbSize);
964  if (cbSize < 8)
965  {
966  WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
967  return ERROR_INVALID_DATA;
968  }
969 
970  if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
971  return ERROR_INVALID_DATA;
972 
973  Stream_Read_UINT32(s, packetType);
974  if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
975  {
976  WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
977  return ERROR_INVALID_DATA;
978  }
979 
980  if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
981  return ERROR_INVALID_DATA;
982 
983  Stream_Read_UINT8(s, data.PresentationId);
984  Stream_Read_UINT8(s, data.Version);
985  Stream_Read_UINT8(s, data.Flags);
986  Stream_Seek_UINT8(s); /* reserved */
987  Stream_Read_UINT64(s, data.hnsTimestamp);
988  Stream_Read_UINT64(s, data.hnsDuration);
989  Stream_Read_UINT16(s, data.CurrentPacketIndex);
990  Stream_Read_UINT16(s, data.PacketsInSample);
991  Stream_Read_UINT32(s, data.SampleNumber);
992  Stream_Read_UINT32(s, data.cbSample);
993  if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
994  return ERROR_INVALID_DATA;
995  data.pSample = Stream_Pointer(s);
996 
997  /*
998  WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
999  duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
1000  cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
1001  data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
1002  data.cbSample);
1003  */
1004 
1005  return video_VideoData(context, &data);
1006 }
1007 
1013 static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1014 {
1015  free(pChannelCallback);
1016  return CHANNEL_RC_OK;
1017 }
1018 
1019 static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1020 {
1021  free(pChannelCallback);
1022  return CHANNEL_RC_OK;
1023 }
1024 
1030 // NOLINTBEGIN(readability-non-const-parameter)
1031 static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1032  IWTSVirtualChannel* channel, BYTE* Data,
1033  BOOL* pbAccept,
1034  IWTSVirtualChannelCallback** ppCallback)
1035 // NOLINTEND(readability-non-const-parameter)
1036 {
1037  GENERIC_CHANNEL_CALLBACK* callback = NULL;
1038  GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1039 
1040  WINPR_UNUSED(Data);
1041  WINPR_UNUSED(pbAccept);
1042 
1043  callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1044  if (!callback)
1045  {
1046  WLog_ERR(TAG, "calloc failed!");
1047  return CHANNEL_RC_NO_MEMORY;
1048  }
1049 
1050  callback->iface.OnDataReceived = video_control_on_data_received;
1051  callback->iface.OnClose = video_control_on_close;
1052  callback->plugin = listener_callback->plugin;
1053  callback->channel_mgr = listener_callback->channel_mgr;
1054  callback->channel = channel;
1055  listener_callback->channel_callback = callback;
1056 
1057  *ppCallback = (IWTSVirtualChannelCallback*)callback;
1058 
1059  return CHANNEL_RC_OK;
1060 }
1061 
1062 // NOLINTBEGIN(readability-non-const-parameter)
1063 static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1064  IWTSVirtualChannel* pChannel, BYTE* Data,
1065  BOOL* pbAccept,
1066  IWTSVirtualChannelCallback** ppCallback)
1067 // NOLINTEND(readability-non-const-parameter)
1068 {
1069  GENERIC_CHANNEL_CALLBACK* callback = NULL;
1070  GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1071 
1072  WINPR_UNUSED(Data);
1073  WINPR_UNUSED(pbAccept);
1074 
1075  callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1076  if (!callback)
1077  {
1078  WLog_ERR(TAG, "calloc failed!");
1079  return CHANNEL_RC_NO_MEMORY;
1080  }
1081 
1082  callback->iface.OnDataReceived = video_data_on_data_received;
1083  callback->iface.OnClose = video_data_on_close;
1084  callback->plugin = listener_callback->plugin;
1085  callback->channel_mgr = listener_callback->channel_mgr;
1086  callback->channel = pChannel;
1087  listener_callback->channel_callback = callback;
1088 
1089  *ppCallback = (IWTSVirtualChannelCallback*)callback;
1090 
1091  return CHANNEL_RC_OK;
1092 }
1093 
1099 static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1100 {
1101  UINT status = 0;
1102  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1103  GENERIC_LISTENER_CALLBACK* callback = NULL;
1104 
1105  if (video->initialized)
1106  {
1107  WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1108  return ERROR_INVALID_DATA;
1109  }
1110  video->control_callback = callback =
1111  (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1112  if (!callback)
1113  {
1114  WLog_ERR(TAG, "calloc for control callback failed!");
1115  return CHANNEL_RC_NO_MEMORY;
1116  }
1117 
1118  callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1119  callback->plugin = plugin;
1120  callback->channel_mgr = channelMgr;
1121 
1122  status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1123  &callback->iface, &(video->controlListener));
1124 
1125  if (status != CHANNEL_RC_OK)
1126  return status;
1127  video->controlListener->pInterface = video->wtsPlugin.pInterface;
1128 
1129  video->data_callback = callback =
1130  (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1131  if (!callback)
1132  {
1133  WLog_ERR(TAG, "calloc for data callback failed!");
1134  return CHANNEL_RC_NO_MEMORY;
1135  }
1136 
1137  callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1138  callback->plugin = plugin;
1139  callback->channel_mgr = channelMgr;
1140 
1141  status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1142  &callback->iface, &(video->dataListener));
1143 
1144  if (status == CHANNEL_RC_OK)
1145  video->dataListener->pInterface = video->wtsPlugin.pInterface;
1146 
1147  video->initialized = status == CHANNEL_RC_OK;
1148  return status;
1149 }
1150 
1156 static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1157 {
1158  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1159 
1160  if (video->control_callback)
1161  {
1162  IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1163  if (mgr)
1164  IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1165  }
1166  if (video->data_callback)
1167  {
1168  IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1169  if (mgr)
1170  IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1171  }
1172 
1173  if (video->context)
1174  VideoClientContextPriv_free(video->context->priv);
1175 
1176  free(video->control_callback);
1177  free(video->data_callback);
1178  free(video->wtsPlugin.pInterface);
1179  free(pPlugin);
1180  return CHANNEL_RC_OK;
1181 }
1182 
1191 FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1192 {
1193  UINT error = CHANNEL_RC_OK;
1194  VIDEO_PLUGIN* videoPlugin = NULL;
1195  VideoClientContext* videoContext = NULL;
1196  VideoClientContextPriv* priv = NULL;
1197 
1198  videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1199  if (!videoPlugin)
1200  {
1201  videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1202  if (!videoPlugin)
1203  {
1204  WLog_ERR(TAG, "calloc failed!");
1205  return CHANNEL_RC_NO_MEMORY;
1206  }
1207 
1208  videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1209  videoPlugin->wtsPlugin.Connected = NULL;
1210  videoPlugin->wtsPlugin.Disconnected = NULL;
1211  videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1212 
1213  videoContext = (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1214  if (!videoContext)
1215  {
1216  WLog_ERR(TAG, "calloc failed!");
1217  free(videoPlugin);
1218  return CHANNEL_RC_NO_MEMORY;
1219  }
1220 
1221  priv = VideoClientContextPriv_new(videoContext);
1222  if (!priv)
1223  {
1224  WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1225  free(videoContext);
1226  free(videoPlugin);
1227  return CHANNEL_RC_NO_MEMORY;
1228  }
1229 
1230  videoContext->handle = (void*)videoPlugin;
1231  videoContext->priv = priv;
1232  videoContext->timer = video_timer;
1233  videoContext->setGeometry = video_client_context_set_geometry;
1234 
1235  videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1236  videoPlugin->context = videoContext;
1237 
1238  error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1239  }
1240  else
1241  {
1242  WLog_ERR(TAG, "could not get video Plugin.");
1243  return CHANNEL_RC_BAD_CHANNEL;
1244  }
1245 
1246  return error;
1247 }
GENERIC_CHANNEL_CALLBACK
Definition: include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition: include/freerdp/client/channels.h:43
RDP_RECT
Definition: types.h:101
RECTANGLE_16
Definition: types.h:109
RTL_CRITICAL_SECTION
Definition: include/winpr/synch.h:158
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition: channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition: channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition: channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition: channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition: client/video.h:36