FreeRDP
planar.c
1 
22 #include <freerdp/config.h>
23 
24 #include <winpr/crt.h>
25 #include <winpr/wtypes.h>
26 #include <winpr/assert.h>
27 #include <winpr/cast.h>
28 #include <winpr/print.h>
29 
30 #include <freerdp/primitives.h>
31 #include <freerdp/log.h>
32 #include <freerdp/codec/bitmap.h>
33 #include <freerdp/codec/planar.h>
34 
35 #define TAG FREERDP_TAG("codec")
36 
37 #define PLANAR_ALIGN(val, align) \
38  ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
39 
40 typedef struct
41 {
47  BYTE controlByte;
48  BYTE* rawValues;
49 } RDP6_RLE_SEGMENT;
50 
51 typedef struct
52 {
53  UINT32 cSegments;
54  RDP6_RLE_SEGMENT* segments;
55 } RDP6_RLE_SEGMENTS;
56 
57 typedef struct
58 {
67  BYTE formatHeader;
68 } RDP6_BITMAP_STREAM;
69 
70 struct S_BITMAP_PLANAR_CONTEXT
71 {
72  UINT32 maxWidth;
73  UINT32 maxHeight;
74  UINT32 maxPlaneSize;
75 
76  BOOL AllowSkipAlpha;
77  BOOL AllowRunLengthEncoding;
78  BOOL AllowColorSubsampling;
79  BOOL AllowDynamicColorFidelity;
80 
81  UINT32 ColorLossLevel;
82 
83  BYTE* planes[4];
84  BYTE* planesBuffer;
85 
86  BYTE* deltaPlanes[4];
87  BYTE* deltaPlanesBuffer;
88 
89  BYTE* rlePlanes[4];
90  BYTE* rlePlanesBuffer;
91 
92  BYTE* pTempData;
93  UINT32 nTempStep;
94 
95  BOOL bgr;
96  BOOL topdown;
97 };
98 
99 static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
100  UINT32 DstFormat)
101 {
102 
103  if (planar->bgr && alpha)
104  {
105  switch (DstFormat)
106  {
107  case PIXEL_FORMAT_ARGB32:
108  DstFormat = PIXEL_FORMAT_ABGR32;
109  break;
110  case PIXEL_FORMAT_XRGB32:
111  DstFormat = PIXEL_FORMAT_XBGR32;
112  break;
113  case PIXEL_FORMAT_ABGR32:
114  DstFormat = PIXEL_FORMAT_ARGB32;
115  break;
116  case PIXEL_FORMAT_XBGR32:
117  DstFormat = PIXEL_FORMAT_XRGB32;
118  break;
119  case PIXEL_FORMAT_BGRA32:
120  DstFormat = PIXEL_FORMAT_RGBA32;
121  break;
122  case PIXEL_FORMAT_BGRX32:
123  DstFormat = PIXEL_FORMAT_RGBX32;
124  break;
125  case PIXEL_FORMAT_RGBA32:
126  DstFormat = PIXEL_FORMAT_BGRA32;
127  break;
128  case PIXEL_FORMAT_RGBX32:
129  DstFormat = PIXEL_FORMAT_BGRX32;
130  break;
131  case PIXEL_FORMAT_RGB24:
132  DstFormat = PIXEL_FORMAT_BGR24;
133  break;
134  case PIXEL_FORMAT_BGR24:
135  DstFormat = PIXEL_FORMAT_RGB24;
136  break;
137  case PIXEL_FORMAT_RGB16:
138  DstFormat = PIXEL_FORMAT_BGR16;
139  break;
140  case PIXEL_FORMAT_BGR16:
141  DstFormat = PIXEL_FORMAT_RGB16;
142  break;
143  case PIXEL_FORMAT_ARGB15:
144  DstFormat = PIXEL_FORMAT_ABGR15;
145  break;
146  case PIXEL_FORMAT_RGB15:
147  DstFormat = PIXEL_FORMAT_BGR15;
148  break;
149  case PIXEL_FORMAT_ABGR15:
150  DstFormat = PIXEL_FORMAT_ARGB15;
151  break;
152  case PIXEL_FORMAT_BGR15:
153  DstFormat = PIXEL_FORMAT_RGB15;
154  break;
155  default:
156  break;
157  }
158  }
159  return DstFormat;
160 }
161 
162 static INLINE BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
163  UINT32 width, UINT32 height,
164  BYTE* WINPR_RESTRICT outPlane,
165  UINT32* WINPR_RESTRICT dstSize);
166 static INLINE BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
167  UINT32 width, UINT32 height,
168  BYTE* WINPR_RESTRICT outPlane);
169 
170 static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
171  UINT32 nWidth, UINT32 nHeight)
172 {
173  UINT32 used = 0;
174 
175  WINPR_ASSERT(pSrcData);
176 
177  for (UINT32 y = 0; y < nHeight; y++)
178  {
179  for (UINT32 x = 0; x < nWidth;)
180  {
181  if (used >= SrcSize)
182  {
183  WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
184  SrcSize);
185  return -1;
186  }
187 
188  const uint8_t controlByte = pSrcData[used++];
189  uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
190  uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
191 
192  if (nRunLength == 1)
193  {
194  nRunLength = cRawBytes + 16;
195  cRawBytes = 0;
196  }
197  else if (nRunLength == 2)
198  {
199  nRunLength = cRawBytes + 32;
200  cRawBytes = 0;
201  }
202 
203  used += cRawBytes;
204  x += cRawBytes;
205  x += nRunLength;
206 
207  if (x > nWidth)
208  {
209  WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
210  return -1;
211  }
212 
213  if (used > SrcSize)
214  {
215  WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
216  INT32_MAX);
217  return -1;
218  }
219  }
220  }
221 
222  if (used > INT32_MAX)
223  {
224  WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
225  return -1;
226  }
227  return (INT32)used;
228 }
229 
230 static inline UINT8 clamp(INT16 val)
231 {
232  return (UINT8)val;
233 }
234 
235 static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
236  UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
237  UINT32 nWidth, UINT32 nHeight)
238 {
239  BYTE* previousScanline = NULL;
240  const BYTE* srcp = pSrcData;
241 
242  WINPR_ASSERT(nHeight <= INT32_MAX);
243  WINPR_ASSERT(nWidth <= INT32_MAX);
244 
245  for (UINT32 y = 0; y < nHeight; y++)
246  {
247  BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
248  BYTE pixel = 0;
249  BYTE* currentScanline = dstp;
250 
251  for (UINT32 x = 0; x < nWidth;)
252  {
253  BYTE controlByte = *srcp;
254  srcp++;
255 
256  if ((srcp - pSrcData) > SrcSize * 1ll)
257  {
258  WLog_ERR(TAG, "error reading input buffer");
259  return -1;
260  }
261 
262  UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
263  UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
264 
265  if (nRunLength == 1)
266  {
267  nRunLength = cRawBytes + 16;
268  cRawBytes = 0;
269  }
270  else if (nRunLength == 2)
271  {
272  nRunLength = cRawBytes + 32;
273  cRawBytes = 0;
274  }
275 
276  if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
277  {
278  WLog_ERR(TAG, "too many pixels in scanline");
279  return -1;
280  }
281 
282  if (!previousScanline)
283  {
284  /* first scanline, absolute values */
285  while (cRawBytes > 0)
286  {
287  pixel = *srcp;
288  srcp++;
289  *dstp = pixel;
290  dstp++;
291  x++;
292  cRawBytes--;
293  }
294 
295  while (nRunLength > 0)
296  {
297  *dstp = pixel;
298  dstp++;
299  x++;
300  nRunLength--;
301  }
302  }
303  else
304  {
305  /* delta values relative to previous scanline */
306  INT16 p = 0;
307  while (cRawBytes > 0)
308  {
309  UINT8 deltaValue = *srcp;
310  srcp++;
311 
312  if (deltaValue & 1)
313  {
314  deltaValue = deltaValue >> 1;
315  deltaValue = deltaValue + 1;
316  p = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
317  }
318  else
319  {
320  deltaValue = deltaValue >> 1;
321  p = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
322  }
323 
324  const INT16 delta = WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + p);
325  *dstp = clamp(delta);
326  dstp++;
327  x++;
328  cRawBytes--;
329  }
330 
331  while (nRunLength > 0)
332  {
333  const INT16 deltaValue =
334  WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + p);
335  *dstp = clamp(deltaValue);
336  dstp++;
337  x++;
338  nRunLength--;
339  }
340  }
341  }
342 
343  previousScanline = currentScanline;
344  }
345 
346  return (INT32)(srcp - pSrcData);
347 }
348 
349 static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
350  BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
351  UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
352  UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
353 {
354  INT32 beg = 0;
355  INT32 end = 0;
356  INT32 inc = 0;
357  BYTE* previousScanline = NULL;
358  const BYTE* srcp = pSrcData;
359 
360  WINPR_ASSERT(nHeight <= INT32_MAX);
361  WINPR_ASSERT(nWidth <= INT32_MAX);
362  WINPR_ASSERT(nDstStep <= INT32_MAX);
363 
364  previousScanline = NULL;
365 
366  if (vFlip)
367  {
368  beg = (INT32)nHeight - 1;
369  end = -1;
370  inc = -1;
371  }
372  else
373  {
374  beg = 0;
375  end = (INT32)nHeight;
376  inc = 1;
377  }
378 
379  for (INT32 y = beg; y != end; y += inc)
380  {
381  const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
382  BYTE* dstp = &pDstData[off];
383  BYTE* currentScanline = dstp;
384  INT16 pixel = 0;
385 
386  for (INT32 x = 0; x < (INT32)nWidth;)
387  {
388  const BYTE controlByte = *srcp;
389  srcp++;
390 
391  if ((srcp - pSrcData) > SrcSize * 1ll)
392  {
393  WLog_ERR(TAG, "error reading input buffer");
394  return -1;
395  }
396 
397  UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
398  UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
399 
400  if (nRunLength == 1)
401  {
402  nRunLength = cRawBytes + 16;
403  cRawBytes = 0;
404  }
405  else if (nRunLength == 2)
406  {
407  nRunLength = cRawBytes + 32;
408  cRawBytes = 0;
409  }
410 
411  if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
412  {
413  WLog_ERR(TAG, "too many pixels in scanline");
414  return -1;
415  }
416 
417  if (!previousScanline)
418  {
419  /* first scanline, absolute values */
420  while (cRawBytes > 0)
421  {
422  pixel = *srcp;
423  srcp++;
424  *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
425  dstp += 4;
426  x++;
427  cRawBytes--;
428  }
429 
430  while (nRunLength > 0)
431  {
432  *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
433  dstp += 4;
434  x++;
435  nRunLength--;
436  }
437  }
438  else
439  {
440  /* delta values relative to previous scanline */
441  while (cRawBytes > 0)
442  {
443  BYTE deltaValue = *srcp;
444  srcp++;
445 
446  if (deltaValue & 1)
447  {
448  deltaValue = deltaValue >> 1;
449  deltaValue = deltaValue + 1;
450  pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
451  }
452  else
453  {
454  deltaValue = deltaValue >> 1;
455  pixel = deltaValue;
456  }
457 
458  const INT16 delta =
459  WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
460  *dstp = clamp(delta);
461  dstp += 4;
462  x++;
463  cRawBytes--;
464  }
465 
466  while (nRunLength > 0)
467  {
468  const INT16 deltaValue =
469  WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
470  *dstp = clamp(deltaValue);
471  dstp += 4;
472  x++;
473  nRunLength--;
474  }
475  }
476  }
477 
478  previousScanline = currentScanline;
479  }
480 
481  return (INT32)(srcp - pSrcData);
482 }
483 
484 static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
485  UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
486  BOOL vFlip)
487 {
488  INT32 beg = 0;
489  INT32 end = 0;
490  INT32 inc = 0;
491 
492  WINPR_ASSERT(nHeight <= INT32_MAX);
493  WINPR_ASSERT(nWidth <= INT32_MAX);
494  WINPR_ASSERT(nDstStep <= INT32_MAX);
495 
496  if (vFlip)
497  {
498  beg = (INT32)nHeight - 1;
499  end = -1;
500  inc = -1;
501  }
502  else
503  {
504  beg = 0;
505  end = (INT32)nHeight;
506  inc = 1;
507  }
508 
509  for (INT32 y = beg; y != end; y += inc)
510  {
511  const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
512  BYTE* dstp = &pDstData[off];
513 
514  for (INT32 x = 0; x < (INT32)nWidth; ++x)
515  {
516  *dstp = bValue;
517  dstp += 4;
518  }
519  }
520 
521  return 0;
522 }
523 
524 static INLINE BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
525  const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
526  const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
527 {
528  WINPR_ASSERT(ppRgba);
529  WINPR_ASSERT(ppR);
530  WINPR_ASSERT(ppG);
531  WINPR_ASSERT(ppB);
532 
533  switch (DstFormat)
534  {
535  case PIXEL_FORMAT_BGRA32:
536  for (UINT32 x = 0; x < width; x++)
537  {
538  *(*ppRgba)++ = *(*ppB)++;
539  *(*ppRgba)++ = *(*ppG)++;
540  *(*ppRgba)++ = *(*ppR)++;
541  *(*ppRgba)++ = *(*ppA)++;
542  }
543 
544  return TRUE;
545 
546  case PIXEL_FORMAT_BGRX32:
547  for (UINT32 x = 0; x < width; x++)
548  {
549  *(*ppRgba)++ = *(*ppB)++;
550  *(*ppRgba)++ = *(*ppG)++;
551  *(*ppRgba)++ = *(*ppR)++;
552  *(*ppRgba)++ = 0xFF;
553  }
554 
555  return TRUE;
556 
557  default:
558  if (ppA)
559  {
560  for (UINT32 x = 0; x < width; x++)
561  {
562  BYTE alpha = *(*ppA)++;
563  UINT32 color =
564  FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
565  FreeRDPWriteColor(*ppRgba, DstFormat, color);
566  *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
567  }
568  }
569  else
570  {
571  const BYTE alpha = 0xFF;
572 
573  for (UINT32 x = 0; x < width; x++)
574  {
575  UINT32 color =
576  FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
577  FreeRDPWriteColor(*ppRgba, DstFormat, color);
578  *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
579  }
580  }
581 
582  return TRUE;
583  }
584 }
585 
586 static INLINE BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
587  BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
588  UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
589  UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
590  UINT32 totalHeight)
591 {
592  INT32 beg = 0;
593  INT32 end = 0;
594  INT32 inc = 0;
595  const BYTE* pR = pSrcData[0];
596  const BYTE* pG = pSrcData[1];
597  const BYTE* pB = pSrcData[2];
598  const BYTE* pA = pSrcData[3];
599  const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
600 
601  if (vFlip)
602  {
603  beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
604  end = -1;
605  inc = -1;
606  }
607  else
608  {
609  beg = 0;
610  end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
611  inc = 1;
612  }
613 
614  if (nYDst + nHeight > totalHeight)
615  {
616  WLog_ERR(TAG,
617  "planar plane destination Y %" PRIu32 " + height %" PRIu32
618  " exceeds totalHeight %" PRIu32,
619  nYDst, nHeight, totalHeight);
620  return FALSE;
621  }
622 
623  if ((nXDst + nWidth) * bpp > nDstStep)
624  {
625  WLog_ERR(TAG,
626  "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
627  " exceeds stride %" PRIu32,
628  nXDst, nWidth, bpp, nDstStep);
629  return FALSE;
630  }
631 
632  for (INT32 y = beg; y != end; y += inc)
633  {
634  BYTE* pRGB = NULL;
635 
636  if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
637  {
638  WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
639  nHeight);
640  return FALSE;
641  }
642 
643  const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
644  pRGB = &pDstData[off];
645 
646  if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
647  return FALSE;
648  }
649 
650  return TRUE;
651 }
652 
653 static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
654  UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
655  UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
656 {
657  size_t pos = 0;
658  WINPR_UNUSED(planeLength);
659 
660  WINPR_ASSERT(plane);
661  WINPR_ASSERT(deltaPlane);
662 
663  if (nWidth > nPlaneWidth * 2)
664  {
665  WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
666  nPlaneWidth);
667  return FALSE;
668  }
669 
670  if (nHeight > nPlaneHeight * 2)
671  {
672  WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
673  nPlaneHeight);
674  return FALSE;
675  }
676 
677  for (size_t y = 0; y < nHeight; y++)
678  {
679  const BYTE* src = plane + y / 2 * nPlaneWidth;
680 
681  for (UINT32 x = 0; x < nWidth; x++)
682  {
683  deltaPlane[pos++] = src[x / 2];
684  }
685  }
686 
687  return TRUE;
688 }
689 
690 BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
691  const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
692  UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
693  UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
694  UINT32 nDstHeight, BOOL vFlip)
695 {
696  BOOL useAlpha = FALSE;
697  INT32 status = 0;
698  INT32 rleSizes[4] = { 0, 0, 0, 0 };
699  UINT32 rawSizes[4] = { 0 };
700  UINT32 rawWidths[4] = { 0 };
701  UINT32 rawHeights[4] = { 0 };
702  const BYTE* planes[4] = { 0 };
703  const UINT32 w = MIN(nSrcWidth, nDstWidth);
704  const UINT32 h = MIN(nSrcHeight, nDstHeight);
705  const primitives_t* prims = primitives_get();
706 
707  WINPR_ASSERT(planar);
708  WINPR_ASSERT(prims);
709 
710  if (nDstStep <= 0)
711  nDstStep = nDstWidth * FreeRDPGetBytesPerPixel(DstFormat);
712 
713  const BYTE* srcp = pSrcData;
714 
715  if (!pSrcData)
716  {
717  WLog_ERR(TAG, "Invalid argument pSrcData=NULL");
718  return FALSE;
719  }
720 
721  if (!pDstData)
722  {
723  WLog_ERR(TAG, "Invalid argument pDstData=NULL");
724  return FALSE;
725  }
726 
727  const BYTE FormatHeader = *srcp++;
728  const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
729  const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE;
730  const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE;
731  const BYTE alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE;
732 
733  DstFormat = planar_invert_format(planar, alpha, DstFormat);
734 
735  if (alpha)
736  useAlpha = FreeRDPColorHasAlpha(DstFormat);
737 
738  // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
739  // alpha);
740 
741  if (!cll && cs)
742  {
743  WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
744  return FALSE; /* Chroma subsampling requires YCoCg */
745  }
746 
747  const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
748  const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
749  const UINT32 planeSize = nSrcWidth * nSrcHeight;
750  const UINT32 subSize = subWidth * subHeight;
751 
752  if (!cs)
753  {
754  rawSizes[0] = planeSize; /* LumaOrRedPlane */
755  rawWidths[0] = nSrcWidth;
756  rawHeights[0] = nSrcHeight;
757  rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
758  rawWidths[1] = nSrcWidth;
759  rawHeights[1] = nSrcHeight;
760  rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
761  rawWidths[2] = nSrcWidth;
762  rawHeights[2] = nSrcHeight;
763  rawSizes[3] = planeSize; /* AlphaPlane */
764  rawWidths[3] = nSrcWidth;
765  rawHeights[3] = nSrcHeight;
766  }
767  else /* Chroma Subsampling */
768  {
769  rawSizes[0] = planeSize; /* LumaOrRedPlane */
770  rawWidths[0] = nSrcWidth;
771  rawHeights[0] = nSrcHeight;
772  rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
773  rawWidths[1] = subWidth;
774  rawHeights[1] = subHeight;
775  rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
776  rawWidths[2] = subWidth;
777  rawHeights[2] = subHeight;
778  rawSizes[3] = planeSize; /* AlphaPlane */
779  rawWidths[3] = nSrcWidth;
780  rawHeights[3] = nSrcHeight;
781  }
782 
783  const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
784  if (SrcSize < diff)
785  {
786  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
787  return FALSE;
788  }
789 
790  if (!rle) /* RAW */
791  {
792 
793  UINT32 base = planeSize * 3;
794  if (cs)
795  base = planeSize + planeSize / 2;
796 
797  if (alpha)
798  {
799  if ((SrcSize - diff) < (planeSize + base))
800  {
801  WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
802  (planeSize + base));
803  return FALSE;
804  }
805 
806  planes[3] = srcp; /* AlphaPlane */
807  planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
808  planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
809  planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
810 
811  if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
812  {
813  WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", planes[2], rawSizes[2],
814  &pSrcData[SrcSize]);
815  return FALSE;
816  }
817  }
818  else
819  {
820  if ((SrcSize - diff) < base)
821  {
822  WLog_ERR(TAG, "plane size mismatch %" PRIu32 " < %" PRIu32, SrcSize - diff, base);
823  return FALSE;
824  }
825 
826  planes[0] = srcp; /* LumaOrRedPlane */
827  planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
828  planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
829 
830  if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
831  {
832  WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", planes[2], rawSizes[2],
833  &pSrcData[SrcSize]);
834  return FALSE;
835  }
836  }
837  }
838  else /* RLE */
839  {
840  if (alpha)
841  {
842  planes[3] = srcp;
843  rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
844  rawHeights[3]); /* AlphaPlane */
845 
846  if (rleSizes[3] < 0)
847  return FALSE;
848 
849  planes[0] = planes[3] + rleSizes[3];
850  }
851  else
852  planes[0] = srcp;
853 
854  const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
855  if (SrcSize < diff0)
856  {
857  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
858  return FALSE;
859  }
860  rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
861  rawHeights[0]); /* RedPlane */
862 
863  if (rleSizes[0] < 0)
864  return FALSE;
865 
866  planes[1] = planes[0] + rleSizes[0];
867 
868  const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
869  if (SrcSize < diff1)
870  {
871  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
872  return FALSE;
873  }
874  rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
875  rawHeights[1]); /* GreenPlane */
876 
877  if (rleSizes[1] < 1)
878  return FALSE;
879 
880  planes[2] = planes[1] + rleSizes[1];
881  const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
882  if (SrcSize < diff2)
883  {
884  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
885  return FALSE;
886  }
887  rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
888  rawHeights[2]); /* BluePlane */
889 
890  if (rleSizes[2] < 1)
891  return FALSE;
892  }
893 
894  if (!cll) /* RGB */
895  {
896  UINT32 TempFormat = 0;
897  BYTE* pTempData = pDstData;
898  UINT32 nTempStep = nDstStep;
899  UINT32 nTotalHeight = nYDst + nDstHeight;
900 
901  if (useAlpha)
902  TempFormat = PIXEL_FORMAT_BGRA32;
903  else
904  TempFormat = PIXEL_FORMAT_BGRX32;
905 
906  TempFormat = planar_invert_format(planar, alpha, TempFormat);
907 
908  if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
909  {
910  pTempData = planar->pTempData;
911  nTempStep = planar->nTempStep;
912  nTotalHeight = planar->maxHeight;
913  }
914 
915  if (!rle) /* RAW */
916  {
917  if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
918  nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
919  return FALSE;
920 
921  if (alpha)
922  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
923  else /* NoAlpha */
924  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
925 
926  if ((SrcSize - (srcp - pSrcData)) == 1)
927  srcp++; /* pad */
928  }
929  else /* RLE */
930  {
931  status = planar_decompress_plane_rle(
932  planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
933  nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
934 
935  if (status < 0)
936  return FALSE;
937 
938  status = planar_decompress_plane_rle(
939  planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
940  nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */
941 
942  if (status < 0)
943  return FALSE;
944 
945  status = planar_decompress_plane_rle(
946  planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
947  nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
948 
949  if (status < 0)
950  return FALSE;
951 
952  srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
953 
954  if (useAlpha)
955  {
956  status = planar_decompress_plane_rle(
957  planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
958  nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
959  }
960  else
961  status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
962  nSrcHeight, 3, vFlip);
963 
964  if (status < 0)
965  return FALSE;
966 
967  if (alpha)
968  srcp += rleSizes[3];
969  }
970 
971  if (pTempData != pDstData)
972  {
973  if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
974  pTempData, TempFormat, nTempStep, nXDst, nYDst, NULL,
975  FREERDP_FLIP_NONE))
976  {
977  WLog_ERR(TAG, "planar image copy failed");
978  return FALSE;
979  }
980  }
981  }
982  else /* YCoCg */
983  {
984  UINT32 TempFormat = 0;
985  BYTE* pTempData = planar->pTempData;
986  UINT32 nTempStep = planar->nTempStep;
987  UINT32 nTotalHeight = planar->maxHeight;
988  BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
989 
990  if (useAlpha)
991  TempFormat = PIXEL_FORMAT_BGRA32;
992  else
993  TempFormat = PIXEL_FORMAT_BGRX32;
994 
995  if (!pTempData)
996  return FALSE;
997 
998  if (rle) /* RLE encoded data. Decode and handle it like raw data. */
999  {
1000  BYTE* rleBuffer[4] = { 0 };
1001 
1002  if (!planar->rlePlanesBuffer)
1003  return FALSE;
1004 
1005  rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1006  rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1007  rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1008  rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1009  if (useAlpha)
1010  {
1011  status = planar_decompress_plane_rle_only(
1012  planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
1013  rawWidths[3], rawHeights[3]); /* AlphaPlane */
1014 
1015  if (status < 0)
1016  return FALSE;
1017  }
1018 
1019  if (alpha)
1020  srcp += rleSizes[3];
1021 
1022  status = planar_decompress_plane_rle_only(
1023  planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
1024  rawWidths[0], rawHeights[0]); /* LumaPlane */
1025 
1026  if (status < 0)
1027  return FALSE;
1028 
1029  status = planar_decompress_plane_rle_only(
1030  planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
1031  rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */
1032 
1033  if (status < 0)
1034  return FALSE;
1035 
1036  status = planar_decompress_plane_rle_only(
1037  planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
1038  rawWidths[2], rawHeights[2]); /* GreenChromaPlane */
1039 
1040  if (status < 0)
1041  return FALSE;
1042 
1043  planes[0] = rleBuffer[0];
1044  planes[1] = rleBuffer[1];
1045  planes[2] = rleBuffer[2];
1046  planes[3] = rleBuffer[3];
1047  }
1048 
1049  /* RAW */
1050  {
1051  if (cs)
1052  { /* Chroma subsampling for Co and Cg:
1053  * Each pixel contains the value that should be expanded to
1054  * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1055  if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1056  rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1057  return FALSE;
1058 
1059  planes[1] = planar->deltaPlanes[0];
1060  rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1061  rawWidths[1] = nSrcWidth;
1062  rawHeights[1] = nSrcHeight;
1063 
1064  if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1065  rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1066  return FALSE;
1067 
1068  planes[2] = planar->deltaPlanes[1];
1069  rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1070  rawWidths[2] = nSrcWidth;
1071  rawHeights[2] = nSrcHeight;
1072  }
1073 
1074  if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1075  nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1076  return FALSE;
1077 
1078  if (alpha)
1079  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1080  else /* NoAlpha */
1081  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1082 
1083  if ((SrcSize - (srcp - pSrcData)) == 1)
1084  srcp++; /* pad */
1085  }
1086 
1087  WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1088  int rc = prims->YCoCgToRGB_8u_AC4R(
1089  pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
1090  WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
1091  if (rc != PRIMITIVES_SUCCESS)
1092  {
1093  WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1094  return FALSE;
1095  }
1096  }
1097 
1098  WINPR_UNUSED(srcp);
1099  return TRUE;
1100 }
1101 
1102 static INLINE BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1103  const BYTE* WINPR_RESTRICT data, UINT32 format,
1104  UINT32 width, UINT32 height, UINT32 scanline,
1105  BYTE* WINPR_RESTRICT planes[4])
1106 {
1107  WINPR_ASSERT(planar);
1108 
1109  if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1110  return FALSE;
1111 
1112  if (scanline == 0)
1113  scanline = width * FreeRDPGetBytesPerPixel(format);
1114 
1115  if (planar->topdown)
1116  {
1117  UINT32 k = 0;
1118  for (UINT32 i = 0; i < height; i++)
1119  {
1120  const BYTE* pixel = &data[1ULL * scanline * i];
1121 
1122  for (UINT32 j = 0; j < width; j++)
1123  {
1124  const UINT32 color = FreeRDPReadColor(pixel, format);
1125  pixel += FreeRDPGetBytesPerPixel(format);
1126  FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1127  &planes[0][k], NULL);
1128  k++;
1129  }
1130  }
1131  }
1132  else
1133  {
1134  UINT32 k = 0;
1135 
1136  for (INT64 i = (INT64)height - 1; i >= 0; i--)
1137  {
1138  const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1139 
1140  for (UINT32 j = 0; j < width; j++)
1141  {
1142  const UINT32 color = FreeRDPReadColor(pixel, format);
1143  pixel += FreeRDPGetBytesPerPixel(format);
1144  FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1145  &planes[0][k], NULL);
1146  k++;
1147  }
1148  }
1149  }
1150  return TRUE;
1151 }
1152 
1153 static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1154  UINT32 cRawBytes, UINT32 nRunLength,
1155  BYTE* WINPR_RESTRICT pOutBuffer,
1156  UINT32 outBufferSize)
1157 {
1158  const BYTE* pInput = NULL;
1159  BYTE* pOutput = NULL;
1160  BYTE controlByte = 0;
1161  UINT32 nBytesToWrite = 0;
1162  pInput = pInBuffer;
1163  pOutput = pOutBuffer;
1164 
1165  if (!cRawBytes && !nRunLength)
1166  return 0;
1167 
1168  if (nRunLength < 3)
1169  {
1170  cRawBytes += nRunLength;
1171  nRunLength = 0;
1172  }
1173 
1174  while (cRawBytes)
1175  {
1176  if (cRawBytes < 16)
1177  {
1178  if (nRunLength > 15)
1179  {
1180  if (nRunLength < 18)
1181  {
1182  controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1183  nRunLength -= 13;
1184  cRawBytes = 0;
1185  }
1186  else
1187  {
1188  controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1189  nRunLength -= 15;
1190  cRawBytes = 0;
1191  }
1192  }
1193  else
1194  {
1195  controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1196  nRunLength = 0;
1197  cRawBytes = 0;
1198  }
1199  }
1200  else
1201  {
1202  controlByte = PLANAR_CONTROL_BYTE(0, 15);
1203  cRawBytes -= 15;
1204  }
1205 
1206  if (outBufferSize < 1)
1207  return 0;
1208 
1209  outBufferSize--;
1210  *pOutput = controlByte;
1211  pOutput++;
1212  nBytesToWrite = (controlByte >> 4);
1213 
1214  if (nBytesToWrite)
1215  {
1216  if (outBufferSize < nBytesToWrite)
1217  return 0;
1218 
1219  outBufferSize -= nBytesToWrite;
1220  CopyMemory(pOutput, pInput, nBytesToWrite);
1221  pOutput += nBytesToWrite;
1222  pInput += nBytesToWrite;
1223  }
1224  }
1225 
1226  while (nRunLength)
1227  {
1228  if (nRunLength > 47)
1229  {
1230  if (nRunLength < 50)
1231  {
1232  controlByte = PLANAR_CONTROL_BYTE(2, 13);
1233  nRunLength -= 45;
1234  }
1235  else
1236  {
1237  controlByte = PLANAR_CONTROL_BYTE(2, 15);
1238  nRunLength -= 47;
1239  }
1240  }
1241  else if (nRunLength > 31)
1242  {
1243  controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1244  nRunLength = 0;
1245  }
1246  else if (nRunLength > 15)
1247  {
1248  controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1249  nRunLength = 0;
1250  }
1251  else
1252  {
1253  controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1254  nRunLength = 0;
1255  }
1256 
1257  if (outBufferSize < 1)
1258  return 0;
1259 
1260  --outBufferSize;
1261  *pOutput = controlByte;
1262  pOutput++;
1263  }
1264 
1265  const intptr_t diff = (pOutput - pOutBuffer);
1266  if ((diff < 0) || (diff > UINT32_MAX))
1267  return 0;
1268  return (UINT32)diff;
1269 }
1270 
1271 static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1272  UINT32 inBufferSize,
1273  BYTE* WINPR_RESTRICT pOutBuffer,
1274  UINT32 outBufferSize)
1275 {
1276  BYTE symbol = 0;
1277  const BYTE* pInput = NULL;
1278  BYTE* pOutput = NULL;
1279  const BYTE* pBytes = NULL;
1280  UINT32 cRawBytes = 0;
1281  UINT32 nRunLength = 0;
1282  UINT32 bSymbolMatch = 0;
1283  UINT32 nBytesWritten = 0;
1284  UINT32 nTotalBytesWritten = 0;
1285  symbol = 0;
1286  cRawBytes = 0;
1287  nRunLength = 0;
1288  pInput = pInBuffer;
1289  pOutput = pOutBuffer;
1290  nTotalBytesWritten = 0;
1291 
1292  if (!outBufferSize)
1293  return 0;
1294 
1295  do
1296  {
1297  if (!inBufferSize)
1298  break;
1299 
1300  bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
1301  symbol = *pInput;
1302  pInput++;
1303  inBufferSize--;
1304 
1305  if (nRunLength && !bSymbolMatch)
1306  {
1307  if (nRunLength < 3)
1308  {
1309  cRawBytes += nRunLength;
1310  nRunLength = 0;
1311  }
1312  else
1313  {
1314  pBytes = pInput - (cRawBytes + nRunLength + 1);
1315  nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1316  pOutput, outBufferSize);
1317  nRunLength = 0;
1318 
1319  if (!nBytesWritten || (nBytesWritten > outBufferSize))
1320  return nRunLength;
1321 
1322  nTotalBytesWritten += nBytesWritten;
1323  outBufferSize -= nBytesWritten;
1324  pOutput += nBytesWritten;
1325  cRawBytes = 0;
1326  }
1327  }
1328 
1329  nRunLength += bSymbolMatch;
1330  cRawBytes += (!bSymbolMatch) ? TRUE : FALSE;
1331  } while (outBufferSize);
1332 
1333  if (cRawBytes || nRunLength)
1334  {
1335  pBytes = pInput - (cRawBytes + nRunLength);
1336  nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1337  pOutput, outBufferSize);
1338 
1339  if (!nBytesWritten)
1340  return 0;
1341 
1342  nTotalBytesWritten += nBytesWritten;
1343  }
1344 
1345  if (inBufferSize)
1346  return 0;
1347 
1348  return nTotalBytesWritten;
1349 }
1350 
1351 BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1352  UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1353  UINT32* WINPR_RESTRICT dstSize)
1354 {
1355  UINT32 index = 0;
1356  const BYTE* pInput = NULL;
1357  BYTE* pOutput = NULL;
1358  UINT32 outBufferSize = 0;
1359  UINT32 nBytesWritten = 0;
1360  UINT32 nTotalBytesWritten = 0;
1361 
1362  if (!outPlane)
1363  return FALSE;
1364 
1365  index = 0;
1366  pInput = inPlane;
1367  pOutput = outPlane;
1368  outBufferSize = *dstSize;
1369  nTotalBytesWritten = 0;
1370 
1371  while (outBufferSize)
1372  {
1373  nBytesWritten =
1374  freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1375 
1376  if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1377  return FALSE;
1378 
1379  outBufferSize -= nBytesWritten;
1380  nTotalBytesWritten += nBytesWritten;
1381  pOutput += nBytesWritten;
1382  pInput += width;
1383  index++;
1384 
1385  if (index >= height)
1386  break;
1387  }
1388 
1389  *dstSize = nTotalBytesWritten;
1390  return TRUE;
1391 }
1392 
1393 static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1394  UINT32 width, UINT32 height,
1395  BYTE* WINPR_RESTRICT outPlanes,
1396  UINT32* WINPR_RESTRICT dstSizes,
1397  BOOL skipAlpha)
1398 {
1399  UINT32 outPlanesSize = width * height * 4;
1400 
1401  /* AlphaPlane */
1402  if (skipAlpha)
1403  {
1404  dstSizes[0] = 0;
1405  }
1406  else
1407  {
1408  dstSizes[0] = outPlanesSize;
1409 
1410  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1411  &dstSizes[0]))
1412  return FALSE;
1413 
1414  outPlanes += dstSizes[0];
1415  outPlanesSize -= dstSizes[0];
1416  }
1417 
1418  /* LumaOrRedPlane */
1419  dstSizes[1] = outPlanesSize;
1420 
1421  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1422  &dstSizes[1]))
1423  return FALSE;
1424 
1425  outPlanes += dstSizes[1];
1426  outPlanesSize -= dstSizes[1];
1427  /* OrangeChromaOrGreenPlane */
1428  dstSizes[2] = outPlanesSize;
1429 
1430  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1431  &dstSizes[2]))
1432  return FALSE;
1433 
1434  outPlanes += dstSizes[2];
1435  outPlanesSize -= dstSizes[2];
1436  /* GreenChromeOrBluePlane */
1437  dstSizes[3] = outPlanesSize;
1438 
1439  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1440  &dstSizes[3]))
1441  return FALSE;
1442 
1443  return TRUE;
1444 }
1445 
1446 BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1447  UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1448 {
1449  BYTE* outPtr = NULL;
1450  const BYTE* srcPtr = NULL;
1451  const BYTE* prevLinePtr = NULL;
1452 
1453  if (!outPlane)
1454  {
1455  if (width * height == 0)
1456  return NULL;
1457 
1458  if (!(outPlane = (BYTE*)calloc(height, width)))
1459  return NULL;
1460  }
1461 
1462  // first line is copied as is
1463  CopyMemory(outPlane, inPlane, width);
1464  outPtr = outPlane + width;
1465  srcPtr = inPlane + width;
1466  prevLinePtr = inPlane;
1467 
1468  for (UINT32 y = 1; y < height; y++)
1469  {
1470  for (UINT32 x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
1471  {
1472  INT32 delta = *srcPtr - *prevLinePtr;
1473  BYTE s2c = WINPR_ASSERTING_INT_CAST(
1474  BYTE, (delta >= 0) ? delta : (~((BYTE)(-delta)) + 1) & 0xFF);
1475  s2c = WINPR_ASSERTING_INT_CAST(
1476  BYTE,
1477  (s2c >= 0) ? (s2c << 1) & 0xFF : (((UINT32)(~((BYTE)s2c) + 1) << 1) - 1) & 0xFF);
1478  *outPtr = s2c;
1479  }
1480  }
1481 
1482  return outPlane;
1483 }
1484 
1485 static INLINE BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1486  UINT32 width, UINT32 height,
1487  BYTE* WINPR_RESTRICT outPlanes[4])
1488 {
1489  for (UINT32 i = 0; i < 4; i++)
1490  {
1491  outPlanes[i] =
1492  freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1493 
1494  if (!outPlanes[i])
1495  return FALSE;
1496  }
1497 
1498  return TRUE;
1499 }
1500 
1501 BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1502  const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1503  UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1504  UINT32* WINPR_RESTRICT pDstSize)
1505 {
1506  UINT32 size = 0;
1507  BYTE* dstp = NULL;
1508  UINT32 planeSize = 0;
1509  UINT32 dstSizes[4] = { 0 };
1510  BYTE FormatHeader = 0;
1511 
1512  if (!context || !context->rlePlanesBuffer)
1513  return NULL;
1514 
1515  if (context->AllowSkipAlpha)
1516  FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1517 
1518  planeSize = width * height;
1519 
1520  if (!context->AllowSkipAlpha)
1521  format = planar_invert_format(context, TRUE, format);
1522 
1523  if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1524  context->planes))
1525  return NULL;
1526 
1527  if (context->AllowRunLengthEncoding)
1528  {
1529  if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1530  context->deltaPlanes))
1531  return NULL;
1532 
1533  if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1534  context->rlePlanesBuffer, dstSizes,
1535  context->AllowSkipAlpha))
1536  return NULL;
1537 
1538  {
1539  uint32_t offset = 0;
1540  FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1541  context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1542  offset += dstSizes[0];
1543  context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1544  offset += dstSizes[1];
1545  context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1546  offset += dstSizes[2];
1547  context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1548 
1549 #if defined(WITH_DEBUG_CODECS)
1550  WLog_DBG(TAG,
1551  "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1552  " / %" PRIu32 "] ",
1553  dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1554 #endif
1555  }
1556  }
1557 
1558  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1559  {
1560  if (!context->AllowRunLengthEncoding)
1561  return NULL;
1562 
1563  if (context->rlePlanes[0] == NULL)
1564  return NULL;
1565 
1566  if (context->rlePlanes[1] == NULL)
1567  return NULL;
1568 
1569  if (context->rlePlanes[2] == NULL)
1570  return NULL;
1571 
1572  if (context->rlePlanes[3] == NULL)
1573  return NULL;
1574  }
1575 
1576  if (!dstData)
1577  {
1578  size = 1;
1579 
1580  if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1581  {
1582  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1583  size += dstSizes[0];
1584  else
1585  size += planeSize;
1586  }
1587 
1588  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1589  size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1590  else
1591  size += (planeSize * 3);
1592 
1593  if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1594  size++;
1595 
1596  dstData = malloc(size);
1597 
1598  if (!dstData)
1599  return NULL;
1600 
1601  *pDstSize = size;
1602  }
1603 
1604  dstp = dstData;
1605  *dstp = FormatHeader; /* FormatHeader */
1606  dstp++;
1607 
1608  /* AlphaPlane */
1609 
1610  if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1611  {
1612  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1613  {
1614  CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1615  dstp += dstSizes[0];
1616  }
1617  else
1618  {
1619  CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1620  dstp += planeSize;
1621  }
1622  }
1623 
1624  /* LumaOrRedPlane */
1625 
1626  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1627  {
1628  CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1629  dstp += dstSizes[1];
1630  }
1631  else
1632  {
1633  CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1634  dstp += planeSize;
1635  }
1636 
1637  /* OrangeChromaOrGreenPlane */
1638 
1639  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1640  {
1641  CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1642  dstp += dstSizes[2];
1643  }
1644  else
1645  {
1646  CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1647  dstp += planeSize;
1648  }
1649 
1650  /* GreenChromeOrBluePlane */
1651 
1652  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1653  {
1654  CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1655  dstp += dstSizes[3];
1656  }
1657  else
1658  {
1659  CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1660  dstp += planeSize;
1661  }
1662 
1663  /* Pad1 (1 byte) */
1664 
1665  if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1666  {
1667  *dstp = 0;
1668  dstp++;
1669  }
1670 
1671  const intptr_t diff = (dstp - dstData);
1672  if ((diff < 0) || (diff > UINT32_MAX))
1673  {
1674  free(dstData);
1675  return NULL;
1676  }
1677  size = (UINT32)diff;
1678  *pDstSize = size;
1679  return dstData;
1680 }
1681 
1682 BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1683  UINT32 width, UINT32 height)
1684 {
1685  if (!context)
1686  return FALSE;
1687 
1688  context->bgr = FALSE;
1689  context->maxWidth = PLANAR_ALIGN(width, 4);
1690  context->maxHeight = PLANAR_ALIGN(height, 4);
1691  {
1692  const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1693  if (tmp > UINT32_MAX)
1694  return FALSE;
1695  context->maxPlaneSize = (UINT32)tmp;
1696  }
1697 
1698  if (context->maxWidth > UINT32_MAX / 4)
1699  return FALSE;
1700  context->nTempStep = context->maxWidth * 4;
1701 
1702  memset((void*)context->planes, 0, sizeof(context->planes));
1703  memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
1704  memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1705 
1706  if (context->maxPlaneSize > 0)
1707  {
1708  void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1709  if (!tmp)
1710  return FALSE;
1711  context->planesBuffer = tmp;
1712 
1713  tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1714  if (!tmp)
1715  return FALSE;
1716  context->pTempData = tmp;
1717 
1718  tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1719  if (!tmp)
1720  return FALSE;
1721  context->deltaPlanesBuffer = tmp;
1722 
1723  tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1724  if (!tmp)
1725  return FALSE;
1726  context->rlePlanesBuffer = tmp;
1727 
1728  context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1729  context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1730  context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1731  context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1732  context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1733  context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1734  context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1735  context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1736  }
1737  return TRUE;
1738 }
1739 
1740 BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1741  UINT32 maxHeight)
1742 {
1743  BITMAP_PLANAR_CONTEXT* context =
1744  (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1745 
1746  if (!context)
1747  return NULL;
1748 
1749  if (flags & PLANAR_FORMAT_HEADER_NA)
1750  context->AllowSkipAlpha = TRUE;
1751 
1752  if (flags & PLANAR_FORMAT_HEADER_RLE)
1753  context->AllowRunLengthEncoding = TRUE;
1754 
1755  if (flags & PLANAR_FORMAT_HEADER_CS)
1756  context->AllowColorSubsampling = TRUE;
1757 
1758  context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1759 
1760  if (context->ColorLossLevel)
1761  context->AllowDynamicColorFidelity = TRUE;
1762 
1763  if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1764  {
1765  WINPR_PRAGMA_DIAG_PUSH
1766  WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1767  freerdp_bitmap_planar_context_free(context);
1768  WINPR_PRAGMA_DIAG_POP
1769  return NULL;
1770  }
1771 
1772  return context;
1773 }
1774 
1775 void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1776 {
1777  if (!context)
1778  return;
1779 
1780  winpr_aligned_free(context->pTempData);
1781  winpr_aligned_free(context->planesBuffer);
1782  winpr_aligned_free(context->deltaPlanesBuffer);
1783  winpr_aligned_free(context->rlePlanesBuffer);
1784  winpr_aligned_free(context);
1785 }
1786 
1787 void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1788 {
1789  WINPR_ASSERT(planar);
1790  planar->bgr = bgr;
1791 }
1792 
1793 void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1794 {
1795  WINPR_ASSERT(planar);
1796  planar->topdown = topdown;
1797 }
primitives_t
Definition: primitives.h:209