FreeRDP
planar.c
1 
22 #include <freerdp/config.h>
23 
24 #include <winpr/crt.h>
25 #include <winpr/wtypes.h>
26 #include <winpr/assert.h>
27 #include <winpr/print.h>
28 
29 #include <freerdp/primitives.h>
30 #include <freerdp/log.h>
31 #include <freerdp/codec/bitmap.h>
32 #include <freerdp/codec/planar.h>
33 
34 #define TAG FREERDP_TAG("codec")
35 
36 #define PLANAR_ALIGN(val, align) \
37  ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
38 
39 typedef struct
40 {
46  BYTE controlByte;
47  BYTE* rawValues;
48 } RDP6_RLE_SEGMENT;
49 
50 typedef struct
51 {
52  UINT32 cSegments;
53  RDP6_RLE_SEGMENT* segments;
54 } RDP6_RLE_SEGMENTS;
55 
56 typedef struct
57 {
66  BYTE formatHeader;
67 } RDP6_BITMAP_STREAM;
68 
69 struct S_BITMAP_PLANAR_CONTEXT
70 {
71  UINT32 maxWidth;
72  UINT32 maxHeight;
73  UINT32 maxPlaneSize;
74 
75  BOOL AllowSkipAlpha;
76  BOOL AllowRunLengthEncoding;
77  BOOL AllowColorSubsampling;
78  BOOL AllowDynamicColorFidelity;
79 
80  UINT32 ColorLossLevel;
81 
82  BYTE* planes[4];
83  BYTE* planesBuffer;
84 
85  BYTE* deltaPlanes[4];
86  BYTE* deltaPlanesBuffer;
87 
88  BYTE* rlePlanes[4];
89  BYTE* rlePlanesBuffer;
90 
91  BYTE* pTempData;
92  UINT32 nTempStep;
93 
94  BOOL bgr;
95  BOOL topdown;
96 };
97 
98 static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
99  UINT32 DstFormat)
100 {
101 
102  if (planar->bgr && alpha)
103  {
104  switch (DstFormat)
105  {
106  case PIXEL_FORMAT_ARGB32:
107  DstFormat = PIXEL_FORMAT_ABGR32;
108  break;
109  case PIXEL_FORMAT_XRGB32:
110  DstFormat = PIXEL_FORMAT_XBGR32;
111  break;
112  case PIXEL_FORMAT_ABGR32:
113  DstFormat = PIXEL_FORMAT_ARGB32;
114  break;
115  case PIXEL_FORMAT_XBGR32:
116  DstFormat = PIXEL_FORMAT_XRGB32;
117  break;
118  case PIXEL_FORMAT_BGRA32:
119  DstFormat = PIXEL_FORMAT_RGBA32;
120  break;
121  case PIXEL_FORMAT_BGRX32:
122  DstFormat = PIXEL_FORMAT_RGBX32;
123  break;
124  case PIXEL_FORMAT_RGBA32:
125  DstFormat = PIXEL_FORMAT_BGRA32;
126  break;
127  case PIXEL_FORMAT_RGBX32:
128  DstFormat = PIXEL_FORMAT_BGRX32;
129  break;
130  case PIXEL_FORMAT_RGB24:
131  DstFormat = PIXEL_FORMAT_BGR24;
132  break;
133  case PIXEL_FORMAT_BGR24:
134  DstFormat = PIXEL_FORMAT_RGB24;
135  break;
136  case PIXEL_FORMAT_RGB16:
137  DstFormat = PIXEL_FORMAT_BGR16;
138  break;
139  case PIXEL_FORMAT_BGR16:
140  DstFormat = PIXEL_FORMAT_RGB16;
141  break;
142  case PIXEL_FORMAT_ARGB15:
143  DstFormat = PIXEL_FORMAT_ABGR15;
144  break;
145  case PIXEL_FORMAT_RGB15:
146  DstFormat = PIXEL_FORMAT_BGR15;
147  break;
148  case PIXEL_FORMAT_ABGR15:
149  DstFormat = PIXEL_FORMAT_ARGB15;
150  break;
151  case PIXEL_FORMAT_BGR15:
152  DstFormat = PIXEL_FORMAT_RGB15;
153  break;
154  default:
155  break;
156  }
157  }
158  return DstFormat;
159 }
160 
161 static INLINE BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
162  UINT32 width, UINT32 height,
163  BYTE* WINPR_RESTRICT outPlane,
164  UINT32* WINPR_RESTRICT dstSize);
165 static INLINE BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
166  UINT32 width, UINT32 height,
167  BYTE* WINPR_RESTRICT outPlane);
168 
169 static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
170  UINT32 nWidth, UINT32 nHeight)
171 {
172  UINT32 used = 0;
173  BYTE controlByte = 0;
174 
175  WINPR_ASSERT(pSrcData);
176 
177  for (UINT32 y = 0; y < nHeight; y++)
178  {
179  for (UINT32 x = 0; x < nWidth;)
180  {
181  int cRawBytes = 0;
182  int nRunLength = 0;
183 
184  if (used >= SrcSize)
185  {
186  WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
187  SrcSize);
188  return -1;
189  }
190 
191  controlByte = pSrcData[used++];
192  nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
193  cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
194 
195  if (nRunLength == 1)
196  {
197  nRunLength = cRawBytes + 16;
198  cRawBytes = 0;
199  }
200  else if (nRunLength == 2)
201  {
202  nRunLength = cRawBytes + 32;
203  cRawBytes = 0;
204  }
205 
206  used += cRawBytes;
207  x += cRawBytes;
208  x += nRunLength;
209 
210  if (x > nWidth)
211  {
212  WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
213  return -1;
214  }
215 
216  if (used > SrcSize)
217  {
218  WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
219  INT32_MAX);
220  return -1;
221  }
222  }
223  }
224 
225  if (used > INT32_MAX)
226  {
227  WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
228  return -1;
229  }
230  return (INT32)used;
231 }
232 
233 static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
234  UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
235  UINT32 nWidth, UINT32 nHeight)
236 {
237  UINT32 pixel = 0;
238  UINT32 cRawBytes = 0;
239  UINT32 nRunLength = 0;
240  INT32 deltaValue = 0;
241  BYTE controlByte = 0;
242  BYTE* currentScanline = NULL;
243  BYTE* previousScanline = NULL;
244  const BYTE* srcp = pSrcData;
245 
246  WINPR_ASSERT(nHeight <= INT32_MAX);
247  WINPR_ASSERT(nWidth <= INT32_MAX);
248 
249  previousScanline = NULL;
250 
251  for (INT32 y = 0; y < (INT32)nHeight; y++)
252  {
253  BYTE* dstp = &pDstData[(1ULL * (y) * (INT32)nWidth)];
254  pixel = 0;
255  currentScanline = dstp;
256 
257  for (INT32 x = 0; x < (INT32)nWidth;)
258  {
259  controlByte = *srcp;
260  srcp++;
261 
262  if ((srcp - pSrcData) > SrcSize * 1ll)
263  {
264  WLog_ERR(TAG, "error reading input buffer");
265  return -1;
266  }
267 
268  nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
269  cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
270 
271  if (nRunLength == 1)
272  {
273  nRunLength = cRawBytes + 16;
274  cRawBytes = 0;
275  }
276  else if (nRunLength == 2)
277  {
278  nRunLength = cRawBytes + 32;
279  cRawBytes = 0;
280  }
281 
282  if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
283  {
284  WLog_ERR(TAG, "too many pixels in scanline");
285  return -1;
286  }
287 
288  if (!previousScanline)
289  {
290  /* first scanline, absolute values */
291  while (cRawBytes > 0)
292  {
293  pixel = *srcp;
294  srcp++;
295  *dstp = pixel;
296  dstp++;
297  x++;
298  cRawBytes--;
299  }
300 
301  while (nRunLength > 0)
302  {
303  *dstp = pixel;
304  dstp++;
305  x++;
306  nRunLength--;
307  }
308  }
309  else
310  {
311  /* delta values relative to previous scanline */
312  while (cRawBytes > 0)
313  {
314  deltaValue = *srcp;
315  srcp++;
316 
317  if (deltaValue & 1)
318  {
319  deltaValue = deltaValue >> 1;
320  deltaValue = deltaValue + 1;
321  pixel = -deltaValue;
322  }
323  else
324  {
325  deltaValue = deltaValue >> 1;
326  pixel = deltaValue;
327  }
328 
329  deltaValue = previousScanline[x] + pixel;
330  *dstp = deltaValue;
331  dstp++;
332  x++;
333  cRawBytes--;
334  }
335 
336  while (nRunLength > 0)
337  {
338  deltaValue = previousScanline[x] + pixel;
339  *dstp = deltaValue;
340  dstp++;
341  x++;
342  nRunLength--;
343  }
344  }
345  }
346 
347  previousScanline = currentScanline;
348  }
349 
350  return (INT32)(srcp - pSrcData);
351 }
352 
353 static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
354  BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
355  UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
356  UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
357 {
358  UINT32 pixel = 0;
359  UINT32 cRawBytes = 0;
360  UINT32 nRunLength = 0;
361  INT32 deltaValue = 0;
362  INT32 beg = 0;
363  INT32 end = 0;
364  INT32 inc = 0;
365  BYTE controlByte = 0;
366  BYTE* currentScanline = NULL;
367  BYTE* previousScanline = NULL;
368  const BYTE* srcp = pSrcData;
369 
370  WINPR_ASSERT(nHeight <= INT32_MAX);
371  WINPR_ASSERT(nWidth <= INT32_MAX);
372  WINPR_ASSERT(nDstStep <= INT32_MAX);
373 
374  previousScanline = NULL;
375 
376  if (vFlip)
377  {
378  beg = (INT32)nHeight - 1;
379  end = -1;
380  inc = -1;
381  }
382  else
383  {
384  beg = 0;
385  end = (INT32)nHeight;
386  inc = 1;
387  }
388 
389  for (INT32 y = beg; y != end; y += inc)
390  {
391  BYTE* dstp = &pDstData[((nYDst + y) * nDstStep) + (nXDst * 4) + nChannel];
392  pixel = 0;
393  currentScanline = dstp;
394 
395  for (INT32 x = 0; x < (INT32)nWidth;)
396  {
397  controlByte = *srcp;
398  srcp++;
399 
400  if ((srcp - pSrcData) > SrcSize * 1ll)
401  {
402  WLog_ERR(TAG, "error reading input buffer");
403  return -1;
404  }
405 
406  nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
407  cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
408 
409  if (nRunLength == 1)
410  {
411  nRunLength = cRawBytes + 16;
412  cRawBytes = 0;
413  }
414  else if (nRunLength == 2)
415  {
416  nRunLength = cRawBytes + 32;
417  cRawBytes = 0;
418  }
419 
420  if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
421  {
422  WLog_ERR(TAG, "too many pixels in scanline");
423  return -1;
424  }
425 
426  if (!previousScanline)
427  {
428  /* first scanline, absolute values */
429  while (cRawBytes > 0)
430  {
431  pixel = *srcp;
432  srcp++;
433  *dstp = pixel;
434  dstp += 4;
435  x++;
436  cRawBytes--;
437  }
438 
439  while (nRunLength > 0)
440  {
441  *dstp = pixel;
442  dstp += 4;
443  x++;
444  nRunLength--;
445  }
446  }
447  else
448  {
449  /* delta values relative to previous scanline */
450  while (cRawBytes > 0)
451  {
452  deltaValue = *srcp;
453  srcp++;
454 
455  if (deltaValue & 1)
456  {
457  deltaValue = deltaValue >> 1;
458  deltaValue = deltaValue + 1;
459  pixel = -deltaValue;
460  }
461  else
462  {
463  deltaValue = deltaValue >> 1;
464  pixel = deltaValue;
465  }
466 
467  deltaValue = previousScanline[4LL * x] + pixel;
468  *dstp = deltaValue;
469  dstp += 4;
470  x++;
471  cRawBytes--;
472  }
473 
474  while (nRunLength > 0)
475  {
476  deltaValue = previousScanline[4LL * x] + pixel;
477  *dstp = deltaValue;
478  dstp += 4;
479  x++;
480  nRunLength--;
481  }
482  }
483  }
484 
485  previousScanline = currentScanline;
486  }
487 
488  return (INT32)(srcp - pSrcData);
489 }
490 
491 static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
492  UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
493  BOOL vFlip)
494 {
495  INT32 beg = 0;
496  INT32 end = 0;
497  INT32 inc = 0;
498 
499  WINPR_ASSERT(nHeight <= INT32_MAX);
500  WINPR_ASSERT(nWidth <= INT32_MAX);
501  WINPR_ASSERT(nDstStep <= INT32_MAX);
502 
503  if (vFlip)
504  {
505  beg = (INT32)nHeight - 1;
506  end = -1;
507  inc = -1;
508  }
509  else
510  {
511  beg = 0;
512  end = (INT32)nHeight;
513  inc = 1;
514  }
515 
516  for (INT32 y = beg; y != end; y += inc)
517  {
518  BYTE* dstp = &pDstData[((nYDst + y) * nDstStep) + (nXDst * 4) + nChannel];
519 
520  for (INT32 x = 0; x < (INT32)nWidth; ++x)
521  {
522  *dstp = bValue;
523  dstp += 4;
524  }
525  }
526 
527  return 0;
528 }
529 
530 static INLINE BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
531  const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
532  const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
533 {
534  WINPR_ASSERT(ppRgba);
535  WINPR_ASSERT(ppR);
536  WINPR_ASSERT(ppG);
537  WINPR_ASSERT(ppB);
538 
539  switch (DstFormat)
540  {
541  case PIXEL_FORMAT_BGRA32:
542  for (UINT32 x = 0; x < width; x++)
543  {
544  *(*ppRgba)++ = *(*ppB)++;
545  *(*ppRgba)++ = *(*ppG)++;
546  *(*ppRgba)++ = *(*ppR)++;
547  *(*ppRgba)++ = *(*ppA)++;
548  }
549 
550  return TRUE;
551 
552  case PIXEL_FORMAT_BGRX32:
553  for (UINT32 x = 0; x < width; x++)
554  {
555  *(*ppRgba)++ = *(*ppB)++;
556  *(*ppRgba)++ = *(*ppG)++;
557  *(*ppRgba)++ = *(*ppR)++;
558  *(*ppRgba)++ = 0xFF;
559  }
560 
561  return TRUE;
562 
563  default:
564  if (ppA)
565  {
566  for (UINT32 x = 0; x < width; x++)
567  {
568  BYTE alpha = *(*ppA)++;
569  UINT32 color =
570  FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
571  FreeRDPWriteColor(*ppRgba, DstFormat, color);
572  *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
573  }
574  }
575  else
576  {
577  const BYTE alpha = 0xFF;
578 
579  for (UINT32 x = 0; x < width; x++)
580  {
581  UINT32 color =
582  FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
583  FreeRDPWriteColor(*ppRgba, DstFormat, color);
584  *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
585  }
586  }
587 
588  return TRUE;
589  }
590 }
591 
592 static INLINE BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
593  BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
594  UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
595  UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
596  UINT32 totalHeight)
597 {
598  INT32 beg = 0;
599  INT32 end = 0;
600  INT32 inc = 0;
601  const BYTE* pR = pSrcData[0];
602  const BYTE* pG = pSrcData[1];
603  const BYTE* pB = pSrcData[2];
604  const BYTE* pA = pSrcData[3];
605  const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
606 
607  if (vFlip)
608  {
609  beg = nHeight - 1;
610  end = -1;
611  inc = -1;
612  }
613  else
614  {
615  beg = 0;
616  end = nHeight;
617  inc = 1;
618  }
619 
620  if (nYDst + nHeight > totalHeight)
621  {
622  WLog_ERR(TAG,
623  "planar plane destination Y %" PRIu32 " + height %" PRIu32
624  " exceeds totalHeight %" PRIu32,
625  nYDst, nHeight, totalHeight);
626  return FALSE;
627  }
628 
629  if ((nXDst + nWidth) * bpp > nDstStep)
630  {
631  WLog_ERR(TAG,
632  "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
633  " exceeds stride %" PRIu32,
634  nXDst, nWidth, bpp, nDstStep);
635  return FALSE;
636  }
637 
638  for (INT32 y = beg; y != end; y += inc)
639  {
640  BYTE* pRGB = NULL;
641 
642  if (y > (INT64)nHeight)
643  {
644  WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
645  nHeight);
646  return FALSE;
647  }
648 
649  pRGB = &pDstData[((nYDst + y) * nDstStep) + (nXDst * bpp)];
650 
651  if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
652  return FALSE;
653  }
654 
655  return TRUE;
656 }
657 
658 static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
659  UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
660  UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
661 {
662  size_t pos = 0;
663  WINPR_UNUSED(planeLength);
664 
665  WINPR_ASSERT(plane);
666  WINPR_ASSERT(deltaPlane);
667 
668  if (nWidth > nPlaneWidth * 2)
669  {
670  WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
671  nPlaneWidth);
672  return FALSE;
673  }
674 
675  if (nHeight > nPlaneHeight * 2)
676  {
677  WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
678  nPlaneHeight);
679  return FALSE;
680  }
681 
682  for (size_t y = 0; y < nHeight; y++)
683  {
684  const BYTE* src = plane + y / 2 * nPlaneWidth;
685 
686  for (UINT32 x = 0; x < nWidth; x++)
687  {
688  deltaPlane[pos++] = src[x / 2];
689  }
690  }
691 
692  return TRUE;
693 }
694 
695 BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
696  const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
697  UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
698  UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
699  UINT32 nDstHeight, BOOL vFlip)
700 {
701  BOOL cs = 0;
702  BOOL rle = 0;
703  UINT32 cll = 0;
704  BOOL alpha = 0;
705  BOOL useAlpha = FALSE;
706  INT32 status = 0;
707  const BYTE* srcp = NULL;
708  UINT32 subSize = 0;
709  UINT32 subWidth = 0;
710  UINT32 subHeight = 0;
711  UINT32 planeSize = 0;
712  INT32 rleSizes[4] = { 0, 0, 0, 0 };
713  UINT32 rawSizes[4];
714  UINT32 rawWidths[4];
715  UINT32 rawHeights[4];
716  BYTE FormatHeader = 0;
717  const BYTE* planes[4] = { 0 };
718  const UINT32 w = MIN(nSrcWidth, nDstWidth);
719  const UINT32 h = MIN(nSrcHeight, nDstHeight);
720  const primitives_t* prims = primitives_get();
721 
722  WINPR_ASSERT(planar);
723  WINPR_ASSERT(prims);
724 
725  if (nDstStep <= 0)
726  nDstStep = nDstWidth * FreeRDPGetBytesPerPixel(DstFormat);
727 
728  srcp = pSrcData;
729 
730  if (!pSrcData)
731  {
732  WLog_ERR(TAG, "Invalid argument pSrcData=NULL");
733  return FALSE;
734  }
735 
736  if (!pDstData)
737  {
738  WLog_ERR(TAG, "Invalid argument pDstData=NULL");
739  return FALSE;
740  }
741 
742  FormatHeader = *srcp++;
743  cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
744  cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE;
745  rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE;
746  alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE;
747 
748  DstFormat = planar_invert_format(planar, alpha, DstFormat);
749 
750  if (alpha)
751  useAlpha = FreeRDPColorHasAlpha(DstFormat);
752 
753  // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
754  // alpha);
755 
756  if (!cll && cs)
757  {
758  WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
759  return FALSE; /* Chroma subsampling requires YCoCg */
760  }
761 
762  subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
763  subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
764  planeSize = nSrcWidth * nSrcHeight;
765  subSize = subWidth * subHeight;
766 
767  if (!cs)
768  {
769  rawSizes[0] = planeSize; /* LumaOrRedPlane */
770  rawWidths[0] = nSrcWidth;
771  rawHeights[0] = nSrcHeight;
772  rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
773  rawWidths[1] = nSrcWidth;
774  rawHeights[1] = nSrcHeight;
775  rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
776  rawWidths[2] = nSrcWidth;
777  rawHeights[2] = nSrcHeight;
778  rawSizes[3] = planeSize; /* AlphaPlane */
779  rawWidths[3] = nSrcWidth;
780  rawHeights[3] = nSrcHeight;
781  }
782  else /* Chroma Subsampling */
783  {
784  rawSizes[0] = planeSize; /* LumaOrRedPlane */
785  rawWidths[0] = nSrcWidth;
786  rawHeights[0] = nSrcHeight;
787  rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
788  rawWidths[1] = subWidth;
789  rawHeights[1] = subHeight;
790  rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
791  rawWidths[2] = subWidth;
792  rawHeights[2] = subHeight;
793  rawSizes[3] = planeSize; /* AlphaPlane */
794  rawWidths[3] = nSrcWidth;
795  rawHeights[3] = nSrcHeight;
796  }
797 
798  const size_t diff = srcp - pSrcData;
799  if (SrcSize < diff)
800  {
801  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
802  return FALSE;
803  }
804 
805  if (!rle) /* RAW */
806  {
807 
808  UINT32 base = planeSize * 3;
809  if (cs)
810  base = planeSize + planeSize / 2;
811 
812  if (alpha)
813  {
814  if ((SrcSize - diff) < (planeSize + base))
815  {
816  WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
817  (planeSize + base));
818  return FALSE;
819  }
820 
821  planes[3] = srcp; /* AlphaPlane */
822  planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
823  planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
824  planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
825 
826  if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
827  {
828  WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", planes[2], rawSizes[2],
829  &pSrcData[SrcSize]);
830  return FALSE;
831  }
832  }
833  else
834  {
835  if ((SrcSize - diff) < base)
836  {
837  WLog_ERR(TAG, "plane size mismatch %" PRIu32 " < %" PRIu32, SrcSize - diff, base);
838  return FALSE;
839  }
840 
841  planes[0] = srcp; /* LumaOrRedPlane */
842  planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
843  planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
844 
845  if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
846  {
847  WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", planes[2], rawSizes[2],
848  &pSrcData[SrcSize]);
849  return FALSE;
850  }
851  }
852  }
853  else /* RLE */
854  {
855  if (alpha)
856  {
857  planes[3] = srcp;
858  rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
859  rawHeights[3]); /* AlphaPlane */
860 
861  if (rleSizes[3] < 0)
862  return FALSE;
863 
864  planes[0] = planes[3] + rleSizes[3];
865  }
866  else
867  planes[0] = srcp;
868 
869  const size_t diff0 = (planes[0] - pSrcData);
870  if (SrcSize < diff0)
871  {
872  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
873  return FALSE;
874  }
875  rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
876  rawHeights[0]); /* RedPlane */
877 
878  if (rleSizes[0] < 0)
879  return FALSE;
880 
881  planes[1] = planes[0] + rleSizes[0];
882 
883  const size_t diff1 = (planes[1] - pSrcData);
884  if (SrcSize < diff1)
885  {
886  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
887  return FALSE;
888  }
889  rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
890  rawHeights[1]); /* GreenPlane */
891 
892  if (rleSizes[1] < 1)
893  return FALSE;
894 
895  planes[2] = planes[1] + rleSizes[1];
896  const size_t diff2 = (planes[2] - pSrcData);
897  if (SrcSize < diff2)
898  {
899  WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
900  return FALSE;
901  }
902  rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
903  rawHeights[2]); /* BluePlane */
904 
905  if (rleSizes[2] < 1)
906  return FALSE;
907  }
908 
909  if (!cll) /* RGB */
910  {
911  UINT32 TempFormat = 0;
912  BYTE* pTempData = pDstData;
913  UINT32 nTempStep = nDstStep;
914  UINT32 nTotalHeight = nYDst + nDstHeight;
915 
916  if (useAlpha)
917  TempFormat = PIXEL_FORMAT_BGRA32;
918  else
919  TempFormat = PIXEL_FORMAT_BGRX32;
920 
921  TempFormat = planar_invert_format(planar, alpha, TempFormat);
922 
923  if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
924  {
925  pTempData = planar->pTempData;
926  nTempStep = planar->nTempStep;
927  nTotalHeight = planar->maxHeight;
928  }
929 
930  if (!rle) /* RAW */
931  {
932  if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
933  nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
934  return FALSE;
935 
936  if (alpha)
937  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
938  else /* NoAlpha */
939  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
940 
941  if ((SrcSize - (srcp - pSrcData)) == 1)
942  srcp++; /* pad */
943  }
944  else /* RLE */
945  {
946  status =
947  planar_decompress_plane_rle(planes[0], rleSizes[0], pTempData, nTempStep, nXDst,
948  nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
949 
950  if (status < 0)
951  return FALSE;
952 
953  status = planar_decompress_plane_rle(planes[1], rleSizes[1], pTempData, nTempStep,
954  nXDst, nYDst, nSrcWidth, nSrcHeight, 1,
955  vFlip); /* GreenPlane */
956 
957  if (status < 0)
958  return FALSE;
959 
960  status =
961  planar_decompress_plane_rle(planes[2], rleSizes[2], pTempData, nTempStep, nXDst,
962  nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
963 
964  if (status < 0)
965  return FALSE;
966 
967  srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
968 
969  if (useAlpha)
970  {
971  status = planar_decompress_plane_rle(planes[3], rleSizes[3], pTempData, nTempStep,
972  nXDst, nYDst, nSrcWidth, nSrcHeight, 3,
973  vFlip); /* AlphaPlane */
974  }
975  else
976  status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
977  nSrcHeight, 3, vFlip);
978 
979  if (status < 0)
980  return FALSE;
981 
982  if (alpha)
983  srcp += rleSizes[3];
984  }
985 
986  if (pTempData != pDstData)
987  {
988  if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
989  pTempData, TempFormat, nTempStep, nXDst, nYDst, NULL,
990  FREERDP_FLIP_NONE))
991  {
992  WLog_ERR(TAG, "planar image copy failed");
993  return FALSE;
994  }
995  }
996  }
997  else /* YCoCg */
998  {
999  UINT32 TempFormat = 0;
1000  BYTE* pTempData = planar->pTempData;
1001  UINT32 nTempStep = planar->nTempStep;
1002  UINT32 nTotalHeight = planar->maxHeight;
1003  BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
1004 
1005  if (useAlpha)
1006  TempFormat = PIXEL_FORMAT_BGRA32;
1007  else
1008  TempFormat = PIXEL_FORMAT_BGRX32;
1009 
1010  if (!pTempData)
1011  return FALSE;
1012 
1013  if (rle) /* RLE encoded data. Decode and handle it like raw data. */
1014  {
1015  BYTE* rleBuffer[4] = { 0 };
1016 
1017  if (!planar->rlePlanesBuffer)
1018  return FALSE;
1019 
1020  rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1021  rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1022  rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1023  rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1024  if (useAlpha)
1025  {
1026  status =
1027  planar_decompress_plane_rle_only(planes[3], rleSizes[3], rleBuffer[3],
1028  rawWidths[3], rawHeights[3]); /* AlphaPlane */
1029 
1030  if (status < 0)
1031  return FALSE;
1032  }
1033 
1034  if (alpha)
1035  srcp += rleSizes[3];
1036 
1037  status = planar_decompress_plane_rle_only(planes[0], rleSizes[0], rleBuffer[0],
1038  rawWidths[0], rawHeights[0]); /* LumaPlane */
1039 
1040  if (status < 0)
1041  return FALSE;
1042 
1043  status =
1044  planar_decompress_plane_rle_only(planes[1], rleSizes[1], rleBuffer[1], rawWidths[1],
1045  rawHeights[1]); /* OrangeChromaPlane */
1046 
1047  if (status < 0)
1048  return FALSE;
1049 
1050  status =
1051  planar_decompress_plane_rle_only(planes[2], rleSizes[2], rleBuffer[2], rawWidths[2],
1052  rawHeights[2]); /* GreenChromaPlane */
1053 
1054  if (status < 0)
1055  return FALSE;
1056 
1057  planes[0] = rleBuffer[0];
1058  planes[1] = rleBuffer[1];
1059  planes[2] = rleBuffer[2];
1060  planes[3] = rleBuffer[3];
1061  }
1062 
1063  /* RAW */
1064  {
1065  if (cs)
1066  { /* Chroma subsampling for Co and Cg:
1067  * Each pixel contains the value that should be expanded to
1068  * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1069  if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1070  rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1071  return FALSE;
1072 
1073  planes[1] = planar->deltaPlanes[0];
1074  rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1075  rawWidths[1] = nSrcWidth;
1076  rawHeights[1] = nSrcHeight;
1077 
1078  if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1079  rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1080  return FALSE;
1081 
1082  planes[2] = planar->deltaPlanes[1];
1083  rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1084  rawWidths[2] = nSrcWidth;
1085  rawHeights[2] = nSrcHeight;
1086  }
1087 
1088  if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1089  nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1090  return FALSE;
1091 
1092  if (alpha)
1093  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1094  else /* NoAlpha */
1095  srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1096 
1097  if ((SrcSize - (srcp - pSrcData)) == 1)
1098  srcp++; /* pad */
1099  }
1100 
1101  WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1102  int rc = prims->YCoCgToRGB_8u_AC4R(pTempData, nTempStep, dst, DstFormat, nDstStep, w, h,
1103  cll, useAlpha);
1104  if (rc != PRIMITIVES_SUCCESS)
1105  {
1106  WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1107  return FALSE;
1108  }
1109  }
1110 
1111  WINPR_UNUSED(srcp);
1112  return TRUE;
1113 }
1114 
1115 static INLINE BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1116  const BYTE* WINPR_RESTRICT data, UINT32 format,
1117  UINT32 width, UINT32 height, UINT32 scanline,
1118  BYTE* WINPR_RESTRICT planes[4])
1119 {
1120  WINPR_ASSERT(planar);
1121 
1122  if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1123  return FALSE;
1124 
1125  if (scanline == 0)
1126  scanline = width * FreeRDPGetBytesPerPixel(format);
1127 
1128  if (planar->topdown)
1129  {
1130  UINT32 k = 0;
1131  for (UINT32 i = 0; i < height; i++)
1132  {
1133  const BYTE* pixel = &data[1ULL * scanline * i];
1134 
1135  for (UINT32 j = 0; j < width; j++)
1136  {
1137  const UINT32 color = FreeRDPReadColor(pixel, format);
1138  pixel += FreeRDPGetBytesPerPixel(format);
1139  FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1140  &planes[0][k], NULL);
1141  k++;
1142  }
1143  }
1144  }
1145  else
1146  {
1147  UINT32 k = 0;
1148 
1149  for (INT64 i = (INT64)height - 1; i >= 0; i--)
1150  {
1151  const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1152 
1153  for (UINT32 j = 0; j < width; j++)
1154  {
1155  const UINT32 color = FreeRDPReadColor(pixel, format);
1156  pixel += FreeRDPGetBytesPerPixel(format);
1157  FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1158  &planes[0][k], NULL);
1159  k++;
1160  }
1161  }
1162  }
1163  return TRUE;
1164 }
1165 
1166 static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1167  UINT32 cRawBytes, UINT32 nRunLength,
1168  BYTE* WINPR_RESTRICT pOutBuffer,
1169  UINT32 outBufferSize)
1170 {
1171  const BYTE* pInput = NULL;
1172  BYTE* pOutput = NULL;
1173  BYTE controlByte = 0;
1174  UINT32 nBytesToWrite = 0;
1175  pInput = pInBuffer;
1176  pOutput = pOutBuffer;
1177 
1178  if (!cRawBytes && !nRunLength)
1179  return 0;
1180 
1181  if (nRunLength < 3)
1182  {
1183  cRawBytes += nRunLength;
1184  nRunLength = 0;
1185  }
1186 
1187  while (cRawBytes)
1188  {
1189  if (cRawBytes < 16)
1190  {
1191  if (nRunLength > 15)
1192  {
1193  if (nRunLength < 18)
1194  {
1195  controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1196  nRunLength -= 13;
1197  cRawBytes = 0;
1198  }
1199  else
1200  {
1201  controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1202  nRunLength -= 15;
1203  cRawBytes = 0;
1204  }
1205  }
1206  else
1207  {
1208  controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1209  nRunLength = 0;
1210  cRawBytes = 0;
1211  }
1212  }
1213  else
1214  {
1215  controlByte = PLANAR_CONTROL_BYTE(0, 15);
1216  cRawBytes -= 15;
1217  }
1218 
1219  if (outBufferSize < 1)
1220  return 0;
1221 
1222  outBufferSize--;
1223  *pOutput = controlByte;
1224  pOutput++;
1225  nBytesToWrite = (controlByte >> 4);
1226 
1227  if (nBytesToWrite)
1228  {
1229  if (outBufferSize < nBytesToWrite)
1230  return 0;
1231 
1232  outBufferSize -= nBytesToWrite;
1233  CopyMemory(pOutput, pInput, nBytesToWrite);
1234  pOutput += nBytesToWrite;
1235  pInput += nBytesToWrite;
1236  }
1237  }
1238 
1239  while (nRunLength)
1240  {
1241  if (nRunLength > 47)
1242  {
1243  if (nRunLength < 50)
1244  {
1245  controlByte = PLANAR_CONTROL_BYTE(2, 13);
1246  nRunLength -= 45;
1247  }
1248  else
1249  {
1250  controlByte = PLANAR_CONTROL_BYTE(2, 15);
1251  nRunLength -= 47;
1252  }
1253  }
1254  else if (nRunLength > 31)
1255  {
1256  controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1257  nRunLength = 0;
1258  }
1259  else if (nRunLength > 15)
1260  {
1261  controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1262  nRunLength = 0;
1263  }
1264  else
1265  {
1266  controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1267  nRunLength = 0;
1268  }
1269 
1270  if (outBufferSize < 1)
1271  return 0;
1272 
1273  --outBufferSize;
1274  *pOutput = controlByte;
1275  pOutput++;
1276  }
1277 
1278  const intptr_t diff = (pOutput - pOutBuffer);
1279  if ((diff < 0) || (diff > UINT32_MAX))
1280  return 0;
1281  return (UINT32)diff;
1282 }
1283 
1284 static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1285  UINT32 inBufferSize,
1286  BYTE* WINPR_RESTRICT pOutBuffer,
1287  UINT32 outBufferSize)
1288 {
1289  BYTE symbol = 0;
1290  const BYTE* pInput = NULL;
1291  BYTE* pOutput = NULL;
1292  const BYTE* pBytes = NULL;
1293  UINT32 cRawBytes = 0;
1294  UINT32 nRunLength = 0;
1295  UINT32 bSymbolMatch = 0;
1296  UINT32 nBytesWritten = 0;
1297  UINT32 nTotalBytesWritten = 0;
1298  symbol = 0;
1299  cRawBytes = 0;
1300  nRunLength = 0;
1301  pInput = pInBuffer;
1302  pOutput = pOutBuffer;
1303  nTotalBytesWritten = 0;
1304 
1305  if (!outBufferSize)
1306  return 0;
1307 
1308  do
1309  {
1310  if (!inBufferSize)
1311  break;
1312 
1313  bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
1314  symbol = *pInput;
1315  pInput++;
1316  inBufferSize--;
1317 
1318  if (nRunLength && !bSymbolMatch)
1319  {
1320  if (nRunLength < 3)
1321  {
1322  cRawBytes += nRunLength;
1323  nRunLength = 0;
1324  }
1325  else
1326  {
1327  pBytes = pInput - (cRawBytes + nRunLength + 1);
1328  nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1329  pOutput, outBufferSize);
1330  nRunLength = 0;
1331 
1332  if (!nBytesWritten || (nBytesWritten > outBufferSize))
1333  return nRunLength;
1334 
1335  nTotalBytesWritten += nBytesWritten;
1336  outBufferSize -= nBytesWritten;
1337  pOutput += nBytesWritten;
1338  cRawBytes = 0;
1339  }
1340  }
1341 
1342  nRunLength += bSymbolMatch;
1343  cRawBytes += (!bSymbolMatch) ? TRUE : FALSE;
1344  } while (outBufferSize);
1345 
1346  if (cRawBytes || nRunLength)
1347  {
1348  pBytes = pInput - (cRawBytes + nRunLength);
1349  nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1350  pOutput, outBufferSize);
1351 
1352  if (!nBytesWritten)
1353  return 0;
1354 
1355  nTotalBytesWritten += nBytesWritten;
1356  }
1357 
1358  if (inBufferSize)
1359  return 0;
1360 
1361  return nTotalBytesWritten;
1362 }
1363 
1364 BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1365  UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1366  UINT32* WINPR_RESTRICT dstSize)
1367 {
1368  UINT32 index = 0;
1369  const BYTE* pInput = NULL;
1370  BYTE* pOutput = NULL;
1371  UINT32 outBufferSize = 0;
1372  UINT32 nBytesWritten = 0;
1373  UINT32 nTotalBytesWritten = 0;
1374 
1375  if (!outPlane)
1376  return FALSE;
1377 
1378  index = 0;
1379  pInput = inPlane;
1380  pOutput = outPlane;
1381  outBufferSize = *dstSize;
1382  nTotalBytesWritten = 0;
1383 
1384  while (outBufferSize)
1385  {
1386  nBytesWritten =
1387  freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1388 
1389  if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1390  return FALSE;
1391 
1392  outBufferSize -= nBytesWritten;
1393  nTotalBytesWritten += nBytesWritten;
1394  pOutput += nBytesWritten;
1395  pInput += width;
1396  index++;
1397 
1398  if (index >= height)
1399  break;
1400  }
1401 
1402  *dstSize = nTotalBytesWritten;
1403  return TRUE;
1404 }
1405 
1406 static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1407  UINT32 width, UINT32 height,
1408  BYTE* WINPR_RESTRICT outPlanes,
1409  UINT32* WINPR_RESTRICT dstSizes,
1410  BOOL skipAlpha)
1411 {
1412  UINT32 outPlanesSize = width * height * 4;
1413 
1414  /* AlphaPlane */
1415  if (skipAlpha)
1416  {
1417  dstSizes[0] = 0;
1418  }
1419  else
1420  {
1421  dstSizes[0] = outPlanesSize;
1422 
1423  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1424  &dstSizes[0]))
1425  return FALSE;
1426 
1427  outPlanes += dstSizes[0];
1428  outPlanesSize -= dstSizes[0];
1429  }
1430 
1431  /* LumaOrRedPlane */
1432  dstSizes[1] = outPlanesSize;
1433 
1434  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1435  &dstSizes[1]))
1436  return FALSE;
1437 
1438  outPlanes += dstSizes[1];
1439  outPlanesSize -= dstSizes[1];
1440  /* OrangeChromaOrGreenPlane */
1441  dstSizes[2] = outPlanesSize;
1442 
1443  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1444  &dstSizes[2]))
1445  return FALSE;
1446 
1447  outPlanes += dstSizes[2];
1448  outPlanesSize -= dstSizes[2];
1449  /* GreenChromeOrBluePlane */
1450  dstSizes[3] = outPlanesSize;
1451 
1452  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1453  &dstSizes[3]))
1454  return FALSE;
1455 
1456  return TRUE;
1457 }
1458 
1459 BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1460  UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1461 {
1462  char s2c = 0;
1463  BYTE* outPtr = NULL;
1464  const BYTE* srcPtr = NULL;
1465  const BYTE* prevLinePtr = NULL;
1466 
1467  if (!outPlane)
1468  {
1469  if (width * height == 0)
1470  return NULL;
1471 
1472  if (!(outPlane = (BYTE*)calloc(height, width)))
1473  return NULL;
1474  }
1475 
1476  // first line is copied as is
1477  CopyMemory(outPlane, inPlane, width);
1478  outPtr = outPlane + width;
1479  srcPtr = inPlane + width;
1480  prevLinePtr = inPlane;
1481 
1482  for (UINT32 y = 1; y < height; y++)
1483  {
1484  for (UINT32 x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
1485  {
1486  INT32 delta = *srcPtr - *prevLinePtr;
1487  s2c = (delta >= 0) ? (char)delta : (char)(~((BYTE)(-delta)) + 1);
1488  s2c = (s2c >= 0) ? (char)((UINT32)s2c << 1)
1489  : (char)(((UINT32)(~((BYTE)s2c) + 1) << 1) - 1);
1490  *outPtr = (BYTE)s2c;
1491  }
1492  }
1493 
1494  return outPlane;
1495 }
1496 
1497 static INLINE BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1498  UINT32 width, UINT32 height,
1499  BYTE* WINPR_RESTRICT outPlanes[4])
1500 {
1501  for (UINT32 i = 0; i < 4; i++)
1502  {
1503  outPlanes[i] =
1504  freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1505 
1506  if (!outPlanes[i])
1507  return FALSE;
1508  }
1509 
1510  return TRUE;
1511 }
1512 
1513 BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1514  const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1515  UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1516  UINT32* WINPR_RESTRICT pDstSize)
1517 {
1518  UINT32 size = 0;
1519  BYTE* dstp = NULL;
1520  UINT32 planeSize = 0;
1521  UINT32 dstSizes[4] = { 0 };
1522  BYTE FormatHeader = 0;
1523 
1524  if (!context || !context->rlePlanesBuffer)
1525  return NULL;
1526 
1527  if (context->AllowSkipAlpha)
1528  FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1529 
1530  planeSize = width * height;
1531 
1532  if (!context->AllowSkipAlpha)
1533  format = planar_invert_format(context, TRUE, format);
1534 
1535  if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1536  context->planes))
1537  return NULL;
1538 
1539  if (context->AllowRunLengthEncoding)
1540  {
1541  if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1542  context->deltaPlanes))
1543  return NULL;
1544 
1545  if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1546  context->rlePlanesBuffer, dstSizes,
1547  context->AllowSkipAlpha))
1548  return NULL;
1549 
1550  {
1551  int offset = 0;
1552  FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1553  context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1554  offset += dstSizes[0];
1555  context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1556  offset += dstSizes[1];
1557  context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1558  offset += dstSizes[2];
1559  context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1560 
1561 #if defined(WITH_DEBUG_CODECS)
1562  WLog_DBG(TAG,
1563  "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1564  " / %" PRIu32 "] ",
1565  dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1566 #endif
1567  }
1568  }
1569 
1570  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1571  {
1572  if (!context->AllowRunLengthEncoding)
1573  return NULL;
1574 
1575  if (context->rlePlanes[0] == NULL)
1576  return NULL;
1577 
1578  if (context->rlePlanes[1] == NULL)
1579  return NULL;
1580 
1581  if (context->rlePlanes[2] == NULL)
1582  return NULL;
1583 
1584  if (context->rlePlanes[3] == NULL)
1585  return NULL;
1586  }
1587 
1588  if (!dstData)
1589  {
1590  size = 1;
1591 
1592  if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1593  {
1594  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1595  size += dstSizes[0];
1596  else
1597  size += planeSize;
1598  }
1599 
1600  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1601  size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1602  else
1603  size += (planeSize * 3);
1604 
1605  if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1606  size++;
1607 
1608  dstData = malloc(size);
1609 
1610  if (!dstData)
1611  return NULL;
1612 
1613  *pDstSize = size;
1614  }
1615 
1616  dstp = dstData;
1617  *dstp = FormatHeader; /* FormatHeader */
1618  dstp++;
1619 
1620  /* AlphaPlane */
1621 
1622  if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1623  {
1624  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1625  {
1626  CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1627  dstp += dstSizes[0];
1628  }
1629  else
1630  {
1631  CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1632  dstp += planeSize;
1633  }
1634  }
1635 
1636  /* LumaOrRedPlane */
1637 
1638  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1639  {
1640  CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1641  dstp += dstSizes[1];
1642  }
1643  else
1644  {
1645  CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1646  dstp += planeSize;
1647  }
1648 
1649  /* OrangeChromaOrGreenPlane */
1650 
1651  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1652  {
1653  CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1654  dstp += dstSizes[2];
1655  }
1656  else
1657  {
1658  CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1659  dstp += planeSize;
1660  }
1661 
1662  /* GreenChromeOrBluePlane */
1663 
1664  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1665  {
1666  CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1667  dstp += dstSizes[3];
1668  }
1669  else
1670  {
1671  CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1672  dstp += planeSize;
1673  }
1674 
1675  /* Pad1 (1 byte) */
1676 
1677  if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1678  {
1679  *dstp = 0;
1680  dstp++;
1681  }
1682 
1683  const intptr_t diff = (dstp - dstData);
1684  if ((diff < 0) || (diff > UINT32_MAX))
1685  {
1686  free(dstData);
1687  return NULL;
1688  }
1689  size = (UINT32)diff;
1690  *pDstSize = size;
1691  return dstData;
1692 }
1693 
1694 BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1695  UINT32 width, UINT32 height)
1696 {
1697  if (!context)
1698  return FALSE;
1699 
1700  context->bgr = FALSE;
1701  context->maxWidth = PLANAR_ALIGN(width, 4);
1702  context->maxHeight = PLANAR_ALIGN(height, 4);
1703  {
1704  const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1705  if (tmp > UINT32_MAX)
1706  return FALSE;
1707  context->maxPlaneSize = (UINT32)tmp;
1708  }
1709 
1710  if (context->maxWidth > UINT32_MAX / 4)
1711  return FALSE;
1712  context->nTempStep = context->maxWidth * 4;
1713 
1714  memset(context->planes, 0, sizeof(context->planes));
1715  memset(context->rlePlanes, 0, sizeof(context->rlePlanes));
1716  memset(context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1717 
1718  if (context->maxPlaneSize > 0)
1719  {
1720  void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1721  if (!tmp)
1722  return FALSE;
1723  context->planesBuffer = tmp;
1724 
1725  tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1726  if (!tmp)
1727  return FALSE;
1728  context->pTempData = tmp;
1729 
1730  tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1731  if (!tmp)
1732  return FALSE;
1733  context->deltaPlanesBuffer = tmp;
1734 
1735  tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1736  if (!tmp)
1737  return FALSE;
1738  context->rlePlanesBuffer = tmp;
1739 
1740  context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1741  context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1742  context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1743  context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1744  context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1745  context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1746  context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1747  context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1748  }
1749  return TRUE;
1750 }
1751 
1752 BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1753  UINT32 maxHeight)
1754 {
1755  BITMAP_PLANAR_CONTEXT* context =
1756  (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1757 
1758  if (!context)
1759  return NULL;
1760 
1761  if (flags & PLANAR_FORMAT_HEADER_NA)
1762  context->AllowSkipAlpha = TRUE;
1763 
1764  if (flags & PLANAR_FORMAT_HEADER_RLE)
1765  context->AllowRunLengthEncoding = TRUE;
1766 
1767  if (flags & PLANAR_FORMAT_HEADER_CS)
1768  context->AllowColorSubsampling = TRUE;
1769 
1770  context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1771 
1772  if (context->ColorLossLevel)
1773  context->AllowDynamicColorFidelity = TRUE;
1774 
1775  if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1776  {
1777  WINPR_PRAGMA_DIAG_PUSH
1778  WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1779  freerdp_bitmap_planar_context_free(context);
1780  WINPR_PRAGMA_DIAG_POP
1781  return NULL;
1782  }
1783 
1784  return context;
1785 }
1786 
1787 void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1788 {
1789  if (!context)
1790  return;
1791 
1792  winpr_aligned_free(context->pTempData);
1793  winpr_aligned_free(context->planesBuffer);
1794  winpr_aligned_free(context->deltaPlanesBuffer);
1795  winpr_aligned_free(context->rlePlanesBuffer);
1796  winpr_aligned_free(context);
1797 }
1798 
1799 void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1800 {
1801  WINPR_ASSERT(planar);
1802  planar->bgr = bgr;
1803 }
1804 
1805 void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1806 {
1807  WINPR_ASSERT(planar);
1808  planar->topdown = topdown;
1809 }