FreeRDP
Loading...
Searching...
No Matches
planar.c
1
22#include <freerdp/config.h>
23
24#include <winpr/crt.h>
25#include <winpr/wtypes.h>
26#include <winpr/assert.h>
27#include <winpr/cast.h>
28#include <winpr/print.h>
29
30#include <freerdp/primitives.h>
31#include <freerdp/log.h>
32#include <freerdp/codec/bitmap.h>
33#include <freerdp/codec/planar.h>
34
35#define TAG FREERDP_TAG("codec")
36
37#define PLANAR_ALIGN(val, align) \
38 ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
39
40typedef struct
41{
47 BYTE controlByte;
48 BYTE* rawValues;
49} RDP6_RLE_SEGMENT;
50
51typedef struct
52{
53 UINT32 cSegments;
54 RDP6_RLE_SEGMENT* segments;
55} RDP6_RLE_SEGMENTS;
56
57typedef struct
58{
67 BYTE formatHeader;
68} RDP6_BITMAP_STREAM;
69
70struct S_BITMAP_PLANAR_CONTEXT
71{
72 UINT32 maxWidth;
73 UINT32 maxHeight;
74 UINT32 maxPlaneSize;
75
76 BOOL AllowSkipAlpha;
77 BOOL AllowRunLengthEncoding;
78 BOOL AllowColorSubsampling;
79 BOOL AllowDynamicColorFidelity;
80
81 UINT32 ColorLossLevel;
82
83 BYTE* planes[4];
84 BYTE* planesBuffer;
85
86 BYTE* deltaPlanes[4];
87 BYTE* deltaPlanesBuffer;
88
89 BYTE* rlePlanes[4];
90 BYTE* rlePlanesBuffer;
91
92 BYTE* pTempData;
93 UINT32 nTempStep;
94
95 BOOL bgr;
96 BOOL topdown;
97};
98
99static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
100 UINT32 DstFormat)
101{
102
103 if (planar->bgr && alpha)
104 {
105 switch (DstFormat)
106 {
107 case PIXEL_FORMAT_ARGB32:
108 DstFormat = PIXEL_FORMAT_ABGR32;
109 break;
110 case PIXEL_FORMAT_XRGB32:
111 DstFormat = PIXEL_FORMAT_XBGR32;
112 break;
113 case PIXEL_FORMAT_ABGR32:
114 DstFormat = PIXEL_FORMAT_ARGB32;
115 break;
116 case PIXEL_FORMAT_XBGR32:
117 DstFormat = PIXEL_FORMAT_XRGB32;
118 break;
119 case PIXEL_FORMAT_BGRA32:
120 DstFormat = PIXEL_FORMAT_RGBA32;
121 break;
122 case PIXEL_FORMAT_BGRX32:
123 DstFormat = PIXEL_FORMAT_RGBX32;
124 break;
125 case PIXEL_FORMAT_RGBA32:
126 DstFormat = PIXEL_FORMAT_BGRA32;
127 break;
128 case PIXEL_FORMAT_RGBX32:
129 DstFormat = PIXEL_FORMAT_BGRX32;
130 break;
131 case PIXEL_FORMAT_RGB24:
132 DstFormat = PIXEL_FORMAT_BGR24;
133 break;
134 case PIXEL_FORMAT_BGR24:
135 DstFormat = PIXEL_FORMAT_RGB24;
136 break;
137 case PIXEL_FORMAT_RGB16:
138 DstFormat = PIXEL_FORMAT_BGR16;
139 break;
140 case PIXEL_FORMAT_BGR16:
141 DstFormat = PIXEL_FORMAT_RGB16;
142 break;
143 case PIXEL_FORMAT_ARGB15:
144 DstFormat = PIXEL_FORMAT_ABGR15;
145 break;
146 case PIXEL_FORMAT_RGB15:
147 DstFormat = PIXEL_FORMAT_BGR15;
148 break;
149 case PIXEL_FORMAT_ABGR15:
150 DstFormat = PIXEL_FORMAT_ARGB15;
151 break;
152 case PIXEL_FORMAT_BGR15:
153 DstFormat = PIXEL_FORMAT_RGB15;
154 break;
155 default:
156 break;
157 }
158 }
159 return DstFormat;
160}
161
162static INLINE BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
163 UINT32 width, UINT32 height,
164 BYTE* WINPR_RESTRICT outPlane,
165 UINT32* WINPR_RESTRICT dstSize);
166static INLINE BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
167 UINT32 width, UINT32 height,
168 BYTE* WINPR_RESTRICT outPlane);
169
170static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
171 UINT32 nWidth, UINT32 nHeight)
172{
173 UINT32 used = 0;
174
175 WINPR_ASSERT(pSrcData);
176
177 for (UINT32 y = 0; y < nHeight; y++)
178 {
179 for (UINT32 x = 0; x < nWidth;)
180 {
181 if (used >= SrcSize)
182 {
183 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
184 SrcSize);
185 return -1;
186 }
187
188 const uint8_t controlByte = pSrcData[used++];
189 uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
190 uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
191
192 if (nRunLength == 1)
193 {
194 nRunLength = cRawBytes + 16;
195 cRawBytes = 0;
196 }
197 else if (nRunLength == 2)
198 {
199 nRunLength = cRawBytes + 32;
200 cRawBytes = 0;
201 }
202
203 used += cRawBytes;
204 x += cRawBytes;
205 x += nRunLength;
206
207 if (x > nWidth)
208 {
209 WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
210 return -1;
211 }
212
213 if (used > SrcSize)
214 {
215 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
216 INT32_MAX);
217 return -1;
218 }
219 }
220 }
221
222 if (used > INT32_MAX)
223 {
224 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
225 return -1;
226 }
227 return (INT32)used;
228}
229
230static inline UINT8 clamp(INT16 val)
231{
232 return (UINT8)val;
233}
234
235static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
236 UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
237 UINT32 nWidth, UINT32 nHeight)
238{
239 BYTE* previousScanline = NULL;
240 const BYTE* srcp = pSrcData;
241
242 WINPR_ASSERT(nHeight <= INT32_MAX);
243 WINPR_ASSERT(nWidth <= INT32_MAX);
244
245 for (UINT32 y = 0; y < nHeight; y++)
246 {
247 BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
248 INT16 pixel = 0;
249 BYTE* currentScanline = dstp;
250
251 for (UINT32 x = 0; x < nWidth;)
252 {
253 BYTE controlByte = *srcp;
254 srcp++;
255
256 if ((srcp - pSrcData) > SrcSize * 1ll)
257 {
258 WLog_ERR(TAG, "error reading input buffer");
259 return -1;
260 }
261
262 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
263 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
264
265 if (nRunLength == 1)
266 {
267 nRunLength = cRawBytes + 16;
268 cRawBytes = 0;
269 }
270 else if (nRunLength == 2)
271 {
272 nRunLength = cRawBytes + 32;
273 cRawBytes = 0;
274 }
275
276 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
277 {
278 WLog_ERR(TAG, "too many pixels in scanline");
279 return -1;
280 }
281
282 if (!previousScanline)
283 {
284 /* first scanline, absolute values */
285 while (cRawBytes > 0)
286 {
287 pixel = *srcp;
288 srcp++;
289 *dstp = clamp(pixel);
290 dstp++;
291 x++;
292 cRawBytes--;
293 }
294
295 while (nRunLength > 0)
296 {
297 *dstp = clamp(pixel);
298 dstp++;
299 x++;
300 nRunLength--;
301 }
302 }
303 else
304 {
305 /* delta values relative to previous scanline */
306 while (cRawBytes > 0)
307 {
308 UINT8 deltaValue = *srcp;
309 srcp++;
310
311 if (deltaValue & 1)
312 {
313 deltaValue = deltaValue >> 1;
314 deltaValue = deltaValue + 1;
315 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
316 }
317 else
318 {
319 deltaValue = deltaValue >> 1;
320 pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
321 }
322
323 const INT16 delta =
324 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
325 *dstp = clamp(delta);
326 dstp++;
327 x++;
328 cRawBytes--;
329 }
330
331 while (nRunLength > 0)
332 {
333 const INT16 deltaValue =
334 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
335 *dstp = clamp(deltaValue);
336 dstp++;
337 x++;
338 nRunLength--;
339 }
340 }
341 }
342
343 previousScanline = currentScanline;
344 }
345
346 return (INT32)(srcp - pSrcData);
347}
348
349static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
350 BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
351 UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
352 UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
353{
354 INT32 beg = 0;
355 INT32 end = 0;
356 INT32 inc = 0;
357 BYTE* previousScanline = NULL;
358 const BYTE* srcp = pSrcData;
359
360 WINPR_ASSERT(nHeight <= INT32_MAX);
361 WINPR_ASSERT(nWidth <= INT32_MAX);
362 WINPR_ASSERT(nDstStep <= INT32_MAX);
363
364 previousScanline = NULL;
365
366 if (vFlip)
367 {
368 beg = (INT32)nHeight - 1;
369 end = -1;
370 inc = -1;
371 }
372 else
373 {
374 beg = 0;
375 end = (INT32)nHeight;
376 inc = 1;
377 }
378
379 for (INT32 y = beg; y != end; y += inc)
380 {
381 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
382 BYTE* dstp = &pDstData[off];
383 BYTE* currentScanline = dstp;
384 INT16 pixel = 0;
385
386 for (INT32 x = 0; x < (INT32)nWidth;)
387 {
388 const BYTE controlByte = *srcp;
389 srcp++;
390
391 if ((srcp - pSrcData) > SrcSize * 1ll)
392 {
393 WLog_ERR(TAG, "error reading input buffer");
394 return -1;
395 }
396
397 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
398 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
399
400 if (nRunLength == 1)
401 {
402 nRunLength = cRawBytes + 16;
403 cRawBytes = 0;
404 }
405 else if (nRunLength == 2)
406 {
407 nRunLength = cRawBytes + 32;
408 cRawBytes = 0;
409 }
410
411 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
412 {
413 WLog_ERR(TAG, "too many pixels in scanline");
414 return -1;
415 }
416
417 if (!previousScanline)
418 {
419 /* first scanline, absolute values */
420 while (cRawBytes > 0)
421 {
422 pixel = *srcp;
423 srcp++;
424 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
425 dstp += 4;
426 x++;
427 cRawBytes--;
428 }
429
430 while (nRunLength > 0)
431 {
432 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
433 dstp += 4;
434 x++;
435 nRunLength--;
436 }
437 }
438 else
439 {
440 /* delta values relative to previous scanline */
441 while (cRawBytes > 0)
442 {
443 BYTE deltaValue = *srcp;
444 srcp++;
445
446 if (deltaValue & 1)
447 {
448 deltaValue = deltaValue >> 1;
449 deltaValue = deltaValue + 1;
450 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
451 }
452 else
453 {
454 deltaValue = deltaValue >> 1;
455 pixel = deltaValue;
456 }
457
458 const INT16 delta =
459 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
460 *dstp = clamp(delta);
461 dstp += 4;
462 x++;
463 cRawBytes--;
464 }
465
466 while (nRunLength > 0)
467 {
468 const INT16 deltaValue =
469 WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
470 *dstp = clamp(deltaValue);
471 dstp += 4;
472 x++;
473 nRunLength--;
474 }
475 }
476 }
477
478 previousScanline = currentScanline;
479 }
480
481 return (INT32)(srcp - pSrcData);
482}
483
484static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
485 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
486 BOOL vFlip)
487{
488 INT32 beg = 0;
489 INT32 end = 0;
490 INT32 inc = 0;
491
492 WINPR_ASSERT(nHeight <= INT32_MAX);
493 WINPR_ASSERT(nWidth <= INT32_MAX);
494 WINPR_ASSERT(nDstStep <= INT32_MAX);
495
496 if (vFlip)
497 {
498 beg = (INT32)nHeight - 1;
499 end = -1;
500 inc = -1;
501 }
502 else
503 {
504 beg = 0;
505 end = (INT32)nHeight;
506 inc = 1;
507 }
508
509 for (INT32 y = beg; y != end; y += inc)
510 {
511 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
512 BYTE* dstp = &pDstData[off];
513
514 for (INT32 x = 0; x < (INT32)nWidth; ++x)
515 {
516 *dstp = bValue;
517 dstp += 4;
518 }
519 }
520
521 return 0;
522}
523
524static INLINE BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
525 const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
526 const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
527{
528 WINPR_ASSERT(ppRgba);
529 WINPR_ASSERT(ppR);
530 WINPR_ASSERT(ppG);
531 WINPR_ASSERT(ppB);
532
533 switch (DstFormat)
534 {
535 case PIXEL_FORMAT_BGRA32:
536 for (UINT32 x = 0; x < width; x++)
537 {
538 *(*ppRgba)++ = *(*ppB)++;
539 *(*ppRgba)++ = *(*ppG)++;
540 *(*ppRgba)++ = *(*ppR)++;
541 *(*ppRgba)++ = *(*ppA)++;
542 }
543
544 return TRUE;
545
546 case PIXEL_FORMAT_BGRX32:
547 for (UINT32 x = 0; x < width; x++)
548 {
549 *(*ppRgba)++ = *(*ppB)++;
550 *(*ppRgba)++ = *(*ppG)++;
551 *(*ppRgba)++ = *(*ppR)++;
552 *(*ppRgba)++ = 0xFF;
553 }
554
555 return TRUE;
556
557 default:
558 if (ppA)
559 {
560 for (UINT32 x = 0; x < width; x++)
561 {
562 BYTE alpha = *(*ppA)++;
563 UINT32 color =
564 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
565 FreeRDPWriteColor(*ppRgba, DstFormat, color);
566 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
567 }
568 }
569 else
570 {
571 const BYTE alpha = 0xFF;
572
573 for (UINT32 x = 0; x < width; x++)
574 {
575 UINT32 color =
576 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
577 FreeRDPWriteColor(*ppRgba, DstFormat, color);
578 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
579 }
580 }
581
582 return TRUE;
583 }
584}
585
586static INLINE BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
587 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
588 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
589 UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
590 UINT32 totalHeight)
591{
592 INT32 beg = 0;
593 INT32 end = 0;
594 INT32 inc = 0;
595 const BYTE* pR = pSrcData[0];
596 const BYTE* pG = pSrcData[1];
597 const BYTE* pB = pSrcData[2];
598 const BYTE* pA = pSrcData[3];
599 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
600
601 if (vFlip)
602 {
603 beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
604 end = -1;
605 inc = -1;
606 }
607 else
608 {
609 beg = 0;
610 end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
611 inc = 1;
612 }
613
614 if (nYDst + nHeight > totalHeight)
615 {
616 WLog_ERR(TAG,
617 "planar plane destination Y %" PRIu32 " + height %" PRIu32
618 " exceeds totalHeight %" PRIu32,
619 nYDst, nHeight, totalHeight);
620 return FALSE;
621 }
622
623 if ((nXDst + nWidth) * bpp > nDstStep)
624 {
625 WLog_ERR(TAG,
626 "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
627 " exceeds stride %" PRIu32,
628 nXDst, nWidth, bpp, nDstStep);
629 return FALSE;
630 }
631
632 for (INT32 y = beg; y != end; y += inc)
633 {
634 BYTE* pRGB = NULL;
635
636 if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
637 {
638 WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
639 nHeight);
640 return FALSE;
641 }
642
643 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
644 pRGB = &pDstData[off];
645
646 if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
647 return FALSE;
648 }
649
650 return TRUE;
651}
652
653static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
654 UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
655 UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
656{
657 size_t pos = 0;
658 WINPR_UNUSED(planeLength);
659
660 WINPR_ASSERT(plane);
661 WINPR_ASSERT(deltaPlane);
662
663 if (nWidth > nPlaneWidth * 2)
664 {
665 WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
666 nPlaneWidth);
667 return FALSE;
668 }
669
670 if (nHeight > nPlaneHeight * 2)
671 {
672 WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
673 nPlaneHeight);
674 return FALSE;
675 }
676
677 for (size_t y = 0; y < nHeight; y++)
678 {
679 const BYTE* src = plane + y / 2 * nPlaneWidth;
680
681 for (UINT32 x = 0; x < nWidth; x++)
682 {
683 deltaPlane[pos++] = src[x / 2];
684 }
685 }
686
687 return TRUE;
688}
689
690BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
691 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
692 UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
693 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
694 UINT32 nDstHeight, BOOL vFlip)
695{
696 BOOL useAlpha = FALSE;
697 INT32 status = 0;
698 INT32 rleSizes[4] = { 0, 0, 0, 0 };
699 UINT32 rawSizes[4] = { 0 };
700 UINT32 rawWidths[4] = { 0 };
701 UINT32 rawHeights[4] = { 0 };
702 const BYTE* planes[4] = { 0 };
703 const UINT32 w = MIN(nSrcWidth, nDstWidth);
704 const UINT32 h = MIN(nSrcHeight, nDstHeight);
705 const primitives_t* prims = primitives_get();
706
707 WINPR_ASSERT(planar);
708 WINPR_ASSERT(prims);
709
710 if (nDstStep <= 0)
711 nDstStep = nDstWidth * FreeRDPGetBytesPerPixel(DstFormat);
712
713 const BYTE* srcp = pSrcData;
714
715 if (!pSrcData)
716 {
717 WLog_ERR(TAG, "Invalid argument pSrcData=NULL");
718 return FALSE;
719 }
720
721 if (!pDstData)
722 {
723 WLog_ERR(TAG, "Invalid argument pDstData=NULL");
724 return FALSE;
725 }
726
727 const BYTE FormatHeader = *srcp++;
728 const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
729 const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE;
730 const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE;
731 const BYTE alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE;
732
733 DstFormat = planar_invert_format(planar, alpha, DstFormat);
734
735 if (alpha)
736 useAlpha = FreeRDPColorHasAlpha(DstFormat);
737
738 // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
739 // alpha);
740
741 if (!cll && cs)
742 {
743 WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
744 return FALSE; /* Chroma subsampling requires YCoCg */
745 }
746
747 const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
748 const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
749 const UINT32 planeSize = nSrcWidth * nSrcHeight;
750 const UINT32 subSize = subWidth * subHeight;
751
752 if (!cs)
753 {
754 rawSizes[0] = planeSize; /* LumaOrRedPlane */
755 rawWidths[0] = nSrcWidth;
756 rawHeights[0] = nSrcHeight;
757 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
758 rawWidths[1] = nSrcWidth;
759 rawHeights[1] = nSrcHeight;
760 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
761 rawWidths[2] = nSrcWidth;
762 rawHeights[2] = nSrcHeight;
763 rawSizes[3] = planeSize; /* AlphaPlane */
764 rawWidths[3] = nSrcWidth;
765 rawHeights[3] = nSrcHeight;
766 }
767 else /* Chroma Subsampling */
768 {
769 rawSizes[0] = planeSize; /* LumaOrRedPlane */
770 rawWidths[0] = nSrcWidth;
771 rawHeights[0] = nSrcHeight;
772 rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
773 rawWidths[1] = subWidth;
774 rawHeights[1] = subHeight;
775 rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
776 rawWidths[2] = subWidth;
777 rawHeights[2] = subHeight;
778 rawSizes[3] = planeSize; /* AlphaPlane */
779 rawWidths[3] = nSrcWidth;
780 rawHeights[3] = nSrcHeight;
781 }
782
783 const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
784 if (SrcSize < diff)
785 {
786 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
787 return FALSE;
788 }
789
790 if (!rle) /* RAW */
791 {
792
793 UINT32 base = planeSize * 3;
794 if (cs)
795 base = planeSize + planeSize / 2;
796
797 if (alpha)
798 {
799 if ((SrcSize - diff) < (planeSize + base))
800 {
801 WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
802 (planeSize + base));
803 return FALSE;
804 }
805
806 planes[3] = srcp; /* AlphaPlane */
807 planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
808 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
809 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
810
811 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
812 {
813 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", planes[2], rawSizes[2],
814 &pSrcData[SrcSize]);
815 return FALSE;
816 }
817 }
818 else
819 {
820 if ((SrcSize - diff) < base)
821 {
822 WLog_ERR(TAG, "plane size mismatch %" PRIu32 " < %" PRIu32, SrcSize - diff, base);
823 return FALSE;
824 }
825
826 planes[0] = srcp; /* LumaOrRedPlane */
827 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
828 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
829
830 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
831 {
832 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", planes[2], rawSizes[2],
833 &pSrcData[SrcSize]);
834 return FALSE;
835 }
836 }
837 }
838 else /* RLE */
839 {
840 if (alpha)
841 {
842 planes[3] = srcp;
843 rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
844 rawHeights[3]); /* AlphaPlane */
845
846 if (rleSizes[3] < 0)
847 return FALSE;
848
849 planes[0] = planes[3] + rleSizes[3];
850 }
851 else
852 planes[0] = srcp;
853
854 const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
855 if (SrcSize < diff0)
856 {
857 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
858 return FALSE;
859 }
860 rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
861 rawHeights[0]); /* RedPlane */
862
863 if (rleSizes[0] < 0)
864 return FALSE;
865
866 planes[1] = planes[0] + rleSizes[0];
867
868 const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
869 if (SrcSize < diff1)
870 {
871 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
872 return FALSE;
873 }
874 rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
875 rawHeights[1]); /* GreenPlane */
876
877 if (rleSizes[1] < 1)
878 return FALSE;
879
880 planes[2] = planes[1] + rleSizes[1];
881 const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
882 if (SrcSize < diff2)
883 {
884 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
885 return FALSE;
886 }
887 rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
888 rawHeights[2]); /* BluePlane */
889
890 if (rleSizes[2] < 1)
891 return FALSE;
892 }
893
894 if (!cll) /* RGB */
895 {
896 UINT32 TempFormat = 0;
897 BYTE* pTempData = pDstData;
898 UINT32 nTempStep = nDstStep;
899 UINT32 nTotalHeight = nYDst + nDstHeight;
900
901 if (useAlpha)
902 TempFormat = PIXEL_FORMAT_BGRA32;
903 else
904 TempFormat = PIXEL_FORMAT_BGRX32;
905
906 TempFormat = planar_invert_format(planar, alpha, TempFormat);
907
908 if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
909 {
910 pTempData = planar->pTempData;
911 nTempStep = planar->nTempStep;
912 nTotalHeight = planar->maxHeight;
913 }
914
915 if (!rle) /* RAW */
916 {
917 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
918 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
919 return FALSE;
920
921 if (alpha)
922 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
923 else /* NoAlpha */
924 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
925
926 if ((SrcSize - (srcp - pSrcData)) == 1)
927 srcp++; /* pad */
928 }
929 else /* RLE */
930 {
931 status = planar_decompress_plane_rle(
932 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
933 nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
934
935 if (status < 0)
936 return FALSE;
937
938 status = planar_decompress_plane_rle(
939 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
940 nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */
941
942 if (status < 0)
943 return FALSE;
944
945 status = planar_decompress_plane_rle(
946 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
947 nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
948
949 if (status < 0)
950 return FALSE;
951
952 srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
953
954 if (useAlpha)
955 {
956 status = planar_decompress_plane_rle(
957 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
958 nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
959 }
960 else
961 status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
962 nSrcHeight, 3, vFlip);
963
964 if (status < 0)
965 return FALSE;
966
967 if (alpha)
968 srcp += rleSizes[3];
969 }
970
971 if (pTempData != pDstData)
972 {
973 if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
974 pTempData, TempFormat, nTempStep, nXDst, nYDst, NULL,
975 FREERDP_FLIP_NONE))
976 {
977 WLog_ERR(TAG, "planar image copy failed");
978 return FALSE;
979 }
980 }
981 }
982 else /* YCoCg */
983 {
984 UINT32 TempFormat = 0;
985 BYTE* pTempData = planar->pTempData;
986 UINT32 nTempStep = planar->nTempStep;
987 UINT32 nTotalHeight = planar->maxHeight;
988 BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
989
990 if (useAlpha)
991 TempFormat = PIXEL_FORMAT_BGRA32;
992 else
993 TempFormat = PIXEL_FORMAT_BGRX32;
994
995 if (!pTempData)
996 return FALSE;
997
998 if (rle) /* RLE encoded data. Decode and handle it like raw data. */
999 {
1000 BYTE* rleBuffer[4] = { 0 };
1001
1002 if (!planar->rlePlanesBuffer)
1003 return FALSE;
1004
1005 rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1006 rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1007 rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1008 rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1009 if (useAlpha)
1010 {
1011 status = planar_decompress_plane_rle_only(
1012 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
1013 rawWidths[3], rawHeights[3]); /* AlphaPlane */
1014
1015 if (status < 0)
1016 return FALSE;
1017 }
1018
1019 if (alpha)
1020 srcp += rleSizes[3];
1021
1022 status = planar_decompress_plane_rle_only(
1023 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
1024 rawWidths[0], rawHeights[0]); /* LumaPlane */
1025
1026 if (status < 0)
1027 return FALSE;
1028
1029 status = planar_decompress_plane_rle_only(
1030 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
1031 rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */
1032
1033 if (status < 0)
1034 return FALSE;
1035
1036 status = planar_decompress_plane_rle_only(
1037 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
1038 rawWidths[2], rawHeights[2]); /* GreenChromaPlane */
1039
1040 if (status < 0)
1041 return FALSE;
1042
1043 planes[0] = rleBuffer[0];
1044 planes[1] = rleBuffer[1];
1045 planes[2] = rleBuffer[2];
1046 planes[3] = rleBuffer[3];
1047 }
1048
1049 /* RAW */
1050 {
1051 if (cs)
1052 { /* Chroma subsampling for Co and Cg:
1053 * Each pixel contains the value that should be expanded to
1054 * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1055 if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1056 rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1057 return FALSE;
1058
1059 planes[1] = planar->deltaPlanes[0];
1060 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1061 rawWidths[1] = nSrcWidth;
1062 rawHeights[1] = nSrcHeight;
1063
1064 if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1065 rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1066 return FALSE;
1067
1068 planes[2] = planar->deltaPlanes[1];
1069 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1070 rawWidths[2] = nSrcWidth;
1071 rawHeights[2] = nSrcHeight;
1072 }
1073
1074 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1075 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1076 return FALSE;
1077
1078 if (alpha)
1079 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1080 else /* NoAlpha */
1081 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1082
1083 if ((SrcSize - (srcp - pSrcData)) == 1)
1084 srcp++; /* pad */
1085 }
1086
1087 WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1088 int rc = prims->YCoCgToRGB_8u_AC4R(
1089 pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
1090 WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
1091 if (rc != PRIMITIVES_SUCCESS)
1092 {
1093 WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1094 return FALSE;
1095 }
1096 }
1097
1098 WINPR_UNUSED(srcp);
1099 return TRUE;
1100}
1101
1102static INLINE BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1103 const BYTE* WINPR_RESTRICT data, UINT32 format,
1104 UINT32 width, UINT32 height, UINT32 scanline,
1105 BYTE* WINPR_RESTRICT planes[4])
1106{
1107 WINPR_ASSERT(planar);
1108
1109 if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1110 return FALSE;
1111
1112 if (scanline == 0)
1113 scanline = width * FreeRDPGetBytesPerPixel(format);
1114
1115 if (planar->topdown)
1116 {
1117 UINT32 k = 0;
1118 for (UINT32 i = 0; i < height; i++)
1119 {
1120 const BYTE* pixel = &data[1ULL * scanline * i];
1121
1122 for (UINT32 j = 0; j < width; j++)
1123 {
1124 const UINT32 color = FreeRDPReadColor(pixel, format);
1125 pixel += FreeRDPGetBytesPerPixel(format);
1126 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1127 &planes[0][k], NULL);
1128 k++;
1129 }
1130 }
1131 }
1132 else
1133 {
1134 UINT32 k = 0;
1135
1136 for (INT64 i = (INT64)height - 1; i >= 0; i--)
1137 {
1138 const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1139
1140 for (UINT32 j = 0; j < width; j++)
1141 {
1142 const UINT32 color = FreeRDPReadColor(pixel, format);
1143 pixel += FreeRDPGetBytesPerPixel(format);
1144 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1145 &planes[0][k], NULL);
1146 k++;
1147 }
1148 }
1149 }
1150 return TRUE;
1151}
1152
1153static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1154 UINT32 cRawBytes, UINT32 nRunLength,
1155 BYTE* WINPR_RESTRICT pOutBuffer,
1156 UINT32 outBufferSize)
1157{
1158 const BYTE* pInput = NULL;
1159 BYTE* pOutput = NULL;
1160 BYTE controlByte = 0;
1161 UINT32 nBytesToWrite = 0;
1162 pInput = pInBuffer;
1163 pOutput = pOutBuffer;
1164
1165 if (!cRawBytes && !nRunLength)
1166 return 0;
1167
1168 if (nRunLength < 3)
1169 {
1170 cRawBytes += nRunLength;
1171 nRunLength = 0;
1172 }
1173
1174 while (cRawBytes)
1175 {
1176 if (cRawBytes < 16)
1177 {
1178 if (nRunLength > 15)
1179 {
1180 if (nRunLength < 18)
1181 {
1182 controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1183 nRunLength -= 13;
1184 cRawBytes = 0;
1185 }
1186 else
1187 {
1188 controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1189 nRunLength -= 15;
1190 cRawBytes = 0;
1191 }
1192 }
1193 else
1194 {
1195 controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1196 nRunLength = 0;
1197 cRawBytes = 0;
1198 }
1199 }
1200 else
1201 {
1202 controlByte = PLANAR_CONTROL_BYTE(0, 15);
1203 cRawBytes -= 15;
1204 }
1205
1206 if (outBufferSize < 1)
1207 return 0;
1208
1209 outBufferSize--;
1210 *pOutput = controlByte;
1211 pOutput++;
1212 nBytesToWrite = (controlByte >> 4);
1213
1214 if (nBytesToWrite)
1215 {
1216 if (outBufferSize < nBytesToWrite)
1217 return 0;
1218
1219 outBufferSize -= nBytesToWrite;
1220 CopyMemory(pOutput, pInput, nBytesToWrite);
1221 pOutput += nBytesToWrite;
1222 pInput += nBytesToWrite;
1223 }
1224 }
1225
1226 while (nRunLength)
1227 {
1228 if (nRunLength > 47)
1229 {
1230 if (nRunLength < 50)
1231 {
1232 controlByte = PLANAR_CONTROL_BYTE(2, 13);
1233 nRunLength -= 45;
1234 }
1235 else
1236 {
1237 controlByte = PLANAR_CONTROL_BYTE(2, 15);
1238 nRunLength -= 47;
1239 }
1240 }
1241 else if (nRunLength > 31)
1242 {
1243 controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1244 nRunLength = 0;
1245 }
1246 else if (nRunLength > 15)
1247 {
1248 controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1249 nRunLength = 0;
1250 }
1251 else
1252 {
1253 controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1254 nRunLength = 0;
1255 }
1256
1257 if (outBufferSize < 1)
1258 return 0;
1259
1260 --outBufferSize;
1261 *pOutput = controlByte;
1262 pOutput++;
1263 }
1264
1265 const intptr_t diff = (pOutput - pOutBuffer);
1266 if ((diff < 0) || (diff > UINT32_MAX))
1267 return 0;
1268 return (UINT32)diff;
1269}
1270
1271static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1272 UINT32 inBufferSize,
1273 BYTE* WINPR_RESTRICT pOutBuffer,
1274 UINT32 outBufferSize)
1275{
1276 BYTE symbol = 0;
1277 const BYTE* pInput = NULL;
1278 BYTE* pOutput = NULL;
1279 const BYTE* pBytes = NULL;
1280 UINT32 cRawBytes = 0;
1281 UINT32 nRunLength = 0;
1282 UINT32 bSymbolMatch = 0;
1283 UINT32 nBytesWritten = 0;
1284 UINT32 nTotalBytesWritten = 0;
1285 symbol = 0;
1286 cRawBytes = 0;
1287 nRunLength = 0;
1288 pInput = pInBuffer;
1289 pOutput = pOutBuffer;
1290 nTotalBytesWritten = 0;
1291
1292 if (!outBufferSize)
1293 return 0;
1294
1295 do
1296 {
1297 if (!inBufferSize)
1298 break;
1299
1300 bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
1301 symbol = *pInput;
1302 pInput++;
1303 inBufferSize--;
1304
1305 if (nRunLength && !bSymbolMatch)
1306 {
1307 if (nRunLength < 3)
1308 {
1309 cRawBytes += nRunLength;
1310 nRunLength = 0;
1311 }
1312 else
1313 {
1314 pBytes = pInput - (cRawBytes + nRunLength + 1);
1315 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1316 pOutput, outBufferSize);
1317 nRunLength = 0;
1318
1319 if (!nBytesWritten || (nBytesWritten > outBufferSize))
1320 return nRunLength;
1321
1322 nTotalBytesWritten += nBytesWritten;
1323 outBufferSize -= nBytesWritten;
1324 pOutput += nBytesWritten;
1325 cRawBytes = 0;
1326 }
1327 }
1328
1329 nRunLength += bSymbolMatch;
1330 cRawBytes += (!bSymbolMatch) ? TRUE : FALSE;
1331 } while (outBufferSize);
1332
1333 if (cRawBytes || nRunLength)
1334 {
1335 pBytes = pInput - (cRawBytes + nRunLength);
1336 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1337 pOutput, outBufferSize);
1338
1339 if (!nBytesWritten)
1340 return 0;
1341
1342 nTotalBytesWritten += nBytesWritten;
1343 }
1344
1345 if (inBufferSize)
1346 return 0;
1347
1348 return nTotalBytesWritten;
1349}
1350
1351BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1352 UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1353 UINT32* WINPR_RESTRICT dstSize)
1354{
1355 UINT32 index = 0;
1356 const BYTE* pInput = NULL;
1357 BYTE* pOutput = NULL;
1358 UINT32 outBufferSize = 0;
1359 UINT32 nBytesWritten = 0;
1360 UINT32 nTotalBytesWritten = 0;
1361
1362 if (!outPlane)
1363 return FALSE;
1364
1365 index = 0;
1366 pInput = inPlane;
1367 pOutput = outPlane;
1368 outBufferSize = *dstSize;
1369 nTotalBytesWritten = 0;
1370
1371 while (outBufferSize)
1372 {
1373 nBytesWritten =
1374 freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1375
1376 if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1377 return FALSE;
1378
1379 outBufferSize -= nBytesWritten;
1380 nTotalBytesWritten += nBytesWritten;
1381 pOutput += nBytesWritten;
1382 pInput += width;
1383 index++;
1384
1385 if (index >= height)
1386 break;
1387 }
1388
1389 *dstSize = nTotalBytesWritten;
1390 return TRUE;
1391}
1392
1393static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1394 UINT32 width, UINT32 height,
1395 BYTE* WINPR_RESTRICT outPlanes,
1396 UINT32* WINPR_RESTRICT dstSizes,
1397 BOOL skipAlpha)
1398{
1399 UINT32 outPlanesSize = width * height * 4;
1400
1401 /* AlphaPlane */
1402 if (skipAlpha)
1403 {
1404 dstSizes[0] = 0;
1405 }
1406 else
1407 {
1408 dstSizes[0] = outPlanesSize;
1409
1410 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1411 &dstSizes[0]))
1412 return FALSE;
1413
1414 outPlanes += dstSizes[0];
1415 outPlanesSize -= dstSizes[0];
1416 }
1417
1418 /* LumaOrRedPlane */
1419 dstSizes[1] = outPlanesSize;
1420
1421 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1422 &dstSizes[1]))
1423 return FALSE;
1424
1425 outPlanes += dstSizes[1];
1426 outPlanesSize -= dstSizes[1];
1427 /* OrangeChromaOrGreenPlane */
1428 dstSizes[2] = outPlanesSize;
1429
1430 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1431 &dstSizes[2]))
1432 return FALSE;
1433
1434 outPlanes += dstSizes[2];
1435 outPlanesSize -= dstSizes[2];
1436 /* GreenChromeOrBluePlane */
1437 dstSizes[3] = outPlanesSize;
1438
1439 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1440 &dstSizes[3]))
1441 return FALSE;
1442
1443 return TRUE;
1444}
1445
1446BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1447 UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1448{
1449 BYTE* outPtr = NULL;
1450 const BYTE* srcPtr = NULL;
1451 const BYTE* prevLinePtr = NULL;
1452
1453 if (!outPlane)
1454 {
1455 if (width * height == 0)
1456 return NULL;
1457
1458 if (!(outPlane = (BYTE*)calloc(height, width)))
1459 return NULL;
1460 }
1461
1462 // first line is copied as is
1463 CopyMemory(outPlane, inPlane, width);
1464 outPtr = outPlane + width;
1465 srcPtr = inPlane + width;
1466 prevLinePtr = inPlane;
1467
1468 for (UINT32 y = 1; y < height; y++)
1469 {
1470 for (UINT32 x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
1471 {
1472 INT32 delta = *srcPtr - *prevLinePtr;
1473 BYTE s2c = WINPR_ASSERTING_INT_CAST(
1474 BYTE, (delta >= 0) ? delta : (~((BYTE)(-delta)) + 1) & 0xFF);
1475 s2c = WINPR_ASSERTING_INT_CAST(
1476 BYTE,
1477 (s2c >= 0) ? (s2c << 1) & 0xFF : (((UINT32)(~((BYTE)s2c) + 1) << 1) - 1) & 0xFF);
1478 *outPtr = s2c;
1479 }
1480 }
1481
1482 return outPlane;
1483}
1484
1485static INLINE BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1486 UINT32 width, UINT32 height,
1487 BYTE* WINPR_RESTRICT outPlanes[4])
1488{
1489 for (UINT32 i = 0; i < 4; i++)
1490 {
1491 outPlanes[i] =
1492 freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1493
1494 if (!outPlanes[i])
1495 return FALSE;
1496 }
1497
1498 return TRUE;
1499}
1500
1501BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1502 const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1503 UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1504 UINT32* WINPR_RESTRICT pDstSize)
1505{
1506 UINT32 size = 0;
1507 BYTE* dstp = NULL;
1508 UINT32 planeSize = 0;
1509 UINT32 dstSizes[4] = { 0 };
1510 BYTE FormatHeader = 0;
1511
1512 if (!context || !context->rlePlanesBuffer)
1513 return NULL;
1514
1515 if (context->AllowSkipAlpha)
1516 FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1517
1518 planeSize = width * height;
1519
1520 if (!context->AllowSkipAlpha)
1521 format = planar_invert_format(context, TRUE, format);
1522
1523 if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1524 context->planes))
1525 return NULL;
1526
1527 if (context->AllowRunLengthEncoding)
1528 {
1529 if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1530 context->deltaPlanes))
1531 return NULL;
1532
1533 if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1534 context->rlePlanesBuffer, dstSizes,
1535 context->AllowSkipAlpha))
1536 return NULL;
1537
1538 {
1539 uint32_t offset = 0;
1540 FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1541 context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1542 offset += dstSizes[0];
1543 context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1544 offset += dstSizes[1];
1545 context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1546 offset += dstSizes[2];
1547 context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1548
1549#if defined(WITH_DEBUG_CODECS)
1550 WLog_DBG(TAG,
1551 "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1552 " / %" PRIu32 "] ",
1553 dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1554#endif
1555 }
1556 }
1557
1558 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1559 {
1560 if (!context->AllowRunLengthEncoding)
1561 return NULL;
1562
1563 if (context->rlePlanes[0] == NULL)
1564 return NULL;
1565
1566 if (context->rlePlanes[1] == NULL)
1567 return NULL;
1568
1569 if (context->rlePlanes[2] == NULL)
1570 return NULL;
1571
1572 if (context->rlePlanes[3] == NULL)
1573 return NULL;
1574 }
1575
1576 if (!dstData)
1577 {
1578 size = 1;
1579
1580 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1581 {
1582 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1583 size += dstSizes[0];
1584 else
1585 size += planeSize;
1586 }
1587
1588 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1589 size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1590 else
1591 size += (planeSize * 3);
1592
1593 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1594 size++;
1595
1596 dstData = malloc(size);
1597
1598 if (!dstData)
1599 return NULL;
1600
1601 *pDstSize = size;
1602 }
1603
1604 dstp = dstData;
1605 *dstp = FormatHeader; /* FormatHeader */
1606 dstp++;
1607
1608 /* AlphaPlane */
1609
1610 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1611 {
1612 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1613 {
1614 CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1615 dstp += dstSizes[0];
1616 }
1617 else
1618 {
1619 CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1620 dstp += planeSize;
1621 }
1622 }
1623
1624 /* LumaOrRedPlane */
1625
1626 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1627 {
1628 CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1629 dstp += dstSizes[1];
1630 }
1631 else
1632 {
1633 CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1634 dstp += planeSize;
1635 }
1636
1637 /* OrangeChromaOrGreenPlane */
1638
1639 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1640 {
1641 CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1642 dstp += dstSizes[2];
1643 }
1644 else
1645 {
1646 CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1647 dstp += planeSize;
1648 }
1649
1650 /* GreenChromeOrBluePlane */
1651
1652 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1653 {
1654 CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1655 dstp += dstSizes[3];
1656 }
1657 else
1658 {
1659 CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1660 dstp += planeSize;
1661 }
1662
1663 /* Pad1 (1 byte) */
1664
1665 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1666 {
1667 *dstp = 0;
1668 dstp++;
1669 }
1670
1671 const intptr_t diff = (dstp - dstData);
1672 if ((diff < 0) || (diff > UINT32_MAX))
1673 {
1674 free(dstData);
1675 return NULL;
1676 }
1677 size = (UINT32)diff;
1678 *pDstSize = size;
1679 return dstData;
1680}
1681
1682BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1683 UINT32 width, UINT32 height)
1684{
1685 if (!context)
1686 return FALSE;
1687
1688 context->bgr = FALSE;
1689 context->maxWidth = PLANAR_ALIGN(width, 4);
1690 context->maxHeight = PLANAR_ALIGN(height, 4);
1691 {
1692 const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1693 if (tmp > UINT32_MAX)
1694 return FALSE;
1695 context->maxPlaneSize = (UINT32)tmp;
1696 }
1697
1698 if (context->maxWidth > UINT32_MAX / 4)
1699 return FALSE;
1700 context->nTempStep = context->maxWidth * 4;
1701
1702 memset((void*)context->planes, 0, sizeof(context->planes));
1703 memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
1704 memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1705
1706 if (context->maxPlaneSize > 0)
1707 {
1708 void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1709 if (!tmp)
1710 return FALSE;
1711 context->planesBuffer = tmp;
1712
1713 tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1714 if (!tmp)
1715 return FALSE;
1716 context->pTempData = tmp;
1717
1718 tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1719 if (!tmp)
1720 return FALSE;
1721 context->deltaPlanesBuffer = tmp;
1722
1723 tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1724 if (!tmp)
1725 return FALSE;
1726 context->rlePlanesBuffer = tmp;
1727
1728 context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1729 context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1730 context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1731 context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1732 context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1733 context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1734 context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1735 context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1736 }
1737 return TRUE;
1738}
1739
1740BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1741 UINT32 maxHeight)
1742{
1743 BITMAP_PLANAR_CONTEXT* context =
1744 (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1745
1746 if (!context)
1747 return NULL;
1748
1749 if (flags & PLANAR_FORMAT_HEADER_NA)
1750 context->AllowSkipAlpha = TRUE;
1751
1752 if (flags & PLANAR_FORMAT_HEADER_RLE)
1753 context->AllowRunLengthEncoding = TRUE;
1754
1755 if (flags & PLANAR_FORMAT_HEADER_CS)
1756 context->AllowColorSubsampling = TRUE;
1757
1758 context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1759
1760 if (context->ColorLossLevel)
1761 context->AllowDynamicColorFidelity = TRUE;
1762
1763 if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1764 {
1765 WINPR_PRAGMA_DIAG_PUSH
1766 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1767 freerdp_bitmap_planar_context_free(context);
1768 WINPR_PRAGMA_DIAG_POP
1769 return NULL;
1770 }
1771
1772 return context;
1773}
1774
1775void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1776{
1777 if (!context)
1778 return;
1779
1780 winpr_aligned_free(context->pTempData);
1781 winpr_aligned_free(context->planesBuffer);
1782 winpr_aligned_free(context->deltaPlanesBuffer);
1783 winpr_aligned_free(context->rlePlanesBuffer);
1784 winpr_aligned_free(context);
1785}
1786
1787void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1788{
1789 WINPR_ASSERT(planar);
1790 planar->bgr = bgr;
1791}
1792
1793void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1794{
1795 WINPR_ASSERT(planar);
1796 planar->topdown = topdown;
1797}
primitives_t
Definition primitives.h:255