FreeRDP
Loading...
Searching...
No Matches
planar.c
1
22#include <freerdp/config.h>
23
24#include <winpr/crt.h>
25#include <winpr/wtypes.h>
26#include <winpr/assert.h>
27#include <winpr/cast.h>
28#include <winpr/print.h>
29
30#include <freerdp/primitives.h>
31#include <freerdp/log.h>
32#include <freerdp/codec/bitmap.h>
33#include <freerdp/codec/planar.h>
34
35#define TAG FREERDP_TAG("codec")
36
37#define PLANAR_ALIGN(val, align) \
38 ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
39
40typedef struct
41{
47 BYTE controlByte;
48 BYTE* rawValues;
49} RDP6_RLE_SEGMENT;
50
51typedef struct
52{
53 UINT32 cSegments;
54 RDP6_RLE_SEGMENT* segments;
55} RDP6_RLE_SEGMENTS;
56
57typedef struct
58{
67 BYTE formatHeader;
68} RDP6_BITMAP_STREAM;
69
70struct S_BITMAP_PLANAR_CONTEXT
71{
72 UINT32 maxWidth;
73 UINT32 maxHeight;
74 UINT32 maxPlaneSize;
75
76 BOOL AllowSkipAlpha;
77 BOOL AllowRunLengthEncoding;
78 BOOL AllowColorSubsampling;
79 BOOL AllowDynamicColorFidelity;
80
81 UINT32 ColorLossLevel;
82
83 BYTE* planes[4];
84 BYTE* planesBuffer;
85
86 BYTE* deltaPlanes[4];
87 BYTE* deltaPlanesBuffer;
88
89 BYTE* rlePlanes[4];
90 BYTE* rlePlanesBuffer;
91
92 BYTE* pTempData;
93 UINT32 nTempStep;
94
95 BOOL bgr;
96 BOOL topdown;
97};
98
99static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes)
100{
101 return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4)));
102}
103
104static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte)
105{
106 return (controlByte & 0x0F);
107}
108static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
109{
110 return ((controlByte >> 4) & 0x0F);
111}
112
113static inline UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
114 UINT32 DstFormat)
115{
116 WINPR_ASSERT(planar);
117 if (planar->bgr && alpha)
118 {
119 switch (DstFormat)
120 {
121 case PIXEL_FORMAT_ARGB32:
122 DstFormat = PIXEL_FORMAT_ABGR32;
123 break;
124 case PIXEL_FORMAT_XRGB32:
125 DstFormat = PIXEL_FORMAT_XBGR32;
126 break;
127 case PIXEL_FORMAT_ABGR32:
128 DstFormat = PIXEL_FORMAT_ARGB32;
129 break;
130 case PIXEL_FORMAT_XBGR32:
131 DstFormat = PIXEL_FORMAT_XRGB32;
132 break;
133 case PIXEL_FORMAT_BGRA32:
134 DstFormat = PIXEL_FORMAT_RGBA32;
135 break;
136 case PIXEL_FORMAT_BGRX32:
137 DstFormat = PIXEL_FORMAT_RGBX32;
138 break;
139 case PIXEL_FORMAT_RGBA32:
140 DstFormat = PIXEL_FORMAT_BGRA32;
141 break;
142 case PIXEL_FORMAT_RGBX32:
143 DstFormat = PIXEL_FORMAT_BGRX32;
144 break;
145 case PIXEL_FORMAT_RGB24:
146 DstFormat = PIXEL_FORMAT_BGR24;
147 break;
148 case PIXEL_FORMAT_BGR24:
149 DstFormat = PIXEL_FORMAT_RGB24;
150 break;
151 case PIXEL_FORMAT_RGB16:
152 DstFormat = PIXEL_FORMAT_BGR16;
153 break;
154 case PIXEL_FORMAT_BGR16:
155 DstFormat = PIXEL_FORMAT_RGB16;
156 break;
157 case PIXEL_FORMAT_ARGB15:
158 DstFormat = PIXEL_FORMAT_ABGR15;
159 break;
160 case PIXEL_FORMAT_RGB15:
161 DstFormat = PIXEL_FORMAT_BGR15;
162 break;
163 case PIXEL_FORMAT_ABGR15:
164 DstFormat = PIXEL_FORMAT_ARGB15;
165 break;
166 case PIXEL_FORMAT_BGR15:
167 DstFormat = PIXEL_FORMAT_RGB15;
168 break;
169 default:
170 break;
171 }
172 }
173 return DstFormat;
174}
175
176static inline BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
177 UINT32 width, UINT32 height,
178 BYTE* WINPR_RESTRICT outPlane,
179 UINT32* WINPR_RESTRICT dstSize);
180static inline BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
181 UINT32 width, UINT32 height,
182 BYTE* WINPR_RESTRICT outPlane);
183
184static inline INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
185 UINT32 nWidth, UINT32 nHeight)
186{
187 UINT32 used = 0;
188
189 WINPR_ASSERT(pSrcData);
190
191 for (UINT32 y = 0; y < nHeight; y++)
192 {
193 for (UINT32 x = 0; x < nWidth;)
194 {
195 if (used >= SrcSize)
196 {
197 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
198 SrcSize);
199 return -1;
200 }
201
202 const uint8_t controlByte = pSrcData[used++];
203 uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
204 uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
205
206 if (nRunLength == 1)
207 {
208 nRunLength = cRawBytes + 16;
209 cRawBytes = 0;
210 }
211 else if (nRunLength == 2)
212 {
213 nRunLength = cRawBytes + 32;
214 cRawBytes = 0;
215 }
216
217 used += cRawBytes;
218 x += cRawBytes;
219 x += nRunLength;
220
221 if (x > nWidth)
222 {
223 WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
224 return -1;
225 }
226
227 if (used > SrcSize)
228 {
229 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRId32, used,
230 INT32_MAX);
231 return -1;
232 }
233 }
234 }
235
236 if (used > INT32_MAX)
237 {
238 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
239 return -1;
240 }
241 return (INT32)used;
242}
243
244static inline UINT8 clamp(INT16 val)
245{
246 return (UINT8)val;
247}
248
249static inline bool check_source_available(const BYTE* buffer, size_t length, const BYTE* cur,
250 size_t required)
251{
252 WINPR_ASSERT(cur >= buffer);
253 const size_t len = WINPR_ASSERTING_INT_CAST(size_t, (cur - buffer));
254
255 if ((SIZE_MAX - len < required) || (len + required > length))
256 {
257 WLog_ERR(TAG, "error reading input buffer");
258 return false;
259 }
260 return true;
261}
262
263static inline INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
264 UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
265 UINT32 nWidth, UINT32 nHeight)
266{
267 BYTE* previousScanline = nullptr;
268 const BYTE* srcp = pSrcData;
269
270 WINPR_ASSERT(nHeight <= INT32_MAX);
271 WINPR_ASSERT(nWidth <= INT32_MAX);
272
273 for (UINT32 y = 0; y < nHeight; y++)
274 {
275 BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
276 INT16 pixel = 0;
277 BYTE* currentScanline = dstp;
278
279 for (UINT32 x = 0; x < nWidth;)
280 {
281 if (!check_source_available(pSrcData, SrcSize, srcp, 1))
282 return -1;
283
284 const BYTE controlByte = *srcp++;
285
286 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
287 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
288
289 if (nRunLength == 1)
290 {
291 nRunLength = cRawBytes + 16;
292 cRawBytes = 0;
293 }
294 else if (nRunLength == 2)
295 {
296 nRunLength = cRawBytes + 32;
297 cRawBytes = 0;
298 }
299
300 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
301 {
302 WLog_ERR(TAG, "too many pixels in scanline");
303 return -1;
304 }
305
306 if (!previousScanline)
307 {
308 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
309 return -1;
310 /* first scanline, absolute values */
311 while (cRawBytes > 0)
312 {
313 pixel = *srcp++;
314 *dstp = clamp(pixel);
315 dstp++;
316 x++;
317 cRawBytes--;
318 }
319
320 while (nRunLength > 0)
321 {
322 *dstp = clamp(pixel);
323 dstp++;
324 x++;
325 nRunLength--;
326 }
327 }
328 else
329 {
330 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
331 return -1;
332
333 /* delta values relative to previous scanline */
334 while (cRawBytes > 0)
335 {
336 UINT8 deltaValue = *srcp++;
337
338 if (deltaValue & 1)
339 {
340 deltaValue = deltaValue >> 1;
341 deltaValue = deltaValue + 1;
342 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
343 }
344 else
345 {
346 deltaValue = deltaValue >> 1;
347 pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
348 }
349
350 const INT16 delta =
351 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
352 *dstp = clamp(delta);
353 dstp++;
354 x++;
355 cRawBytes--;
356 }
357
358 while (nRunLength > 0)
359 {
360 const INT16 deltaValue =
361 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
362 *dstp = clamp(deltaValue);
363 dstp++;
364 x++;
365 nRunLength--;
366 }
367 }
368 }
369
370 previousScanline = currentScanline;
371 }
372
373 return (INT32)(srcp - pSrcData);
374}
375
376static inline INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
377 BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
378 UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
379 UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
380{
381 INT32 beg = 0;
382 INT32 end = 0;
383 INT32 inc = 0;
384 BYTE* previousScanline = nullptr;
385 const BYTE* srcp = pSrcData;
386
387 WINPR_ASSERT(nHeight <= INT32_MAX);
388 WINPR_ASSERT(nWidth <= INT32_MAX);
389 WINPR_ASSERT(nDstStep <= INT32_MAX);
390
391 if (vFlip)
392 {
393 beg = (INT32)nHeight - 1;
394 end = -1;
395 inc = -1;
396 }
397 else
398 {
399 beg = 0;
400 end = (INT32)nHeight;
401 inc = 1;
402 }
403
404 for (INT32 y = beg; y != end; y += inc)
405 {
406 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
407 BYTE* dstp = &pDstData[off];
408 BYTE* currentScanline = dstp;
409 INT16 pixel = 0;
410
411 for (INT32 x = 0; x < (INT32)nWidth;)
412 {
413 if (!check_source_available(pSrcData, SrcSize, srcp, 1))
414 return -1;
415
416 const BYTE controlByte = *srcp++;
417
418 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
419 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
420
421 if (nRunLength == 1)
422 {
423 nRunLength = cRawBytes + 16;
424 cRawBytes = 0;
425 }
426 else if (nRunLength == 2)
427 {
428 nRunLength = cRawBytes + 32;
429 cRawBytes = 0;
430 }
431
432 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
433 {
434 WLog_ERR(TAG, "too many pixels in scanline");
435 return -1;
436 }
437
438 if (!previousScanline)
439 {
440 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
441 return -1;
442
443 /* first scanline, absolute values */
444 while (cRawBytes > 0)
445 {
446 pixel = *srcp++;
447
448 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
449 dstp += 4;
450 x++;
451 cRawBytes--;
452 }
453
454 while (nRunLength > 0)
455 {
456 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
457 dstp += 4;
458 x++;
459 nRunLength--;
460 }
461 }
462 else
463 {
464 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
465 return -1;
466
467 /* delta values relative to previous scanline */
468 while (cRawBytes > 0)
469 {
470 BYTE deltaValue = *srcp++;
471
472 if (deltaValue & 1)
473 {
474 deltaValue = deltaValue >> 1;
475 deltaValue = deltaValue + 1;
476 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
477 }
478 else
479 {
480 deltaValue = deltaValue >> 1;
481 pixel = deltaValue;
482 }
483
484 const INT16 delta =
485 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
486 *dstp = clamp(delta);
487 dstp += 4;
488 x++;
489 cRawBytes--;
490 }
491
492 while (nRunLength > 0)
493 {
494 const INT16 deltaValue =
495 WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
496 *dstp = clamp(deltaValue);
497 dstp += 4;
498 x++;
499 nRunLength--;
500 }
501 }
502 }
503
504 previousScanline = currentScanline;
505 }
506
507 return (INT32)(srcp - pSrcData);
508}
509
510static inline INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
511 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
512 BOOL vFlip)
513{
514 INT32 beg = 0;
515 INT32 end = (INT32)nHeight;
516 INT32 inc = 1;
517
518 WINPR_ASSERT(nHeight <= INT32_MAX);
519 WINPR_ASSERT(nWidth <= INT32_MAX);
520 WINPR_ASSERT(nDstStep <= INT32_MAX);
521
522 if (vFlip)
523 {
524 beg = (INT32)nHeight - 1;
525 end = -1;
526 inc = -1;
527 }
528
529 for (INT32 y = beg; y != end; y += inc)
530 {
531 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
532 BYTE* dstp = &pDstData[off];
533
534 for (INT32 x = 0; x < (INT32)nWidth; ++x)
535 {
536 *dstp = bValue;
537 dstp += 4;
538 }
539 }
540
541 return 0;
542}
543
544static inline BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
545 const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
546 const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
547{
548 WINPR_ASSERT(ppRgba);
549 WINPR_ASSERT(ppR);
550 WINPR_ASSERT(ppG);
551 WINPR_ASSERT(ppB);
552
553 switch (DstFormat)
554 {
555 case PIXEL_FORMAT_BGRA32:
556 for (UINT32 x = 0; x < width; x++)
557 {
558 *(*ppRgba)++ = *(*ppB)++;
559 *(*ppRgba)++ = *(*ppG)++;
560 *(*ppRgba)++ = *(*ppR)++;
561 *(*ppRgba)++ = *(*ppA)++;
562 }
563
564 return TRUE;
565
566 case PIXEL_FORMAT_BGRX32:
567 for (UINT32 x = 0; x < width; x++)
568 {
569 *(*ppRgba)++ = *(*ppB)++;
570 *(*ppRgba)++ = *(*ppG)++;
571 *(*ppRgba)++ = *(*ppR)++;
572 *(*ppRgba)++ = 0xFF;
573 }
574
575 return TRUE;
576
577 default:
578 if (ppA)
579 {
580 for (UINT32 x = 0; x < width; x++)
581 {
582 BYTE alpha = *(*ppA)++;
583 UINT32 color =
584 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
585 FreeRDPWriteColor(*ppRgba, DstFormat, color);
586 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
587 }
588 }
589 else
590 {
591 const BYTE alpha = 0xFF;
592
593 for (UINT32 x = 0; x < width; x++)
594 {
595 UINT32 color =
596 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
597 FreeRDPWriteColor(*ppRgba, DstFormat, color);
598 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
599 }
600 }
601
602 return TRUE;
603 }
604}
605
606static inline BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
607 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
608 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
609 UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
610 UINT32 totalHeight)
611{
612 INT32 beg = 0;
613 INT32 end = 0;
614 INT32 inc = 0;
615 const BYTE* pR = pSrcData[0];
616 const BYTE* pG = pSrcData[1];
617 const BYTE* pB = pSrcData[2];
618 const BYTE* pA = pSrcData[3];
619 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
620
621 if (vFlip)
622 {
623 beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
624 end = -1;
625 inc = -1;
626 }
627 else
628 {
629 beg = 0;
630 end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
631 inc = 1;
632 }
633
634 if (nYDst + nHeight > totalHeight)
635 {
636 WLog_ERR(TAG,
637 "planar plane destination Y %" PRIu32 " + height %" PRIu32
638 " exceeds totalHeight %" PRIu32,
639 nYDst, nHeight, totalHeight);
640 return FALSE;
641 }
642
643 if ((nXDst + nWidth) * bpp > nDstStep)
644 {
645 WLog_ERR(TAG,
646 "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
647 " exceeds stride %" PRIu32,
648 nXDst, nWidth, bpp, nDstStep);
649 return FALSE;
650 }
651
652 for (INT32 y = beg; y != end; y += inc)
653 {
654 BYTE* pRGB = nullptr;
655
656 if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
657 {
658 WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
659 nHeight);
660 return FALSE;
661 }
662
663 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
664 pRGB = &pDstData[off];
665
666 if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
667 return FALSE;
668 }
669
670 return TRUE;
671}
672
673static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
674 UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
675 UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
676{
677 size_t pos = 0;
678 WINPR_UNUSED(planeLength);
679
680 WINPR_ASSERT(plane);
681 WINPR_ASSERT(deltaPlane);
682
683 if (nWidth > nPlaneWidth * 2)
684 {
685 WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
686 nPlaneWidth);
687 return FALSE;
688 }
689
690 if (nHeight > nPlaneHeight * 2)
691 {
692 WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
693 nPlaneHeight);
694 return FALSE;
695 }
696
697 for (size_t y = 0; y < nHeight; y++)
698 {
699 const BYTE* src = plane + y / 2 * nPlaneWidth;
700
701 for (UINT32 x = 0; x < nWidth; x++)
702 {
703 deltaPlane[pos++] = src[x / 2];
704 }
705 }
706
707 return TRUE;
708}
709
710#if !defined(WITHOUT_FREERDP_3x_DEPRECATED)
711BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
712 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
713 UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
714 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
715 UINT32 nDstHeight, BOOL vFlip)
716{
717 return freerdp_bitmap_decompress_planar(planar, pSrcData, SrcSize, nSrcWidth, nSrcHeight,
718 pDstData, DstFormat, nDstStep, nXDst, nYDst, nDstWidth,
719 nDstHeight, vFlip);
720}
721#endif
722
723BOOL freerdp_bitmap_decompress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
724 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
725 UINT32 nSrcWidth, UINT32 nSrcHeight,
726 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
727 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
728 UINT32 nDstHeight, BOOL vFlip)
729{
730 BOOL useAlpha = FALSE;
731 INT32 status = 0;
732 INT32 rleSizes[4] = { 0, 0, 0, 0 };
733 UINT32 rawSizes[4] = WINPR_C_ARRAY_INIT;
734 UINT32 rawWidths[4] = WINPR_C_ARRAY_INIT;
735 UINT32 rawHeights[4] = WINPR_C_ARRAY_INIT;
736 const BYTE* planes[4] = WINPR_C_ARRAY_INIT;
737 const UINT32 w = MIN(nSrcWidth, nDstWidth);
738 const UINT32 h = MIN(nSrcHeight, nDstHeight);
739 const primitives_t* prims = primitives_get();
740
741 WINPR_ASSERT(planar);
742 WINPR_ASSERT(prims);
743
744 if (planar->maxWidth < nSrcWidth)
745 return FALSE;
746 if (planar->maxHeight < nSrcHeight)
747 return FALSE;
748
749 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
750 if (nDstStep <= 0)
751 nDstStep = nDstWidth * bpp;
752
753 const BYTE* srcp = pSrcData;
754
755 if (!pSrcData || (SrcSize < 1))
756 {
757 WLog_ERR(TAG, "Invalid argument pSrcData=%p [size=%" PRIu32 "]",
758 WINPR_CXX_COMPAT_CAST(const void*, pSrcData), SrcSize);
759 return FALSE;
760 }
761
762 if (!pDstData)
763 {
764 WLog_ERR(TAG, "Invalid argument pDstData=nullptr");
765 return FALSE;
766 }
767
768 const BYTE FormatHeader = *srcp++;
769 const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
770 const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) != 0;
771 const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) != 0;
772 const BYTE alpha = !(FormatHeader & PLANAR_FORMAT_HEADER_NA);
773
774 DstFormat = planar_invert_format(planar, alpha, DstFormat);
775
776 if (alpha)
777 useAlpha = FreeRDPColorHasAlpha(DstFormat);
778
779 // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
780 // alpha);
781
782 if (!cll && cs)
783 {
784 WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
785 return FALSE; /* Chroma subsampling requires YCoCg */
786 }
787
788 const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
789 const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
790 const UINT32 planeSize = nSrcWidth * nSrcHeight;
791 const UINT32 subSize = subWidth * subHeight;
792
793 if (!cs)
794 {
795 rawSizes[0] = planeSize; /* LumaOrRedPlane */
796 rawWidths[0] = nSrcWidth;
797 rawHeights[0] = nSrcHeight;
798 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
799 rawWidths[1] = nSrcWidth;
800 rawHeights[1] = nSrcHeight;
801 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
802 rawWidths[2] = nSrcWidth;
803 rawHeights[2] = nSrcHeight;
804 rawSizes[3] = planeSize; /* AlphaPlane */
805 rawWidths[3] = nSrcWidth;
806 rawHeights[3] = nSrcHeight;
807 }
808 else /* Chroma Subsampling */
809 {
810 rawSizes[0] = planeSize; /* LumaOrRedPlane */
811 rawWidths[0] = nSrcWidth;
812 rawHeights[0] = nSrcHeight;
813 rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
814 rawWidths[1] = subWidth;
815 rawHeights[1] = subHeight;
816 rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
817 rawWidths[2] = subWidth;
818 rawHeights[2] = subHeight;
819 rawSizes[3] = planeSize; /* AlphaPlane */
820 rawWidths[3] = nSrcWidth;
821 rawHeights[3] = nSrcHeight;
822 }
823
824 const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
825 if (SrcSize < diff)
826 {
827 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
828 return FALSE;
829 }
830
831 if (!rle) /* RAW */
832 {
833
834 UINT32 base = planeSize * 3;
835 if (cs)
836 base = planeSize + planeSize / 2;
837
838 if (alpha)
839 {
840 if ((SrcSize - diff) < (planeSize + base))
841 {
842 WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
843 (planeSize + base));
844 return FALSE;
845 }
846
847 planes[3] = srcp; /* AlphaPlane */
848 planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
849 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
850 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
851
852 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
853 {
854 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
855 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
856 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
857 return FALSE;
858 }
859 }
860 else
861 {
862 if ((SrcSize - diff) < base)
863 {
864 WLog_ERR(TAG, "plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, base);
865 return FALSE;
866 }
867
868 planes[0] = srcp; /* LumaOrRedPlane */
869 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
870 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
871
872 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
873 {
874 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
875 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
876 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
877 return FALSE;
878 }
879 }
880 }
881 else /* RLE */
882 {
883 if (alpha)
884 {
885 planes[3] = srcp;
886 rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
887 rawHeights[3]); /* AlphaPlane */
888
889 if (rleSizes[3] < 0)
890 return FALSE;
891
892 planes[0] = planes[3] + rleSizes[3];
893 }
894 else
895 planes[0] = srcp;
896
897 const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
898 if (SrcSize < diff0)
899 {
900 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
901 return FALSE;
902 }
903 rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
904 rawHeights[0]); /* RedPlane */
905
906 if (rleSizes[0] < 0)
907 return FALSE;
908
909 planes[1] = planes[0] + rleSizes[0];
910
911 const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
912 if (SrcSize < diff1)
913 {
914 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
915 return FALSE;
916 }
917 rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
918 rawHeights[1]); /* GreenPlane */
919
920 if (rleSizes[1] < 1)
921 return FALSE;
922
923 planes[2] = planes[1] + rleSizes[1];
924 const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
925 if (SrcSize < diff2)
926 {
927 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
928 return FALSE;
929 }
930 rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
931 rawHeights[2]); /* BluePlane */
932
933 if (rleSizes[2] < 1)
934 return FALSE;
935 }
936
937 if (!cll) /* RGB */
938 {
939 UINT32 TempFormat = 0;
940 BYTE* pTempData = pDstData;
941 UINT32 nTempStep = nDstStep;
942 UINT32 nTotalHeight = nYDst + nDstHeight;
943
944 if (useAlpha)
945 TempFormat = PIXEL_FORMAT_BGRA32;
946 else
947 TempFormat = PIXEL_FORMAT_BGRX32;
948
949 TempFormat = planar_invert_format(planar, alpha, TempFormat);
950
951 if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
952 {
953 pTempData = planar->pTempData;
954 nTempStep = planar->nTempStep;
955 nTotalHeight = planar->maxHeight;
956 }
957
958 if (!rle) /* RAW */
959 {
960 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
961 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
962 return FALSE;
963
964 if (alpha)
965 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
966 else /* NoAlpha */
967 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
968
969 if ((SrcSize - (srcp - pSrcData)) == 1)
970 srcp++; /* pad */
971 }
972 else /* RLE */
973 {
974 if (nYDst + nSrcHeight > nTotalHeight)
975 {
976 WLog_ERR(TAG,
977 "planar plane destination Y %" PRIu32 " + height %" PRIu32
978 " exceeds totalHeight %" PRIu32,
979 nYDst, nSrcHeight, nTotalHeight);
980 return FALSE;
981 }
982
983 if ((nXDst + nSrcWidth) * bpp > nTempStep)
984 {
985 WLog_ERR(TAG,
986 "planar plane destination (X %" PRIu32 " + width %" PRIu32
987 ") * bpp %" PRIu32 " exceeds stride %" PRIu32,
988 nXDst, nSrcWidth, bpp, nTempStep);
989 return FALSE;
990 }
991
992 status = planar_decompress_plane_rle(
993 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
994 nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
995
996 if (status < 0)
997 return FALSE;
998
999 status = planar_decompress_plane_rle(
1000 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
1001 nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */
1002
1003 if (status < 0)
1004 return FALSE;
1005
1006 status = planar_decompress_plane_rle(
1007 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
1008 nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
1009
1010 if (status < 0)
1011 return FALSE;
1012
1013 srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
1014
1015 if (useAlpha)
1016 {
1017 status = planar_decompress_plane_rle(
1018 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
1019 nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
1020 }
1021 else
1022 status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
1023 nSrcHeight, 3, vFlip);
1024
1025 if (status < 0)
1026 return FALSE;
1027
1028 if (alpha)
1029 srcp += rleSizes[3];
1030 }
1031
1032 if (pTempData != pDstData)
1033 {
1034 if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
1035 pTempData, TempFormat, nTempStep, nXDst, nYDst,
1036 nullptr, FREERDP_FLIP_NONE))
1037 {
1038 WLog_ERR(TAG, "planar image copy failed");
1039 return FALSE;
1040 }
1041 }
1042 }
1043 else /* YCoCg */
1044 {
1045 UINT32 TempFormat = 0;
1046 BYTE* pTempData = planar->pTempData;
1047 UINT32 nTempStep = planar->nTempStep;
1048 UINT32 nTotalHeight = planar->maxHeight;
1049 BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
1050
1051 if (useAlpha)
1052 TempFormat = PIXEL_FORMAT_BGRA32;
1053 else
1054 TempFormat = PIXEL_FORMAT_BGRX32;
1055
1056 if (!pTempData)
1057 return FALSE;
1058
1059 if (rle) /* RLE encoded data. Decode and handle it like raw data. */
1060 {
1061 BYTE* rleBuffer[4] = WINPR_C_ARRAY_INIT;
1062
1063 if (!planar->rlePlanesBuffer)
1064 return FALSE;
1065
1066 rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1067 rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1068 rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1069 rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1070 if (useAlpha)
1071 {
1072 status = planar_decompress_plane_rle_only(
1073 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
1074 rawWidths[3], rawHeights[3]); /* AlphaPlane */
1075
1076 if (status < 0)
1077 return FALSE;
1078 }
1079
1080 if (alpha)
1081 srcp += rleSizes[3];
1082
1083 status = planar_decompress_plane_rle_only(
1084 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
1085 rawWidths[0], rawHeights[0]); /* LumaPlane */
1086
1087 if (status < 0)
1088 return FALSE;
1089
1090 status = planar_decompress_plane_rle_only(
1091 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
1092 rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */
1093
1094 if (status < 0)
1095 return FALSE;
1096
1097 status = planar_decompress_plane_rle_only(
1098 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
1099 rawWidths[2], rawHeights[2]); /* GreenChromaPlane */
1100
1101 if (status < 0)
1102 return FALSE;
1103
1104 planes[0] = rleBuffer[0];
1105 planes[1] = rleBuffer[1];
1106 planes[2] = rleBuffer[2];
1107 planes[3] = rleBuffer[3];
1108 }
1109
1110 /* RAW */
1111 {
1112 if (cs)
1113 { /* Chroma subsampling for Co and Cg:
1114 * Each pixel contains the value that should be expanded to
1115 * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1116 if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1117 rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1118 return FALSE;
1119
1120 planes[1] = planar->deltaPlanes[0];
1121 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1122 rawWidths[1] = nSrcWidth;
1123 rawHeights[1] = nSrcHeight;
1124
1125 if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1126 rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1127 return FALSE;
1128
1129 planes[2] = planar->deltaPlanes[1];
1130 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1131 rawWidths[2] = nSrcWidth;
1132 rawHeights[2] = nSrcHeight;
1133 }
1134
1135 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1136 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1137 return FALSE;
1138
1139 if (alpha)
1140 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1141 else /* NoAlpha */
1142 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1143
1144 if ((SrcSize - (srcp - pSrcData)) == 1)
1145 srcp++; /* pad */
1146 }
1147
1148 WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1149 int rc = prims->YCoCgToRGB_8u_AC4R(
1150 pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
1151 WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
1152 if (rc != PRIMITIVES_SUCCESS)
1153 {
1154 WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1155 return FALSE;
1156 }
1157 }
1158
1159 WINPR_UNUSED(srcp);
1160 return TRUE;
1161}
1162
1163static inline BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1164 const BYTE* WINPR_RESTRICT data, UINT32 format,
1165 UINT32 width, UINT32 height, UINT32 scanline,
1166 BYTE* WINPR_RESTRICT planes[4])
1167{
1168 WINPR_ASSERT(planar);
1169
1170 if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1171 return FALSE;
1172
1173 if (scanline == 0)
1174 scanline = width * FreeRDPGetBytesPerPixel(format);
1175
1176 if (planar->topdown)
1177 {
1178 UINT32 k = 0;
1179 for (UINT32 i = 0; i < height; i++)
1180 {
1181 const BYTE* pixel = &data[1ULL * scanline * i];
1182
1183 for (UINT32 j = 0; j < width; j++)
1184 {
1185 const UINT32 color = FreeRDPReadColor(pixel, format);
1186 pixel += FreeRDPGetBytesPerPixel(format);
1187 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1188 &planes[0][k], nullptr);
1189 k++;
1190 }
1191 }
1192 }
1193 else
1194 {
1195 UINT32 k = 0;
1196
1197 for (INT64 i = (INT64)height - 1; i >= 0; i--)
1198 {
1199 const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1200
1201 for (UINT32 j = 0; j < width; j++)
1202 {
1203 const UINT32 color = FreeRDPReadColor(pixel, format);
1204 pixel += FreeRDPGetBytesPerPixel(format);
1205 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1206 &planes[0][k], nullptr);
1207 k++;
1208 }
1209 }
1210 }
1211 return TRUE;
1212}
1213
1214static inline UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1215 UINT32 cRawBytes, UINT32 nRunLength,
1216 BYTE* WINPR_RESTRICT pOutBuffer,
1217 UINT32 outBufferSize)
1218{
1219 const BYTE* pInput = pInBuffer;
1220 BYTE* pOutput = pOutBuffer;
1221 BYTE controlByte = 0;
1222 UINT32 nBytesToWrite = 0;
1223
1224 if (!cRawBytes && !nRunLength)
1225 return 0;
1226
1227 if (nRunLength < 3)
1228 {
1229 cRawBytes += nRunLength;
1230 nRunLength = 0;
1231 }
1232
1233 while (cRawBytes)
1234 {
1235 if (cRawBytes < 16)
1236 {
1237 if (nRunLength > 15)
1238 {
1239 if (nRunLength < 18)
1240 {
1241 controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1242 nRunLength -= 13;
1243 cRawBytes = 0;
1244 }
1245 else
1246 {
1247 controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1248 nRunLength -= 15;
1249 cRawBytes = 0;
1250 }
1251 }
1252 else
1253 {
1254 controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1255 nRunLength = 0;
1256 cRawBytes = 0;
1257 }
1258 }
1259 else
1260 {
1261 controlByte = PLANAR_CONTROL_BYTE(0, 15);
1262 cRawBytes -= 15;
1263 }
1264
1265 if (outBufferSize < 1)
1266 return 0;
1267
1268 outBufferSize--;
1269 *pOutput = controlByte;
1270 pOutput++;
1271 nBytesToWrite = (controlByte >> 4);
1272
1273 if (nBytesToWrite)
1274 {
1275 if (outBufferSize < nBytesToWrite)
1276 return 0;
1277
1278 outBufferSize -= nBytesToWrite;
1279 CopyMemory(pOutput, pInput, nBytesToWrite);
1280 pOutput += nBytesToWrite;
1281 pInput += nBytesToWrite;
1282 }
1283 }
1284
1285 while (nRunLength)
1286 {
1287 if (nRunLength > 47)
1288 {
1289 if (nRunLength < 50)
1290 {
1291 controlByte = PLANAR_CONTROL_BYTE(2, 13);
1292 nRunLength -= 45;
1293 }
1294 else
1295 {
1296 controlByte = PLANAR_CONTROL_BYTE(2, 15);
1297 nRunLength -= 47;
1298 }
1299 }
1300 else if (nRunLength > 31)
1301 {
1302 controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1303 nRunLength = 0;
1304 }
1305 else if (nRunLength > 15)
1306 {
1307 controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1308 nRunLength = 0;
1309 }
1310 else
1311 {
1312 controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1313 nRunLength = 0;
1314 }
1315
1316 if (outBufferSize < 1)
1317 return 0;
1318
1319 --outBufferSize;
1320 *pOutput = controlByte;
1321 pOutput++;
1322 }
1323
1324 const intptr_t diff = (pOutput - pOutBuffer);
1325 if ((diff < 0) || (diff > UINT32_MAX))
1326 return 0;
1327 return (UINT32)diff;
1328}
1329
1330static inline UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1331 UINT32 inBufferSize,
1332 BYTE* WINPR_RESTRICT pOutBuffer,
1333 UINT32 outBufferSize)
1334{
1335 BYTE symbol = 0;
1336 const BYTE* pInput = pInBuffer;
1337 BYTE* pOutput = pOutBuffer;
1338 const BYTE* pBytes = nullptr;
1339 UINT32 cRawBytes = 0;
1340 UINT32 nRunLength = 0;
1341 UINT32 nBytesWritten = 0;
1342 UINT32 nTotalBytesWritten = 0;
1343
1344 if (!outBufferSize)
1345 return 0;
1346
1347 do
1348 {
1349 if (!inBufferSize)
1350 break;
1351
1352 const UINT32 bSymbolMatch = (symbol == *pInput) != 0;
1353 symbol = *pInput;
1354 pInput++;
1355 inBufferSize--;
1356
1357 if (nRunLength && !bSymbolMatch)
1358 {
1359 if (nRunLength < 3)
1360 {
1361 cRawBytes += nRunLength;
1362 nRunLength = 0;
1363 }
1364 else
1365 {
1366 pBytes = pInput - (cRawBytes + nRunLength + 1);
1367 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1368 pOutput, outBufferSize);
1369 nRunLength = 0;
1370
1371 if (!nBytesWritten || (nBytesWritten > outBufferSize))
1372 return nRunLength;
1373
1374 nTotalBytesWritten += nBytesWritten;
1375 outBufferSize -= nBytesWritten;
1376 pOutput += nBytesWritten;
1377 cRawBytes = 0;
1378 }
1379 }
1380
1381 nRunLength += bSymbolMatch;
1382 cRawBytes += (!bSymbolMatch) != 0;
1383 } while (outBufferSize);
1384
1385 if (cRawBytes || nRunLength)
1386 {
1387 pBytes = pInput - (cRawBytes + nRunLength);
1388 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1389 pOutput, outBufferSize);
1390
1391 if (!nBytesWritten)
1392 return 0;
1393
1394 nTotalBytesWritten += nBytesWritten;
1395 }
1396
1397 if (inBufferSize)
1398 return 0;
1399
1400 return nTotalBytesWritten;
1401}
1402
1403BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1404 UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1405 UINT32* WINPR_RESTRICT dstSize)
1406{
1407 if (!outPlane)
1408 return FALSE;
1409
1410 UINT32 index = 0;
1411 const BYTE* pInput = inPlane;
1412 BYTE* pOutput = outPlane;
1413 UINT32 outBufferSize = *dstSize;
1414 UINT32 nTotalBytesWritten = 0;
1415
1416 while (outBufferSize > 0)
1417 {
1418 const UINT32 nBytesWritten =
1419 freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1420
1421 if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1422 return FALSE;
1423
1424 outBufferSize -= nBytesWritten;
1425 nTotalBytesWritten += nBytesWritten;
1426 pOutput += nBytesWritten;
1427 pInput += width;
1428 index++;
1429
1430 if (index >= height)
1431 break;
1432 }
1433
1434 *dstSize = nTotalBytesWritten;
1435 return TRUE;
1436}
1437
1438static inline BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1439 UINT32 width, UINT32 height,
1440 BYTE* WINPR_RESTRICT outPlanes,
1441 UINT32* WINPR_RESTRICT dstSizes,
1442 BOOL skipAlpha)
1443{
1444 UINT32 outPlanesSize = width * height * 4;
1445
1446 /* AlphaPlane */
1447 if (skipAlpha)
1448 {
1449 dstSizes[0] = 0;
1450 }
1451 else
1452 {
1453 dstSizes[0] = outPlanesSize;
1454
1455 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1456 &dstSizes[0]))
1457 return FALSE;
1458
1459 outPlanes += dstSizes[0];
1460 outPlanesSize -= dstSizes[0];
1461 }
1462
1463 /* LumaOrRedPlane */
1464 dstSizes[1] = outPlanesSize;
1465
1466 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1467 &dstSizes[1]))
1468 return FALSE;
1469
1470 outPlanes += dstSizes[1];
1471 outPlanesSize -= dstSizes[1];
1472 /* OrangeChromaOrGreenPlane */
1473 dstSizes[2] = outPlanesSize;
1474
1475 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1476 &dstSizes[2]))
1477 return FALSE;
1478
1479 outPlanes += dstSizes[2];
1480 outPlanesSize -= dstSizes[2];
1481 /* GreenChromeOrBluePlane */
1482 dstSizes[3] = outPlanesSize;
1483
1484 return (freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1485 &dstSizes[3]));
1486}
1487
1488BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1489 UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1490{
1491 if (!outPlane)
1492 {
1493 if (width * height == 0)
1494 return nullptr;
1495
1496 outPlane = (BYTE*)calloc(height, width);
1497 if (!outPlane)
1498 return nullptr;
1499 }
1500
1501 // first line is copied as is
1502 CopyMemory(outPlane, inPlane, width);
1503
1504 for (UINT32 y = 1; y < height; y++)
1505 {
1506 const size_t off = 1ull * width * y;
1507 BYTE* outPtr = &outPlane[off];
1508 const BYTE* srcPtr = &inPlane[off];
1509 const BYTE* prevLinePtr = &inPlane[off - width];
1510 for (UINT32 x = 0; x < width; x++)
1511 {
1512 const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
1513 const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
1514 const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
1515 const uint32_t s2c =
1516 (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
1517 outPtr[x] = (BYTE)s2c;
1518 }
1519 }
1520
1521 return outPlane;
1522}
1523
1524static inline BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1525 UINT32 width, UINT32 height,
1526 BYTE* WINPR_RESTRICT outPlanes[4])
1527{
1528 for (UINT32 i = 0; i < 4; i++)
1529 {
1530 outPlanes[i] =
1531 freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1532
1533 if (!outPlanes[i])
1534 return FALSE;
1535 }
1536
1537 return TRUE;
1538}
1539
1540BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1541 const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1542 UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1543 UINT32* WINPR_RESTRICT pDstSize)
1544{
1545 UINT32 size = 0;
1546 BYTE* dstp = nullptr;
1547 UINT32 dstSizes[4] = WINPR_C_ARRAY_INIT;
1548 BYTE FormatHeader = 0;
1549
1550 if (!context || !context->rlePlanesBuffer)
1551 return nullptr;
1552
1553 if (context->AllowSkipAlpha)
1554 FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1555
1556 const UINT32 planeSize = width * height;
1557
1558 if (!context->AllowSkipAlpha)
1559 format = planar_invert_format(context, TRUE, format);
1560
1561 if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1562 context->planes))
1563 return nullptr;
1564
1565 if (context->AllowRunLengthEncoding)
1566 {
1567 if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1568 context->deltaPlanes))
1569 return nullptr;
1570
1571 if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1572 context->rlePlanesBuffer, dstSizes,
1573 context->AllowSkipAlpha))
1574 return nullptr;
1575
1576 {
1577 uint32_t offset = 0;
1578 FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1579 context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1580 offset += dstSizes[0];
1581 context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1582 offset += dstSizes[1];
1583 context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1584 offset += dstSizes[2];
1585 context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1586
1587#if defined(WITH_DEBUG_CODECS)
1588 WLog_DBG(TAG,
1589 "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1590 " / %" PRIu32 "] ",
1591 dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1592#endif
1593 }
1594 }
1595
1596 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1597 {
1598 if (!context->AllowRunLengthEncoding)
1599 return nullptr;
1600
1601 if (context->rlePlanes[0] == nullptr)
1602 return nullptr;
1603
1604 if (context->rlePlanes[1] == nullptr)
1605 return nullptr;
1606
1607 if (context->rlePlanes[2] == nullptr)
1608 return nullptr;
1609
1610 if (context->rlePlanes[3] == nullptr)
1611 return nullptr;
1612 }
1613
1614 if (!dstData)
1615 {
1616 size = 1;
1617
1618 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1619 {
1620 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1621 size += dstSizes[0];
1622 else
1623 size += planeSize;
1624 }
1625
1626 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1627 size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1628 else
1629 size += (planeSize * 3);
1630
1631 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1632 size++;
1633
1634 dstData = malloc(size);
1635
1636 if (!dstData)
1637 return nullptr;
1638
1639 *pDstSize = size;
1640 }
1641
1642 dstp = dstData;
1643 *dstp = FormatHeader; /* FormatHeader */
1644 dstp++;
1645
1646 /* AlphaPlane */
1647
1648 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1649 {
1650 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1651 {
1652 CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1653 dstp += dstSizes[0];
1654 }
1655 else
1656 {
1657 CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1658 dstp += planeSize;
1659 }
1660 }
1661
1662 /* LumaOrRedPlane */
1663
1664 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1665 {
1666 CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1667 dstp += dstSizes[1];
1668 }
1669 else
1670 {
1671 CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1672 dstp += planeSize;
1673 }
1674
1675 /* OrangeChromaOrGreenPlane */
1676
1677 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1678 {
1679 CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1680 dstp += dstSizes[2];
1681 }
1682 else
1683 {
1684 CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1685 dstp += planeSize;
1686 }
1687
1688 /* GreenChromeOrBluePlane */
1689
1690 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1691 {
1692 CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1693 dstp += dstSizes[3];
1694 }
1695 else
1696 {
1697 CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1698 dstp += planeSize;
1699 }
1700
1701 /* Pad1 (1 byte) */
1702
1703 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1704 {
1705 *dstp = 0;
1706 dstp++;
1707 }
1708
1709 const intptr_t diff = (dstp - dstData);
1710 if ((diff < 0) || (diff > UINT32_MAX))
1711 {
1712 free(dstData);
1713 return nullptr;
1714 }
1715 size = (UINT32)diff;
1716 *pDstSize = size;
1717 return dstData;
1718}
1719
1720BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1721 UINT32 width, UINT32 height)
1722{
1723 if (!context)
1724 return FALSE;
1725
1726 context->bgr = FALSE;
1727 context->maxWidth = PLANAR_ALIGN(width, 4);
1728 context->maxHeight = PLANAR_ALIGN(height, 4);
1729 {
1730 const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1731 if (tmp > UINT32_MAX)
1732 return FALSE;
1733 context->maxPlaneSize = (UINT32)tmp;
1734 }
1735
1736 if (context->maxWidth > UINT32_MAX / 4)
1737 return FALSE;
1738 context->nTempStep = context->maxWidth * 4;
1739
1740 memset((void*)context->planes, 0, sizeof(context->planes));
1741 memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
1742 memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1743
1744 if (context->maxPlaneSize > 0)
1745 {
1746 void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1747 if (!tmp)
1748 return FALSE;
1749 context->planesBuffer = tmp;
1750
1751 tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1752 if (!tmp)
1753 return FALSE;
1754 context->pTempData = tmp;
1755
1756 tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1757 if (!tmp)
1758 return FALSE;
1759 context->deltaPlanesBuffer = tmp;
1760
1761 tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1762 if (!tmp)
1763 return FALSE;
1764 context->rlePlanesBuffer = tmp;
1765
1766 context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1767 context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1768 context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1769 context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1770 context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1771 context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1772 context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1773 context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1774 }
1775 return TRUE;
1776}
1777
1778BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1779 UINT32 maxHeight)
1780{
1781 BITMAP_PLANAR_CONTEXT* context =
1782 (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1783
1784 if (!context)
1785 return nullptr;
1786
1787 if (flags & PLANAR_FORMAT_HEADER_NA)
1788 context->AllowSkipAlpha = TRUE;
1789
1790 if (flags & PLANAR_FORMAT_HEADER_RLE)
1791 context->AllowRunLengthEncoding = TRUE;
1792
1793 if (flags & PLANAR_FORMAT_HEADER_CS)
1794 context->AllowColorSubsampling = TRUE;
1795
1796 context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1797
1798 if (context->ColorLossLevel)
1799 context->AllowDynamicColorFidelity = TRUE;
1800
1801 if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1802 {
1803 WINPR_PRAGMA_DIAG_PUSH
1804 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1805 freerdp_bitmap_planar_context_free(context);
1806 WINPR_PRAGMA_DIAG_POP
1807 return nullptr;
1808 }
1809
1810 return context;
1811}
1812
1813void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1814{
1815 if (!context)
1816 return;
1817
1818 winpr_aligned_free(context->pTempData);
1819 winpr_aligned_free(context->planesBuffer);
1820 winpr_aligned_free(context->deltaPlanesBuffer);
1821 winpr_aligned_free(context->rlePlanesBuffer);
1822 winpr_aligned_free(context);
1823}
1824
1825void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1826{
1827 WINPR_ASSERT(planar);
1828 planar->bgr = bgr;
1829}
1830
1831void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1832{
1833 WINPR_ASSERT(planar);
1834 planar->topdown = topdown;
1835}
primitives_t
Definition primitives.h:255