FreeRDP
Loading...
Searching...
No Matches
prim_YUV_sse4.1.c
1
23#include <winpr/wtypes.h>
24#include <freerdp/config.h>
25
26#include <winpr/sysinfo.h>
27#include <winpr/crt.h>
28#include <freerdp/types.h>
29#include <freerdp/primitives.h>
30
31#include "prim_internal.h"
32#include "prim_avxsse.h"
33#include "prim_YUV.h"
34
35#if defined(SSE_AVX_INTRINSICS_ENABLED)
36#include <emmintrin.h>
37#include <tmmintrin.h>
38#include <smmintrin.h>
39
40static primitives_t* generic = nullptr;
41
42/****************************************************************************/
43/* sse41 YUV420 -> RGB conversion */
44/****************************************************************************/
45static inline __m128i* sse41_YUV444Pixel(__m128i* WINPR_RESTRICT dst, __m128i Yraw, __m128i Uraw,
46 __m128i Vraw, UINT8 pos)
47{
48 const __m128i mapY[] = { mm_set_epu32(0x80800380, 0x80800280, 0x80800180, 0x80800080),
49 mm_set_epu32(0x80800780, 0x80800680, 0x80800580, 0x80800480),
50 mm_set_epu32(0x80800B80, 0x80800A80, 0x80800980, 0x80800880),
51 mm_set_epu32(0x80800F80, 0x80800E80, 0x80800D80, 0x80800C80) };
52 const __m128i mapUV[] = { mm_set_epu32(0x80038002, 0x80018000, 0x80808080, 0x80808080),
53 mm_set_epu32(0x80078006, 0x80058004, 0x80808080, 0x80808080),
54 mm_set_epu32(0x800B800A, 0x80098008, 0x80808080, 0x80808080),
55 mm_set_epu32(0x800F800E, 0x800D800C, 0x80808080, 0x80808080) };
56 const __m128i mask[] = { mm_set_epu32(0x80038080, 0x80028080, 0x80018080, 0x80008080),
57 mm_set_epu32(0x80800380, 0x80800280, 0x80800180, 0x80800080),
58 mm_set_epu32(0x80808003, 0x80808002, 0x80808001, 0x80808000) };
59 const __m128i c128 = _mm_set1_epi16(128);
60 __m128i BGRX = _mm_and_si128(LOAD_SI128(dst),
61 mm_set_epu32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000));
62 {
63 __m128i C;
64 __m128i D;
65 __m128i E;
66 /* Load Y values and expand to 32 bit */
67 {
68 C = _mm_shuffle_epi8(Yraw, mapY[pos]); /* Reorder and multiply by 256 */
69 }
70 /* Load U values and expand to 32 bit */
71 {
72 const __m128i U = _mm_shuffle_epi8(Uraw, mapUV[pos]); /* Reorder dcba */
73 D = _mm_sub_epi16(U, c128); /* D = U - 128 */
74 }
75 /* Load V values and expand to 32 bit */
76 {
77 const __m128i V = _mm_shuffle_epi8(Vraw, mapUV[pos]); /* Reorder dcba */
78 E = _mm_sub_epi16(V, c128); /* E = V - 128 */
79 }
80 /* Get the R value */
81 {
82 const __m128i c403 = _mm_set1_epi16(403);
83 const __m128i e403 =
84 _mm_unpackhi_epi16(_mm_mullo_epi16(E, c403), _mm_mulhi_epi16(E, c403));
85 const __m128i Rs = _mm_add_epi32(C, e403);
86 const __m128i R32 = _mm_srai_epi32(Rs, 8);
87 const __m128i R16 = _mm_packs_epi32(R32, _mm_setzero_si128());
88 const __m128i R = _mm_packus_epi16(R16, _mm_setzero_si128());
89 const __m128i packed = _mm_shuffle_epi8(R, mask[0]);
90 BGRX = _mm_or_si128(BGRX, packed);
91 }
92 /* Get the G value */
93 {
94 const __m128i c48 = _mm_set1_epi16(48);
95 const __m128i d48 =
96 _mm_unpackhi_epi16(_mm_mullo_epi16(D, c48), _mm_mulhi_epi16(D, c48));
97 const __m128i c120 = _mm_set1_epi16(120);
98 const __m128i e120 =
99 _mm_unpackhi_epi16(_mm_mullo_epi16(E, c120), _mm_mulhi_epi16(E, c120));
100 const __m128i de = _mm_add_epi32(d48, e120);
101 const __m128i Gs = _mm_sub_epi32(C, de);
102 const __m128i G32 = _mm_srai_epi32(Gs, 8);
103 const __m128i G16 = _mm_packs_epi32(G32, _mm_setzero_si128());
104 const __m128i G = _mm_packus_epi16(G16, _mm_setzero_si128());
105 const __m128i packed = _mm_shuffle_epi8(G, mask[1]);
106 BGRX = _mm_or_si128(BGRX, packed);
107 }
108 /* Get the B value */
109 {
110 const __m128i c475 = _mm_set1_epi16(475);
111 const __m128i d475 =
112 _mm_unpackhi_epi16(_mm_mullo_epi16(D, c475), _mm_mulhi_epi16(D, c475));
113 const __m128i Bs = _mm_add_epi32(C, d475);
114 const __m128i B32 = _mm_srai_epi32(Bs, 8);
115 const __m128i B16 = _mm_packs_epi32(B32, _mm_setzero_si128());
116 const __m128i B = _mm_packus_epi16(B16, _mm_setzero_si128());
117 const __m128i packed = _mm_shuffle_epi8(B, mask[2]);
118 BGRX = _mm_or_si128(BGRX, packed);
119 }
120 }
121 STORE_SI128(dst++, BGRX);
122 return dst;
123}
124
125static inline pstatus_t sse41_YUV420ToRGB_BGRX(const BYTE* WINPR_RESTRICT pSrc[],
126 const UINT32* WINPR_RESTRICT srcStep,
127 BYTE* WINPR_RESTRICT pDst, UINT32 dstStep,
128 const prim_size_t* WINPR_RESTRICT roi)
129{
130 const UINT32 nWidth = roi->width;
131 const UINT32 nHeight = roi->height;
132 const UINT32 pad = roi->width % 16;
133 const __m128i duplicate = _mm_set_epi8(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
134
135 for (size_t y = 0; y < nHeight; y++)
136 {
137 __m128i* dst = WINPR_PACKED_ALIGN_CAST(__m128i*, (pDst + dstStep * y));
138 const BYTE* YData = pSrc[0] + y * srcStep[0];
139 const BYTE* UData = pSrc[1] + (y / 2) * srcStep[1];
140 const BYTE* VData = pSrc[2] + (y / 2) * srcStep[2];
141
142 for (UINT32 x = 0; x < nWidth - pad; x += 16)
143 {
144 const __m128i Y = LOAD_SI128(YData);
145 const __m128i uRaw = LOAD_SI128(UData);
146 const __m128i vRaw = LOAD_SI128(VData);
147 const __m128i U = _mm_shuffle_epi8(uRaw, duplicate);
148 const __m128i V = _mm_shuffle_epi8(vRaw, duplicate);
149 YData += 16;
150 UData += 8;
151 VData += 8;
152 dst = sse41_YUV444Pixel(dst, Y, U, V, 0);
153 dst = sse41_YUV444Pixel(dst, Y, U, V, 1);
154 dst = sse41_YUV444Pixel(dst, Y, U, V, 2);
155 dst = sse41_YUV444Pixel(dst, Y, U, V, 3);
156 }
157
158 for (UINT32 x = 0; x < pad; x++)
159 {
160 const BYTE Y = *YData++;
161 const BYTE U = *UData;
162 const BYTE V = *VData;
163 dst = WINPR_PACKED_ALIGN_CAST(
164 __m128i*, writeYUVPixel((BYTE*)dst, PIXEL_FORMAT_BGRX32, Y, U, V, writePixelBGRX));
165
166 if (x % 2)
167 {
168 UData++;
169 VData++;
170 }
171 }
172 }
173
174 return PRIMITIVES_SUCCESS;
175}
176
177static pstatus_t sse41_YUV420ToRGB(const BYTE* WINPR_RESTRICT pSrc[3], const UINT32 srcStep[3],
178 BYTE* WINPR_RESTRICT pDst, UINT32 dstStep, UINT32 DstFormat,
179 const prim_size_t* WINPR_RESTRICT roi)
180{
181 switch (DstFormat)
182 {
183 case PIXEL_FORMAT_BGRX32:
184 case PIXEL_FORMAT_BGRA32:
185 return sse41_YUV420ToRGB_BGRX(pSrc, srcStep, pDst, dstStep, roi);
186
187 default:
188 return generic->YUV420ToRGB_8u_P3AC4R(pSrc, srcStep, pDst, dstStep, DstFormat, roi);
189 }
190}
191
192static inline void BGRX_fillRGB(size_t offset, BYTE* WINPR_RESTRICT pRGB[2],
193 const BYTE* WINPR_RESTRICT pY[2], const BYTE* WINPR_RESTRICT pU[2],
194 const BYTE* WINPR_RESTRICT pV[2], BOOL filter)
195{
196 WINPR_ASSERT(pRGB);
197 WINPR_ASSERT(pY);
198 WINPR_ASSERT(pU);
199 WINPR_ASSERT(pV);
200
201 const UINT32 DstFormat = PIXEL_FORMAT_BGRX32;
202 const UINT32 bpp = 4;
203
204 for (size_t i = 0; i < 2; i++)
205 {
206 for (size_t j = 0; j < 2; j++)
207 {
208 const BYTE Y = pY[i][offset + j];
209 BYTE U = pU[i][offset + j];
210 BYTE V = pV[i][offset + j];
211 if ((i == 0) && (j == 0) && filter)
212 {
213 const INT32 avgU =
214 4 * pU[0][offset] - pU[0][offset + 1] - pU[1][offset] - pU[1][offset + 1];
215 const INT32 avgV =
216 4 * pV[0][offset] - pV[0][offset + 1] - pV[1][offset] - pV[1][offset + 1];
217
218 U = CONDITIONAL_CLIP(avgU, pU[0][offset]);
219 V = CONDITIONAL_CLIP(avgV, pV[0][offset]);
220 }
221
222 writeYUVPixel(&pRGB[i][(j + offset) * bpp], DstFormat, Y, U, V, writePixelBGRX);
223 }
224 }
225}
226
227/* input are uint16_t vectors */
228static inline __m128i sse41_yuv2x_single(const __m128i Y, __m128i U, __m128i V, const short iMulU,
229 const short iMulV)
230{
231 const __m128i zero = _mm_set1_epi8(0);
232
233 __m128i Ylo = _mm_unpacklo_epi16(Y, zero);
234 __m128i Yhi = _mm_unpackhi_epi16(Y, zero);
235 if (iMulU != 0)
236 {
237 const __m128i addX = _mm_set1_epi16(128);
238 const __m128i D = _mm_sub_epi16(U, addX);
239 const __m128i mulU = _mm_set1_epi16(iMulU);
240 const __m128i mulDlo = _mm_mullo_epi16(D, mulU);
241 const __m128i mulDhi = _mm_mulhi_epi16(D, mulU);
242 const __m128i Dlo = _mm_unpacklo_epi16(mulDlo, mulDhi);
243 Ylo = _mm_add_epi32(Ylo, Dlo);
244
245 const __m128i Dhi = _mm_unpackhi_epi16(mulDlo, mulDhi);
246 Yhi = _mm_add_epi32(Yhi, Dhi);
247 }
248 if (iMulV != 0)
249 {
250 const __m128i addX = _mm_set1_epi16(128);
251 const __m128i E = _mm_sub_epi16(V, addX);
252 const __m128i mul = _mm_set1_epi16(iMulV);
253 const __m128i mulElo = _mm_mullo_epi16(E, mul);
254 const __m128i mulEhi = _mm_mulhi_epi16(E, mul);
255 const __m128i Elo = _mm_unpacklo_epi16(mulElo, mulEhi);
256 const __m128i esumlo = _mm_add_epi32(Ylo, Elo);
257
258 const __m128i Ehi = _mm_unpackhi_epi16(mulElo, mulEhi);
259 const __m128i esumhi = _mm_add_epi32(Yhi, Ehi);
260 Ylo = esumlo;
261 Yhi = esumhi;
262 }
263
264 const __m128i rYlo = _mm_srai_epi32(Ylo, 8);
265 const __m128i rYhi = _mm_srai_epi32(Yhi, 8);
266 const __m128i rY = _mm_packs_epi32(rYlo, rYhi);
267 return rY;
268}
269
270/* Input are uint8_t vectors */
271static inline __m128i sse41_yuv2x(const __m128i Y, __m128i U, __m128i V, const short iMulU,
272 const short iMulV)
273{
274 const __m128i zero = _mm_set1_epi8(0);
275
276 /* Ylo = Y * 256
277 * Ulo = uint8_t -> uint16_t
278 * Vlo = uint8_t -> uint16_t
279 */
280 const __m128i Ylo = _mm_unpacklo_epi8(zero, Y);
281 const __m128i Ulo = _mm_unpacklo_epi8(U, zero);
282 const __m128i Vlo = _mm_unpacklo_epi8(V, zero);
283 const __m128i preslo = sse41_yuv2x_single(Ylo, Ulo, Vlo, iMulU, iMulV);
284
285 const __m128i Yhi = _mm_unpackhi_epi8(zero, Y);
286 const __m128i Uhi = _mm_unpackhi_epi8(U, zero);
287 const __m128i Vhi = _mm_unpackhi_epi8(V, zero);
288 const __m128i preshi = sse41_yuv2x_single(Yhi, Uhi, Vhi, iMulU, iMulV);
289 const __m128i res = _mm_packus_epi16(preslo, preshi);
290
291 return res;
292}
293
294/* const INT32 r = ((256L * C(Y) + 0L * D(U) + 403L * E(V))) >> 8; */
295static inline __m128i sse41_yuv2r(const __m128i Y, __m128i U, __m128i V)
296{
297 return sse41_yuv2x(Y, U, V, 0, 403);
298}
299
300/* const INT32 g = ((256L * C(Y) - 48L * D(U) - 120L * E(V))) >> 8; */
301static inline __m128i sse41_yuv2g(const __m128i Y, __m128i U, __m128i V)
302{
303 return sse41_yuv2x(Y, U, V, -48, -120);
304}
305
306/* const INT32 b = ((256L * C(Y) + 475L * D(U) + 0L * E(V))) >> 8; */
307static inline __m128i sse41_yuv2b(const __m128i Y, __m128i U, __m128i V)
308{
309 return sse41_yuv2x(Y, U, V, 475, 0);
310}
311
312static inline void sse41_BGRX_fillRGB_pixel(BYTE* WINPR_RESTRICT pRGB, __m128i Y, __m128i U,
313 __m128i V)
314{
315 const __m128i zero = _mm_set1_epi8(0);
316 /* Y * 256 */
317 const __m128i r = sse41_yuv2r(Y, U, V);
318 const __m128i rx[2] = { _mm_unpackhi_epi8(r, zero), _mm_unpacklo_epi8(r, zero) };
319
320 const __m128i g = sse41_yuv2g(Y, U, V);
321 const __m128i b = sse41_yuv2b(Y, U, V);
322
323 const __m128i bg[2] = { _mm_unpackhi_epi8(b, g), _mm_unpacklo_epi8(b, g) };
324
325 const __m128i mask = mm_set_epu8(0x00, 0xFF, 0xFF, 0xFF, 0x00, 0xFF, 0xFF, 0xFF, 0x00, 0xFF,
326 0xFF, 0xFF, 0x00, 0xFF, 0xFF, 0xFF);
327
328 __m128i* rgb = WINPR_PACKED_ALIGN_CAST(__m128i*, pRGB);
329 const __m128i bgrx0 = _mm_unpacklo_epi16(bg[1], rx[1]);
330 _mm_maskmoveu_si128(bgrx0, mask, (char*)&rgb[0]);
331 const __m128i bgrx1 = _mm_unpackhi_epi16(bg[1], rx[1]);
332 _mm_maskmoveu_si128(bgrx1, mask, (char*)&rgb[1]);
333 const __m128i bgrx2 = _mm_unpacklo_epi16(bg[0], rx[0]);
334 _mm_maskmoveu_si128(bgrx2, mask, (char*)&rgb[2]);
335 const __m128i bgrx3 = _mm_unpackhi_epi16(bg[0], rx[0]);
336 _mm_maskmoveu_si128(bgrx3, mask, (char*)&rgb[3]);
337}
338
339static inline __m128i odd1sum(__m128i u1)
340{
341 const __m128i zero = _mm_set1_epi8(0);
342 const __m128i u1hi = _mm_unpackhi_epi8(u1, zero);
343 const __m128i u1lo = _mm_unpacklo_epi8(u1, zero);
344 return _mm_hadds_epi16(u1lo, u1hi);
345}
346
347static inline __m128i odd0sum(__m128i u0, __m128i u1sum)
348{
349 /* Mask out even bytes, extend uint8_t to uint16_t by filling in zero bytes,
350 * horizontally add the values */
351 const __m128i mask = mm_set_epu8(0x80, 0x0F, 0x80, 0x0D, 0x80, 0x0B, 0x80, 0x09, 0x80, 0x07,
352 0x80, 0x05, 0x80, 0x03, 0x80, 0x01);
353 const __m128i u0odd = _mm_shuffle_epi8(u0, mask);
354 return _mm_adds_epi16(u1sum, u0odd);
355}
356
357static inline __m128i calcavg(__m128i u0even, __m128i sum)
358{
359 const __m128i u4zero = _mm_slli_epi16(u0even, 2);
360 const __m128i uavg = _mm_sub_epi16(u4zero, sum);
361 const __m128i zero = _mm_set1_epi8(0);
362 const __m128i savg = _mm_packus_epi16(uavg, zero);
363 const __m128i smask = mm_set_epu8(0x80, 0x07, 0x80, 0x06, 0x80, 0x05, 0x80, 0x04, 0x80, 0x03,
364 0x80, 0x02, 0x80, 0x01, 0x80, 0x00);
365 return _mm_shuffle_epi8(savg, smask);
366}
367
368static inline __m128i diffmask(__m128i avg, __m128i u0even)
369{
370 /* Check for values >= 30 to apply the avg value to
371 * use int16 for calculations to avoid issues with signed 8bit integers
372 */
373 const __m128i diff = _mm_subs_epi16(u0even, avg);
374 const __m128i absdiff = _mm_abs_epi16(diff);
375 const __m128i val30 = _mm_set1_epi16(30);
376 return _mm_cmplt_epi16(absdiff, val30);
377}
378
379static inline void sse41_filter(__m128i pU[2])
380{
381 const __m128i u1sum = odd1sum(pU[1]);
382 const __m128i sum = odd0sum(pU[0], u1sum);
383
384 /* Mask out the odd bytes. We don´t need to do anything to make the uint8_t to uint16_t */
385 const __m128i emask = mm_set_epu8(0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff,
386 0x00, 0xff, 0x00, 0xff, 0x00, 0xff);
387 const __m128i u0even = _mm_and_si128(pU[0], emask);
388 const __m128i avg = calcavg(u0even, sum);
389 const __m128i umask = diffmask(avg, u0even);
390
391 const __m128i u0orig = _mm_and_si128(u0even, umask);
392 const __m128i u0avg = _mm_andnot_si128(umask, avg);
393 const __m128i evenresult = _mm_or_si128(u0orig, u0avg);
394 const __m128i omask = mm_set_epu8(0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00,
395 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00);
396 const __m128i u0odd = _mm_and_si128(pU[0], omask);
397 const __m128i result = _mm_or_si128(evenresult, u0odd);
398 pU[0] = result;
399}
400
401static inline void sse41_BGRX_fillRGB(BYTE* WINPR_RESTRICT pRGB[2], const __m128i pY[2],
402 __m128i pU[2], __m128i pV[2])
403{
404 WINPR_ASSERT(pRGB);
405 WINPR_ASSERT(pY);
406 WINPR_ASSERT(pU);
407 WINPR_ASSERT(pV);
408
409 sse41_filter(pU);
410 sse41_filter(pV);
411
412 for (size_t i = 0; i < 2; i++)
413 {
414 sse41_BGRX_fillRGB_pixel(pRGB[i], pY[i], pU[i], pV[i]);
415 }
416}
417
418static inline pstatus_t sse41_YUV444ToRGB_8u_P3AC4R_BGRX_DOUBLE_ROW(
419 BYTE* WINPR_RESTRICT pDst[2], const BYTE* WINPR_RESTRICT YData[2],
420 const BYTE* WINPR_RESTRICT UData[2], const BYTE* WINPR_RESTRICT VData[2], UINT32 nWidth)
421{
422 const UINT32 pad = nWidth % 16;
423
424 size_t x = 0;
425 for (; x < nWidth - pad; x += 16)
426 {
427 const __m128i Y[] = { LOAD_SI128(&YData[0][x]), LOAD_SI128(&YData[1][x]) };
428 __m128i U[] = { LOAD_SI128(&UData[0][x]), LOAD_SI128(&UData[1][x]) };
429 __m128i V[] = { LOAD_SI128(&VData[0][x]), LOAD_SI128(&VData[1][x]) };
430
431 BYTE* dstp[] = { &pDst[0][x * 4], &pDst[1][x * 4] };
432 sse41_BGRX_fillRGB(dstp, Y, U, V);
433 }
434
435 for (; x < nWidth; x += 2)
436 {
437 BGRX_fillRGB(x, pDst, YData, UData, VData, TRUE);
438 }
439
440 return PRIMITIVES_SUCCESS;
441}
442
443static inline void BGRX_fillRGB_single(size_t offset, BYTE* WINPR_RESTRICT pRGB,
444 const BYTE* WINPR_RESTRICT pY, const BYTE* WINPR_RESTRICT pU,
445 const BYTE* WINPR_RESTRICT pV, WINPR_ATTR_UNUSED BOOL filter)
446{
447 WINPR_ASSERT(pRGB);
448 WINPR_ASSERT(pY);
449 WINPR_ASSERT(pU);
450 WINPR_ASSERT(pV);
451
452 const UINT32 bpp = 4;
453
454 for (size_t j = 0; j < 2; j++)
455 {
456 const BYTE Y = pY[offset + j];
457 BYTE U = pU[offset + j];
458 BYTE V = pV[offset + j];
459
460 writeYUVPixel(&pRGB[(j + offset) * bpp], PIXEL_FORMAT_BGRX32, Y, U, V, writePixelBGRX);
461 }
462}
463
464static inline pstatus_t sse41_YUV444ToRGB_8u_P3AC4R_BGRX_SINGLE_ROW(
465 BYTE* WINPR_RESTRICT pDst, const BYTE* WINPR_RESTRICT YData, const BYTE* WINPR_RESTRICT UData,
466 const BYTE* WINPR_RESTRICT VData, UINT32 nWidth)
467{
468 for (size_t x = 0; x < nWidth; x += 2)
469 {
470 BGRX_fillRGB_single(x, pDst, YData, UData, VData, TRUE);
471 }
472
473 return PRIMITIVES_SUCCESS;
474}
475
476static inline pstatus_t sse41_YUV444ToRGB_8u_P3AC4R_BGRX(const BYTE* WINPR_RESTRICT pSrc[],
477 const UINT32 srcStep[],
478 BYTE* WINPR_RESTRICT pDst, UINT32 dstStep,
479 const prim_size_t* WINPR_RESTRICT roi)
480{
481 const UINT32 nWidth = roi->width;
482 const UINT32 nHeight = roi->height;
483
484 size_t y = 0;
485 for (; y < nHeight - nHeight % 2; y += 2)
486 {
487 BYTE* dst[] = { (pDst + dstStep * y), (pDst + dstStep * (y + 1)) };
488 const BYTE* YData[] = { pSrc[0] + y * srcStep[0], pSrc[0] + (y + 1) * srcStep[0] };
489 const BYTE* UData[] = { pSrc[1] + y * srcStep[1], pSrc[1] + (y + 1) * srcStep[1] };
490 const BYTE* VData[] = { pSrc[2] + y * srcStep[2], pSrc[2] + (y + 1) * srcStep[2] };
491
492 const pstatus_t rc =
493 sse41_YUV444ToRGB_8u_P3AC4R_BGRX_DOUBLE_ROW(dst, YData, UData, VData, nWidth);
494 if (rc != PRIMITIVES_SUCCESS)
495 return rc;
496 }
497 for (; y < nHeight; y++)
498 {
499 BYTE* dst = (pDst + dstStep * y);
500 const BYTE* YData = pSrc[0] + y * srcStep[0];
501 const BYTE* UData = pSrc[1] + y * srcStep[1];
502 const BYTE* VData = pSrc[2] + y * srcStep[2];
503 const pstatus_t rc =
504 sse41_YUV444ToRGB_8u_P3AC4R_BGRX_SINGLE_ROW(dst, YData, UData, VData, nWidth);
505 if (rc != PRIMITIVES_SUCCESS)
506 return rc;
507 }
508
509 return PRIMITIVES_SUCCESS;
510}
511
512static pstatus_t sse41_YUV444ToRGB_8u_P3AC4R(const BYTE* WINPR_RESTRICT pSrc[],
513 const UINT32 srcStep[], BYTE* WINPR_RESTRICT pDst,
514 UINT32 dstStep, UINT32 DstFormat,
515 const prim_size_t* WINPR_RESTRICT roi)
516{
517 switch (DstFormat)
518 {
519 case PIXEL_FORMAT_BGRX32:
520 case PIXEL_FORMAT_BGRA32:
521 return sse41_YUV444ToRGB_8u_P3AC4R_BGRX(pSrc, srcStep, pDst, dstStep, roi);
522
523 default:
524 return generic->YUV444ToRGB_8u_P3AC4R(pSrc, srcStep, pDst, dstStep, DstFormat, roi);
525 }
526}
527
528static inline void sse41_I444ToRGB_BGRX_ROW(BYTE* WINPR_RESTRICT pDst,
529 const BYTE* WINPR_RESTRICT YData,
530 const BYTE* WINPR_RESTRICT UData,
531 const BYTE* WINPR_RESTRICT VData, UINT32 nWidth)
532{
533 const UINT32 pad = nWidth % 16;
534
535 size_t x = 0;
536 for (; x < nWidth - pad; x += 16)
537 {
538 const __m128i Y = LOAD_SI128(&YData[x]);
539 const __m128i U = LOAD_SI128(&UData[x]);
540 const __m128i V = LOAD_SI128(&VData[x]);
541
542 sse41_BGRX_fillRGB_pixel(&pDst[x * 4], Y, U, V);
543 }
544
545 for (; x < nWidth; x++)
546 {
547 writeYUVPixel(&pDst[x * 4], PIXEL_FORMAT_BGRX32, YData[x], UData[x], VData[x],
548 writePixelBGRX);
549 }
550}
551
552static pstatus_t sse41_I444ToRGB_8u(const BYTE* WINPR_RESTRICT pSrc[], const UINT32 srcStep[],
553 BYTE* WINPR_RESTRICT pDst, UINT32 dstStep, UINT32 DstFormat,
554 const prim_size_t* WINPR_RESTRICT roi)
555{
556 switch (DstFormat)
557 {
558 case PIXEL_FORMAT_BGRX32:
559 case PIXEL_FORMAT_BGRA32:
560 for (size_t y = 0; y < roi->height; y++)
561 {
562 BYTE* dst = pDst + dstStep * y;
563 const BYTE* YData = pSrc[0] + y * srcStep[0];
564 const BYTE* UData = pSrc[1] + y * srcStep[1];
565 const BYTE* VData = pSrc[2] + y * srcStep[2];
566
567 sse41_I444ToRGB_BGRX_ROW(dst, YData, UData, VData, roi->width);
568 }
569 return PRIMITIVES_SUCCESS;
570
571 default:
572 return generic->I444ToRGB_8u(pSrc, srcStep, pDst, dstStep, DstFormat, roi);
573 }
574}
575
576/****************************************************************************/
577/* sse41 RGB -> YUV420 conversion **/
578/****************************************************************************/
579
601#define BGRX_Y_FACTORS _mm_set_epi8(0, 27, 92, 9, 0, 27, 92, 9, 0, 27, 92, 9, 0, 27, 92, 9)
602#define BGRX_U_FACTORS \
603 _mm_set_epi8(0, -29, -99, 127, 0, -29, -99, 127, 0, -29, -99, 127, 0, -29, -99, 127)
604#define BGRX_V_FACTORS \
605 _mm_set_epi8(0, 127, -116, -12, 0, 127, -116, -12, 0, 127, -116, -12, 0, 127, -116, -12)
606#define CONST128_FACTORS _mm_set1_epi8(-128)
607
608#define Y_SHIFT 7
609#define U_SHIFT 8
610#define V_SHIFT 8
611
612/*
613TODO:
614RGB[AX] can simply be supported using the following factors. And instead of loading the
615globals directly the functions below could be passed pointers to the correct vectors
616depending on the source picture format.
617
618PRIM_ALIGN_128 static const BYTE rgbx_y_factors[] = {
619 27, 92, 9, 0, 27, 92, 9, 0, 27, 92, 9, 0, 27, 92, 9, 0
620};
621PRIM_ALIGN_128 static const BYTE rgbx_u_factors[] = {
622 -15, -49, 64, 0, -15, -49, 64, 0, -15, -49, 64, 0, -15, -49, 64, 0
623};
624PRIM_ALIGN_128 static const BYTE rgbx_v_factors[] = {
625 64, -58, -6, 0, 64, -58, -6, 0, 64, -58, -6, 0, 64, -58, -6, 0
626};
627*/
628
629static inline void sse41_BGRX_TO_YUV(const BYTE* WINPR_RESTRICT pLine1, BYTE* WINPR_RESTRICT pYLine,
630 BYTE* WINPR_RESTRICT pULine, BYTE* WINPR_RESTRICT pVLine)
631{
632 const BYTE r1 = pLine1[2];
633 const BYTE g1 = pLine1[1];
634 const BYTE b1 = pLine1[0];
635
636 if (pYLine)
637 pYLine[0] = RGB2Y(r1, g1, b1);
638 if (pULine)
639 pULine[0] = RGB2U(r1, g1, b1);
640 if (pVLine)
641 pVLine[0] = RGB2V(r1, g1, b1);
642}
643
644/* compute the luma (Y) component from a single rgb source line */
645
646static inline void sse41_RGBToYUV420_BGRX_Y(const BYTE* WINPR_RESTRICT src, BYTE* dst, UINT32 width)
647{
648 const __m128i y_factors = BGRX_Y_FACTORS;
649 const __m128i* argb = WINPR_PACKED_ALIGN_CAST(const __m128i*, src);
650 __m128i* ydst = WINPR_PACKED_ALIGN_CAST(__m128i*, dst);
651
652 UINT32 x = 0;
653
654 for (; x < width - width % 16; x += 16)
655 {
656 /* store 16 rgba pixels in 4 128 bit registers */
657 __m128i x0 = LOAD_SI128(argb++); // 1st 4 pixels
658 {
659 x0 = _mm_maddubs_epi16(x0, y_factors);
660
661 __m128i x1 = LOAD_SI128(argb++); // 2nd 4 pixels
662 x1 = _mm_maddubs_epi16(x1, y_factors);
663 x0 = _mm_hadds_epi16(x0, x1);
664 x0 = _mm_srli_epi16(x0, Y_SHIFT);
665 }
666
667 __m128i x2 = LOAD_SI128(argb++); // 3rd 4 pixels
668 {
669 x2 = _mm_maddubs_epi16(x2, y_factors);
670
671 __m128i x3 = LOAD_SI128(argb++); // 4th 4 pixels
672 x3 = _mm_maddubs_epi16(x3, y_factors);
673 x2 = _mm_hadds_epi16(x2, x3);
674 x2 = _mm_srli_epi16(x2, Y_SHIFT);
675 }
676
677 x0 = _mm_packus_epi16(x0, x2);
678 /* save to y plane */
679 STORE_SI128(ydst++, x0);
680 }
681
682 for (; x < width; x++)
683 {
684 sse41_BGRX_TO_YUV(&src[4ULL * x], &dst[x], nullptr, nullptr);
685 }
686}
687
688/* compute the chrominance (UV) components from two rgb source lines */
689
690static inline void sse41_RGBToYUV420_BGRX_UV(const BYTE* WINPR_RESTRICT src1,
691 const BYTE* WINPR_RESTRICT src2,
692 BYTE* WINPR_RESTRICT dst1, BYTE* WINPR_RESTRICT dst2,
693 UINT32 width)
694{
695 const __m128i u_factors = BGRX_U_FACTORS;
696 const __m128i v_factors = BGRX_V_FACTORS;
697 const __m128i vector128 = CONST128_FACTORS;
698
699 size_t x = 0;
700
701 for (; x < width - width % 16; x += 16)
702 {
703 const __m128i* rgb1 = WINPR_PACKED_ALIGN_CAST(const __m128i*, &src1[4ULL * x]);
704 const __m128i* rgb2 = WINPR_PACKED_ALIGN_CAST(const __m128i*, &src2[4ULL * x]);
705 __m64* udst = WINPR_PACKED_ALIGN_CAST(__m64*, &dst1[x / 2]);
706 __m64* vdst = WINPR_PACKED_ALIGN_CAST(__m64*, &dst2[x / 2]);
707
708 /* subsample 16x2 pixels into 16x1 pixels */
709 __m128i x0 = LOAD_SI128(&rgb1[0]);
710 __m128i x4 = LOAD_SI128(&rgb2[0]);
711 x0 = _mm_avg_epu8(x0, x4);
712
713 __m128i x1 = LOAD_SI128(&rgb1[1]);
714 x4 = LOAD_SI128(&rgb2[1]);
715 x1 = _mm_avg_epu8(x1, x4);
716
717 __m128i x2 = LOAD_SI128(&rgb1[2]);
718 x4 = LOAD_SI128(&rgb2[2]);
719 x2 = _mm_avg_epu8(x2, x4);
720
721 __m128i x3 = LOAD_SI128(&rgb1[3]);
722 x4 = LOAD_SI128(&rgb2[3]);
723 x3 = _mm_avg_epu8(x3, x4);
724
725 /* subsample these 16x1 pixels into 8x1 pixels */
731 x4 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(x0), _mm_castsi128_ps(x1), 0x88));
732 x0 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(x0), _mm_castsi128_ps(x1), 0xdd));
733 x0 = _mm_avg_epu8(x0, x4);
734 x4 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(x2), _mm_castsi128_ps(x3), 0x88));
735 x1 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(x2), _mm_castsi128_ps(x3), 0xdd));
736 x1 = _mm_avg_epu8(x1, x4);
737 /* multiplications and subtotals */
738 x2 = _mm_maddubs_epi16(x0, u_factors);
739 x3 = _mm_maddubs_epi16(x1, u_factors);
740 x4 = _mm_maddubs_epi16(x0, v_factors);
741 __m128i x5 = _mm_maddubs_epi16(x1, v_factors);
742 /* the total sums */
743 x0 = _mm_hadd_epi16(x2, x3);
744 x1 = _mm_hadd_epi16(x4, x5);
745 /* shift the results */
746 x0 = _mm_srai_epi16(x0, U_SHIFT);
747 x1 = _mm_srai_epi16(x1, V_SHIFT);
748 /* pack the 16 words into bytes */
749 x0 = _mm_packs_epi16(x0, x1);
750 /* add 128 */
751 x0 = _mm_sub_epi8(x0, vector128);
752 /* the lower 8 bytes go to the u plane */
753 _mm_storel_pi(udst, _mm_castsi128_ps(x0));
754 /* the upper 8 bytes go to the v plane */
755 _mm_storeh_pi(vdst, _mm_castsi128_ps(x0));
756 }
757
758 for (; x < width - width % 2; x += 2)
759 {
760 BYTE u[4] = WINPR_C_ARRAY_INIT;
761 BYTE v[4] = WINPR_C_ARRAY_INIT;
762 sse41_BGRX_TO_YUV(&src1[4ULL * x], nullptr, &u[0], &v[0]);
763 sse41_BGRX_TO_YUV(&src1[4ULL * (1ULL + x)], nullptr, &u[1], &v[1]);
764 sse41_BGRX_TO_YUV(&src2[4ULL * x], nullptr, &u[2], &v[2]);
765 sse41_BGRX_TO_YUV(&src2[4ULL * (1ULL + x)], nullptr, &u[3], &v[3]);
766 const INT16 u4 = WINPR_ASSERTING_INT_CAST(INT16, (INT16)u[0] + u[1] + u[2] + u[3]);
767 const INT16 uu = WINPR_ASSERTING_INT_CAST(INT16, u4 / 4);
768 const BYTE u8 = CLIP(uu);
769 dst1[x / 2] = u8;
770
771 const INT16 v4 = WINPR_ASSERTING_INT_CAST(INT16, (INT16)v[0] + v[1] + v[2] + v[3]);
772 const INT16 vu = WINPR_ASSERTING_INT_CAST(INT16, v4 / 4);
773 const BYTE v8 = CLIP(vu);
774 dst2[x / 2] = v8;
775 }
776}
777
778static pstatus_t sse41_RGBToYUV420_BGRX(const BYTE* WINPR_RESTRICT pSrc, UINT32 srcStep,
779 BYTE* WINPR_RESTRICT pDst[], const UINT32 dstStep[],
780 const prim_size_t* WINPR_RESTRICT roi)
781{
782 if (roi->height < 1 || roi->width < 1)
783 {
784 return !PRIMITIVES_SUCCESS;
785 }
786
787 /* The loops below take 2x2 pixel blocks; an odd last column and an odd last row have U and V
788 * too, which they would never reach, so the generic routine does those edges. */
789 const UINT32 width = roi->width - roi->width % 2;
790 const size_t height = roi->height - roi->height % 2;
791
792 for (size_t y = 0; y < height; y += 2)
793 {
794 const BYTE* line1 = &pSrc[y * srcStep];
795 const BYTE* line2 = &pSrc[(1ULL + y) * srcStep];
796 BYTE* ydst1 = &pDst[0][y * dstStep[0]];
797 BYTE* ydst2 = &pDst[0][(1ULL + y) * dstStep[0]];
798 BYTE* udst = &pDst[1][y / 2 * dstStep[1]];
799 BYTE* vdst = &pDst[2][y / 2 * dstStep[2]];
800
801 sse41_RGBToYUV420_BGRX_UV(line1, line2, udst, vdst, width);
802 sse41_RGBToYUV420_BGRX_Y(line1, ydst1, width);
803 sse41_RGBToYUV420_BGRX_Y(line2, ydst2, width);
804 }
805
806 WINPR_ASSERT(generic);
807
808 /* the odd last column, every row */
809 if (width < roi->width)
810 {
811 const BYTE* src = &pSrc[4ULL * width];
812 BYTE* dst[3] = { &pDst[0][width], &pDst[1][width / 2], &pDst[2][width / 2] };
813 const prim_size_t rest = { 1, roi->height };
814 const pstatus_t rc =
815 generic->RGBToYUV420_8u_P3AC4R(src, PIXEL_FORMAT_BGRX32, srcStep, dst, dstStep, &rest);
816 if (rc != PRIMITIVES_SUCCESS)
817 return rc;
818 }
819
820 /* the odd last row, left of that column */
821 if ((height < roi->height) && (width > 0))
822 {
823 const BYTE* src = &pSrc[height * srcStep];
824 BYTE* dst[3] = { &pDst[0][height * dstStep[0]], &pDst[1][(height / 2) * dstStep[1]],
825 &pDst[2][(height / 2) * dstStep[2]] };
826 const prim_size_t rest = { width, 1 };
827 return generic->RGBToYUV420_8u_P3AC4R(src, PIXEL_FORMAT_BGRX32, srcStep, dst, dstStep,
828 &rest);
829 }
830
831 return PRIMITIVES_SUCCESS;
832}
833
834static pstatus_t sse41_RGBToYUV420(const BYTE* WINPR_RESTRICT pSrc, UINT32 srcFormat,
835 UINT32 srcStep, BYTE* WINPR_RESTRICT pDst[],
836 const UINT32 dstStep[], const prim_size_t* WINPR_RESTRICT roi)
837{
838 switch (srcFormat)
839 {
840 case PIXEL_FORMAT_BGRX32:
841 case PIXEL_FORMAT_BGRA32:
842 return sse41_RGBToYUV420_BGRX(pSrc, srcStep, pDst, dstStep, roi);
843
844 default:
845 return generic->RGBToYUV420_8u_P3AC4R(pSrc, srcFormat, srcStep, pDst, dstStep, roi);
846 }
847}
848
849/****************************************************************************/
850/* sse41 RGB -> AVC444-YUV conversion **/
851/****************************************************************************/
852
853static inline void sse41_RGBToAVC444YUV_BGRX_DOUBLE_ROW(
854 const BYTE* WINPR_RESTRICT srcEven, const BYTE* WINPR_RESTRICT srcOdd,
855 BYTE* WINPR_RESTRICT b1Even, BYTE* WINPR_RESTRICT b1Odd, BYTE* WINPR_RESTRICT b2,
856 BYTE* WINPR_RESTRICT b3, BYTE* WINPR_RESTRICT b4, BYTE* WINPR_RESTRICT b5,
857 BYTE* WINPR_RESTRICT b6, BYTE* WINPR_RESTRICT b7, UINT32 width)
858{
859 const __m128i* argbEven = WINPR_PACKED_ALIGN_CAST(const __m128i*, srcEven);
860 const __m128i* argbOdd = WINPR_PACKED_ALIGN_CAST(const __m128i*, srcOdd);
861 const __m128i y_factors = BGRX_Y_FACTORS;
862 const __m128i u_factors = BGRX_U_FACTORS;
863 const __m128i v_factors = BGRX_V_FACTORS;
864 const __m128i vector128 = CONST128_FACTORS;
865
866 UINT32 x = 0;
867 for (; x < width - width % 16; x += 16)
868 {
869 /* store 16 rgba pixels in 4 128 bit registers */
870 const __m128i xe1 = LOAD_SI128(argbEven++); // 1st 4 pixels
871 const __m128i xe2 = LOAD_SI128(argbEven++); // 2nd 4 pixels
872 const __m128i xe3 = LOAD_SI128(argbEven++); // 3rd 4 pixels
873 const __m128i xe4 = LOAD_SI128(argbEven++); // 4th 4 pixels
874 const __m128i xo1 = LOAD_SI128(argbOdd++); // 1st 4 pixels
875 const __m128i xo2 = LOAD_SI128(argbOdd++); // 2nd 4 pixels
876 const __m128i xo3 = LOAD_SI128(argbOdd++); // 3rd 4 pixels
877 const __m128i xo4 = LOAD_SI128(argbOdd++); // 4th 4 pixels
878 {
879 /* Y: multiplications with subtotals and horizontal sums */
880 const __m128i ye1 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe1, y_factors),
881 _mm_maddubs_epi16(xe2, y_factors)),
882 Y_SHIFT);
883 const __m128i ye2 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe3, y_factors),
884 _mm_maddubs_epi16(xe4, y_factors)),
885 Y_SHIFT);
886 const __m128i ye = _mm_packus_epi16(ye1, ye2);
887 const __m128i yo1 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo1, y_factors),
888 _mm_maddubs_epi16(xo2, y_factors)),
889 Y_SHIFT);
890 const __m128i yo2 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo3, y_factors),
891 _mm_maddubs_epi16(xo4, y_factors)),
892 Y_SHIFT);
893 const __m128i yo = _mm_packus_epi16(yo1, yo2);
894 /* store y [b1] */
895 STORE_SI128(b1Even, ye);
896 b1Even += 16;
897
898 if (b1Odd)
899 {
900 STORE_SI128(b1Odd, yo);
901 b1Odd += 16;
902 }
903 }
904 {
905 /* We have now
906 * 16 even U values in ue
907 * 16 odd U values in uo
908 *
909 * We need to split these according to
910 * 3.3.8.3.2 YUV420p Stream Combination for YUV444 mode */
911 __m128i ue;
912 __m128i uo = WINPR_C_ARRAY_INIT;
913 {
914 const __m128i ue1 =
915 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe1, u_factors),
916 _mm_maddubs_epi16(xe2, u_factors)),
917 U_SHIFT);
918 const __m128i ue2 =
919 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe3, u_factors),
920 _mm_maddubs_epi16(xe4, u_factors)),
921 U_SHIFT);
922 ue = _mm_sub_epi8(_mm_packs_epi16(ue1, ue2), vector128);
923 }
924
925 if (b1Odd)
926 {
927 const __m128i uo1 =
928 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo1, u_factors),
929 _mm_maddubs_epi16(xo2, u_factors)),
930 U_SHIFT);
931 const __m128i uo2 =
932 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo3, u_factors),
933 _mm_maddubs_epi16(xo4, u_factors)),
934 U_SHIFT);
935 uo = _mm_sub_epi8(_mm_packs_epi16(uo1, uo2), vector128);
936 }
937
938 /* Now we need the following storage distribution:
939 * 2x 2y -> b2
940 * x 2y+1 -> b4
941 * 2x+1 2y -> b6 */
942 if (b1Odd) /* b2 */
943 {
944 const __m128i ueh = _mm_unpackhi_epi8(ue, _mm_setzero_si128());
945 const __m128i uoh = _mm_unpackhi_epi8(uo, _mm_setzero_si128());
946 const __m128i hi = _mm_add_epi16(ueh, uoh);
947 const __m128i uel = _mm_unpacklo_epi8(ue, _mm_setzero_si128());
948 const __m128i uol = _mm_unpacklo_epi8(uo, _mm_setzero_si128());
949 const __m128i lo = _mm_add_epi16(uel, uol);
950 const __m128i added = _mm_hadd_epi16(lo, hi);
951 const __m128i avg16 = _mm_srai_epi16(added, 2);
952 const __m128i avg = _mm_packus_epi16(avg16, avg16);
953 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, b2), avg);
954 }
955 else
956 {
957 const __m128i mask =
958 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
959 (char)0x80, (char)0x80, (char)0x80, 14, 12, 10, 8, 6, 4, 2, 0);
960 const __m128i ud = _mm_shuffle_epi8(ue, mask);
961 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, b2), ud);
962 }
963
964 b2 += 8;
965
966 if (b1Odd) /* b4 */
967 {
968 STORE_SI128(b4, uo);
969 b4 += 16;
970 }
971
972 {
973 /* b6 */
974 const __m128i mask =
975 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
976 (char)0x80, (char)0x80, (char)0x80, 15, 13, 11, 9, 7, 5, 3, 1);
977 const __m128i ude = _mm_shuffle_epi8(ue, mask);
978 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, b6), ude);
979 b6 += 8;
980 }
981 }
982 {
983 /* We have now
984 * 16 even V values in ue
985 * 16 odd V values in uo
986 *
987 * We need to split these according to
988 * 3.3.8.3.2 YUV420p Stream Combination for YUV444 mode */
989 __m128i ve;
990 __m128i vo = WINPR_C_ARRAY_INIT;
991 {
992 const __m128i ve1 =
993 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe1, v_factors),
994 _mm_maddubs_epi16(xe2, v_factors)),
995 V_SHIFT);
996 const __m128i ve2 =
997 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe3, v_factors),
998 _mm_maddubs_epi16(xe4, v_factors)),
999 V_SHIFT);
1000 ve = _mm_sub_epi8(_mm_packs_epi16(ve1, ve2), vector128);
1001 }
1002
1003 if (b1Odd)
1004 {
1005 const __m128i vo1 =
1006 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo1, v_factors),
1007 _mm_maddubs_epi16(xo2, v_factors)),
1008 V_SHIFT);
1009 const __m128i vo2 =
1010 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo3, v_factors),
1011 _mm_maddubs_epi16(xo4, v_factors)),
1012 V_SHIFT);
1013 vo = _mm_sub_epi8(_mm_packs_epi16(vo1, vo2), vector128);
1014 }
1015
1016 /* Now we need the following storage distribution:
1017 * 2x 2y -> b3
1018 * x 2y+1 -> b5
1019 * 2x+1 2y -> b7 */
1020 if (b1Odd) /* b3 */
1021 {
1022 const __m128i veh = _mm_unpackhi_epi8(ve, _mm_setzero_si128());
1023 const __m128i voh = _mm_unpackhi_epi8(vo, _mm_setzero_si128());
1024 const __m128i hi = _mm_add_epi16(veh, voh);
1025 const __m128i vel = _mm_unpacklo_epi8(ve, _mm_setzero_si128());
1026 const __m128i vol = _mm_unpacklo_epi8(vo, _mm_setzero_si128());
1027 const __m128i lo = _mm_add_epi16(vel, vol);
1028 const __m128i added = _mm_hadd_epi16(lo, hi);
1029 const __m128i avg16 = _mm_srai_epi16(added, 2);
1030 const __m128i avg = _mm_packus_epi16(avg16, avg16);
1031 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, b3), avg);
1032 }
1033 else
1034 {
1035 const __m128i mask =
1036 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1037 (char)0x80, (char)0x80, (char)0x80, 14, 12, 10, 8, 6, 4, 2, 0);
1038 const __m128i vd = _mm_shuffle_epi8(ve, mask);
1039 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, b3), vd);
1040 }
1041
1042 b3 += 8;
1043
1044 if (b1Odd) /* b5 */
1045 {
1046 STORE_SI128(b5, vo);
1047 b5 += 16;
1048 }
1049
1050 {
1051 /* b7 */
1052 const __m128i mask =
1053 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1054 (char)0x80, (char)0x80, (char)0x80, 15, 13, 11, 9, 7, 5, 3, 1);
1055 const __m128i vde = _mm_shuffle_epi8(ve, mask);
1056 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, b7), vde);
1057 b7 += 8;
1058 }
1059 }
1060 }
1061
1062 general_RGBToAVC444YUV_BGRX_DOUBLE_ROW(x, srcEven, srcOdd, b1Even, b1Odd, b2, b3, b4, b5, b6,
1063 b7, width);
1064}
1065
1066static pstatus_t sse41_RGBToAVC444YUV_BGRX(const BYTE* WINPR_RESTRICT pSrc,
1067 WINPR_ATTR_UNUSED UINT32 srcFormat, UINT32 srcStep,
1068 BYTE* WINPR_RESTRICT pDst1[], const UINT32 dst1Step[],
1069 BYTE* WINPR_RESTRICT pDst2[], const UINT32 dst2Step[],
1070 const prim_size_t* WINPR_RESTRICT roi)
1071{
1072 if (roi->height < 1 || roi->width < 1)
1073 return !PRIMITIVES_SUCCESS;
1074
1075 size_t y = 0;
1076 for (; y < roi->height - roi->height % 2; y += 2)
1077 {
1078 const BYTE* srcEven = pSrc + y * srcStep;
1079 const BYTE* srcOdd = pSrc + (y + 1) * srcStep;
1080 const size_t i = y >> 1;
1081 const size_t n = (i & (size_t)~7) + i;
1082 BYTE* b1Even = pDst1[0] + y * dst1Step[0];
1083 BYTE* b1Odd = (b1Even + dst1Step[0]);
1084 BYTE* b2 = pDst1[1] + (y / 2) * dst1Step[1];
1085 BYTE* b3 = pDst1[2] + (y / 2) * dst1Step[2];
1086 BYTE* b4 = pDst2[0] + 1ULL * dst2Step[0] * n;
1087 BYTE* b5 = b4 + 8ULL * dst2Step[0];
1088 BYTE* b6 = pDst2[1] + (y / 2) * dst2Step[1];
1089 BYTE* b7 = pDst2[2] + (y / 2) * dst2Step[2];
1090 sse41_RGBToAVC444YUV_BGRX_DOUBLE_ROW(srcEven, srcOdd, b1Even, b1Odd, b2, b3, b4, b5, b6, b7,
1091 roi->width);
1092 }
1093
1094 for (; y < roi->height; y++)
1095 {
1096 const BYTE* srcEven = pSrc + y * srcStep;
1097 BYTE* b1Even = pDst1[0] + y * dst1Step[0];
1098 BYTE* b2 = pDst1[1] + (y / 2) * dst1Step[1];
1099 BYTE* b3 = pDst1[2] + (y / 2) * dst1Step[2];
1100 BYTE* b6 = pDst2[1] + (y / 2) * dst2Step[1];
1101 BYTE* b7 = pDst2[2] + (y / 2) * dst2Step[2];
1102 general_RGBToAVC444YUV_BGRX_DOUBLE_ROW(0, srcEven, nullptr, b1Even, nullptr, b2, b3,
1103 nullptr, nullptr, b6, b7, roi->width);
1104 }
1105
1106 return PRIMITIVES_SUCCESS;
1107}
1108
1109static pstatus_t sse41_RGBToAVC444YUV(const BYTE* WINPR_RESTRICT pSrc, UINT32 srcFormat,
1110 UINT32 srcStep, BYTE* WINPR_RESTRICT pDst1[],
1111 const UINT32 dst1Step[], BYTE* WINPR_RESTRICT pDst2[],
1112 const UINT32 dst2Step[],
1113 const prim_size_t* WINPR_RESTRICT roi)
1114{
1115 switch (srcFormat)
1116 {
1117 case PIXEL_FORMAT_BGRX32:
1118 case PIXEL_FORMAT_BGRA32:
1119 return sse41_RGBToAVC444YUV_BGRX(pSrc, srcFormat, srcStep, pDst1, dst1Step, pDst2,
1120 dst2Step, roi);
1121
1122 default:
1123 return generic->RGBToAVC444YUV(pSrc, srcFormat, srcStep, pDst1, dst1Step, pDst2,
1124 dst2Step, roi);
1125 }
1126}
1127
1128/* Mapping of arguments:
1129 *
1130 * b1 [even lines] -> yLumaDstEven
1131 * b1 [odd lines] -> yLumaDstOdd
1132 * b2 -> uLumaDst
1133 * b3 -> vLumaDst
1134 * b4 -> yChromaDst1
1135 * b5 -> yChromaDst2
1136 * b6 -> uChromaDst1
1137 * b7 -> uChromaDst2
1138 * b8 -> vChromaDst1
1139 * b9 -> vChromaDst2
1140 */
1141static inline void sse41_RGBToAVC444YUVv2_BGRX_DOUBLE_ROW(
1142 const BYTE* WINPR_RESTRICT srcEven, const BYTE* WINPR_RESTRICT srcOdd,
1143 BYTE* WINPR_RESTRICT yLumaDstEven, BYTE* WINPR_RESTRICT yLumaDstOdd,
1144 BYTE* WINPR_RESTRICT uLumaDst, BYTE* WINPR_RESTRICT vLumaDst,
1145 BYTE* WINPR_RESTRICT yEvenChromaDst1, BYTE* WINPR_RESTRICT yEvenChromaDst2,
1146 BYTE* WINPR_RESTRICT yOddChromaDst1, BYTE* WINPR_RESTRICT yOddChromaDst2,
1147 BYTE* WINPR_RESTRICT uChromaDst1, BYTE* WINPR_RESTRICT uChromaDst2,
1148 BYTE* WINPR_RESTRICT vChromaDst1, BYTE* WINPR_RESTRICT vChromaDst2, UINT32 width)
1149{
1150 const __m128i vector128 = CONST128_FACTORS;
1151 const __m128i* argbEven = WINPR_PACKED_ALIGN_CAST(const __m128i*, srcEven);
1152 const __m128i* argbOdd = WINPR_PACKED_ALIGN_CAST(const __m128i*, srcOdd);
1153
1154 UINT32 x = 0;
1155 for (; x < width - width % 16; x += 16)
1156 {
1157 /* store 16 rgba pixels in 4 128 bit registers
1158 * for even and odd rows.
1159 */
1160 const __m128i xe1 = LOAD_SI128(argbEven++); /* 1st 4 pixels */
1161 const __m128i xe2 = LOAD_SI128(argbEven++); /* 2nd 4 pixels */
1162 const __m128i xe3 = LOAD_SI128(argbEven++); /* 3rd 4 pixels */
1163 const __m128i xe4 = LOAD_SI128(argbEven++); /* 4th 4 pixels */
1164 const __m128i xo1 = LOAD_SI128(argbOdd++); /* 1st 4 pixels */
1165 const __m128i xo2 = LOAD_SI128(argbOdd++); /* 2nd 4 pixels */
1166 const __m128i xo3 = LOAD_SI128(argbOdd++); /* 3rd 4 pixels */
1167 const __m128i xo4 = LOAD_SI128(argbOdd++); /* 4th 4 pixels */
1168 {
1169 /* Y: multiplications with subtotals and horizontal sums */
1170 const __m128i y_factors = BGRX_Y_FACTORS;
1171 const __m128i ye1 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe1, y_factors),
1172 _mm_maddubs_epi16(xe2, y_factors)),
1173 Y_SHIFT);
1174 const __m128i ye2 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe3, y_factors),
1175 _mm_maddubs_epi16(xe4, y_factors)),
1176 Y_SHIFT);
1177 const __m128i ye = _mm_packus_epi16(ye1, ye2);
1178 /* store y [b1] */
1179 STORE_SI128(yLumaDstEven, ye);
1180 yLumaDstEven += 16;
1181 }
1182
1183 if (yLumaDstOdd)
1184 {
1185 const __m128i y_factors = BGRX_Y_FACTORS;
1186 const __m128i yo1 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo1, y_factors),
1187 _mm_maddubs_epi16(xo2, y_factors)),
1188 Y_SHIFT);
1189 const __m128i yo2 = _mm_srli_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo3, y_factors),
1190 _mm_maddubs_epi16(xo4, y_factors)),
1191 Y_SHIFT);
1192 const __m128i yo = _mm_packus_epi16(yo1, yo2);
1193 STORE_SI128(yLumaDstOdd, yo);
1194 yLumaDstOdd += 16;
1195 }
1196
1197 {
1198 /* We have now
1199 * 16 even U values in ue
1200 * 16 odd U values in uo
1201 *
1202 * We need to split these according to
1203 * 3.3.8.3.3 YUV420p Stream Combination for YUV444v2 mode */
1204 /* U: multiplications with subtotals and horizontal sums */
1205 __m128i ue;
1206 __m128i uo;
1207 __m128i uavg;
1208 {
1209 const __m128i u_factors = BGRX_U_FACTORS;
1210 const __m128i ue1 =
1211 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe1, u_factors),
1212 _mm_maddubs_epi16(xe2, u_factors)),
1213 U_SHIFT);
1214 const __m128i ue2 =
1215 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe3, u_factors),
1216 _mm_maddubs_epi16(xe4, u_factors)),
1217 U_SHIFT);
1218 const __m128i ueavg = _mm_hadd_epi16(ue1, ue2);
1219 ue = _mm_sub_epi8(_mm_packs_epi16(ue1, ue2), vector128);
1220 uavg = ueavg;
1221 }
1222 {
1223 const __m128i u_factors = BGRX_U_FACTORS;
1224 const __m128i uo1 =
1225 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo1, u_factors),
1226 _mm_maddubs_epi16(xo2, u_factors)),
1227 U_SHIFT);
1228 const __m128i uo2 =
1229 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo3, u_factors),
1230 _mm_maddubs_epi16(xo4, u_factors)),
1231 U_SHIFT);
1232 const __m128i uoavg = _mm_hadd_epi16(uo1, uo2);
1233 uo = _mm_sub_epi8(_mm_packs_epi16(uo1, uo2), vector128);
1234 uavg = _mm_add_epi16(uavg, uoavg);
1235 uavg = _mm_srai_epi16(uavg, 2);
1236 uavg = _mm_packs_epi16(uavg, uoavg);
1237 uavg = _mm_sub_epi8(uavg, vector128);
1238 }
1239 /* Now we need the following storage distribution:
1240 * 2x 2y -> uLumaDst
1241 * 2x+1 y -> yChromaDst1
1242 * 4x 2y+1 -> uChromaDst1
1243 * 4x+2 2y+1 -> vChromaDst1 */
1244 {
1245 const __m128i mask =
1246 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1247 (char)0x80, (char)0x80, (char)0x80, 15, 13, 11, 9, 7, 5, 3, 1);
1248 const __m128i ude = _mm_shuffle_epi8(ue, mask);
1249 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, yEvenChromaDst1), ude);
1250 yEvenChromaDst1 += 8;
1251 }
1252
1253 if (yLumaDstOdd)
1254 {
1255 const __m128i mask =
1256 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1257 (char)0x80, (char)0x80, (char)0x80, 15, 13, 11, 9, 7, 5, 3, 1);
1258 const __m128i udo /* codespell:ignore udo */ = _mm_shuffle_epi8(uo, mask);
1259 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, yOddChromaDst1),
1260 udo); // codespell:ignore udo
1261 yOddChromaDst1 += 8;
1262 }
1263
1264 if (yLumaDstOdd)
1265 {
1266 const __m128i mask =
1267 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1268 (char)0x80, (char)0x80, (char)0x80, 14, 10, 6, 2, 12, 8, 4, 0);
1269 const __m128i ud = _mm_shuffle_epi8(uo, mask);
1270 int* uDst1 = WINPR_PACKED_ALIGN_CAST(int*, uChromaDst1);
1271 int* vDst1 = WINPR_PACKED_ALIGN_CAST(int*, vChromaDst1);
1272 const int* src = (const int*)&ud;
1273 _mm_stream_si32(uDst1, src[0]);
1274 _mm_stream_si32(vDst1, src[1]);
1275 uChromaDst1 += 4;
1276 vChromaDst1 += 4;
1277 }
1278
1279 if (yLumaDstOdd)
1280 {
1281 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, uLumaDst), uavg);
1282 uLumaDst += 8;
1283 }
1284 else
1285 {
1286 const __m128i mask =
1287 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1288 (char)0x80, (char)0x80, (char)0x80, 14, 12, 10, 8, 6, 4, 2, 0);
1289 const __m128i ud = _mm_shuffle_epi8(ue, mask);
1290 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, uLumaDst), ud);
1291 uLumaDst += 8;
1292 }
1293 }
1294
1295 {
1296 /* V: multiplications with subtotals and horizontal sums */
1297 __m128i ve;
1298 __m128i vo;
1299 __m128i vavg;
1300 {
1301 const __m128i v_factors = BGRX_V_FACTORS;
1302 const __m128i ve1 =
1303 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe1, v_factors),
1304 _mm_maddubs_epi16(xe2, v_factors)),
1305 V_SHIFT);
1306 const __m128i ve2 =
1307 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xe3, v_factors),
1308 _mm_maddubs_epi16(xe4, v_factors)),
1309 V_SHIFT);
1310 const __m128i veavg = _mm_hadd_epi16(ve1, ve2);
1311 ve = _mm_sub_epi8(_mm_packs_epi16(ve1, ve2), vector128);
1312 vavg = veavg;
1313 }
1314 {
1315 const __m128i v_factors = BGRX_V_FACTORS;
1316 const __m128i vo1 =
1317 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo1, v_factors),
1318 _mm_maddubs_epi16(xo2, v_factors)),
1319 V_SHIFT);
1320 const __m128i vo2 =
1321 _mm_srai_epi16(_mm_hadd_epi16(_mm_maddubs_epi16(xo3, v_factors),
1322 _mm_maddubs_epi16(xo4, v_factors)),
1323 V_SHIFT);
1324 const __m128i voavg = _mm_hadd_epi16(vo1, vo2);
1325 vo = _mm_sub_epi8(_mm_packs_epi16(vo1, vo2), vector128);
1326 vavg = _mm_add_epi16(vavg, voavg);
1327 vavg = _mm_srai_epi16(vavg, 2);
1328 vavg = _mm_packs_epi16(vavg, voavg);
1329 vavg = _mm_sub_epi8(vavg, vector128);
1330 }
1331 /* Now we need the following storage distribution:
1332 * 2x 2y -> vLumaDst
1333 * 2x+1 y -> yChromaDst2
1334 * 4x 2y+1 -> uChromaDst2
1335 * 4x+2 2y+1 -> vChromaDst2 */
1336 {
1337 const __m128i mask =
1338 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1339 (char)0x80, (char)0x80, (char)0x80, 15, 13, 11, 9, 7, 5, 3, 1);
1340 __m128i vde = _mm_shuffle_epi8(ve, mask);
1341 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, yEvenChromaDst2), vde);
1342 yEvenChromaDst2 += 8;
1343 }
1344
1345 if (yLumaDstOdd)
1346 {
1347 const __m128i mask =
1348 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1349 (char)0x80, (char)0x80, (char)0x80, 15, 13, 11, 9, 7, 5, 3, 1);
1350 __m128i vdo = _mm_shuffle_epi8(vo, mask);
1351 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, yOddChromaDst2), vdo);
1352 yOddChromaDst2 += 8;
1353 }
1354
1355 if (yLumaDstOdd)
1356 {
1357 const __m128i mask =
1358 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1359 (char)0x80, (char)0x80, (char)0x80, 14, 10, 6, 2, 12, 8, 4, 0);
1360 const __m128i vd = _mm_shuffle_epi8(vo, mask);
1361 int* uDst2 = WINPR_PACKED_ALIGN_CAST(int*, uChromaDst2);
1362 int* vDst2 = WINPR_PACKED_ALIGN_CAST(int*, vChromaDst2);
1363 const int* src = (const int*)&vd;
1364 _mm_stream_si32(uDst2, src[0]);
1365 _mm_stream_si32(vDst2, src[1]);
1366 uChromaDst2 += 4;
1367 vChromaDst2 += 4;
1368 }
1369
1370 if (yLumaDstOdd)
1371 {
1372 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, vLumaDst), vavg);
1373 vLumaDst += 8;
1374 }
1375 else
1376 {
1377 const __m128i mask =
1378 _mm_set_epi8((char)0x80, (char)0x80, (char)0x80, (char)0x80, (char)0x80,
1379 (char)0x80, (char)0x80, (char)0x80, 14, 12, 10, 8, 6, 4, 2, 0);
1380 __m128i vd = _mm_shuffle_epi8(ve, mask);
1381 _mm_storel_epi64(WINPR_PACKED_ALIGN_CAST(__m128i*, vLumaDst), vd);
1382 vLumaDst += 8;
1383 }
1384 }
1385 }
1386
1387 general_RGBToAVC444YUVv2_BGRX_DOUBLE_ROW(x, srcEven, srcOdd, yLumaDstEven, yLumaDstOdd,
1388 uLumaDst, vLumaDst, yEvenChromaDst1, yEvenChromaDst2,
1389 yOddChromaDst1, yOddChromaDst2, uChromaDst1,
1390 uChromaDst2, vChromaDst1, vChromaDst2, width);
1391}
1392
1393static pstatus_t sse41_RGBToAVC444YUVv2_BGRX(const BYTE* WINPR_RESTRICT pSrc,
1394 WINPR_ATTR_UNUSED UINT32 srcFormat, UINT32 srcStep,
1395 BYTE* WINPR_RESTRICT pDst1[], const UINT32 dst1Step[],
1396 BYTE* WINPR_RESTRICT pDst2[], const UINT32 dst2Step[],
1397 const prim_size_t* WINPR_RESTRICT roi)
1398{
1399 if (roi->height < 1 || roi->width < 1)
1400 return !PRIMITIVES_SUCCESS;
1401
1402 size_t y = 0;
1403 for (; y < roi->height - roi->height % 2; y += 2)
1404 {
1405 const BYTE* srcEven = (pSrc + y * srcStep);
1406 const BYTE* srcOdd = (srcEven + srcStep);
1407 BYTE* dstLumaYEven = (pDst1[0] + y * dst1Step[0]);
1408 BYTE* dstLumaYOdd = (dstLumaYEven + dst1Step[0]);
1409 BYTE* dstLumaU = (pDst1[1] + (y / 2) * dst1Step[1]);
1410 BYTE* dstLumaV = (pDst1[2] + (y / 2) * dst1Step[2]);
1411 BYTE* dstEvenChromaY1 = (pDst2[0] + y * dst2Step[0]);
1412 BYTE* dstEvenChromaY2 = dstEvenChromaY1 + roi->width / 2;
1413 BYTE* dstOddChromaY1 = dstEvenChromaY1 + dst2Step[0];
1414 BYTE* dstOddChromaY2 = dstEvenChromaY2 + dst2Step[0];
1415 BYTE* dstChromaU1 = (pDst2[1] + (y / 2) * dst2Step[1]);
1416 BYTE* dstChromaV1 = (pDst2[2] + (y / 2) * dst2Step[2]);
1417 BYTE* dstChromaU2 = dstChromaU1 + roi->width / 4;
1418 BYTE* dstChromaV2 = dstChromaV1 + roi->width / 4;
1419 sse41_RGBToAVC444YUVv2_BGRX_DOUBLE_ROW(srcEven, srcOdd, dstLumaYEven, dstLumaYOdd, dstLumaU,
1420 dstLumaV, dstEvenChromaY1, dstEvenChromaY2,
1421 dstOddChromaY1, dstOddChromaY2, dstChromaU1,
1422 dstChromaU2, dstChromaV1, dstChromaV2, roi->width);
1423 }
1424
1425 for (; y < roi->height; y++)
1426 {
1427 const BYTE* srcEven = (pSrc + y * srcStep);
1428 BYTE* dstLumaYEven = (pDst1[0] + y * dst1Step[0]);
1429 BYTE* dstLumaU = (pDst1[1] + (y / 2) * dst1Step[1]);
1430 BYTE* dstLumaV = (pDst1[2] + (y / 2) * dst1Step[2]);
1431 BYTE* dstEvenChromaY1 = (pDst2[0] + y * dst2Step[0]);
1432 BYTE* dstEvenChromaY2 = dstEvenChromaY1 + roi->width / 2;
1433 BYTE* dstChromaU1 = (pDst2[1] + (y / 2) * dst2Step[1]);
1434 BYTE* dstChromaV1 = (pDst2[2] + (y / 2) * dst2Step[2]);
1435 BYTE* dstChromaU2 = dstChromaU1 + roi->width / 4;
1436 BYTE* dstChromaV2 = dstChromaV1 + roi->width / 4;
1437 general_RGBToAVC444YUVv2_BGRX_DOUBLE_ROW(0, srcEven, nullptr, dstLumaYEven, nullptr,
1438 dstLumaU, dstLumaV, dstEvenChromaY1,
1439 dstEvenChromaY2, nullptr, nullptr, dstChromaU1,
1440 dstChromaU2, dstChromaV1, dstChromaV2, roi->width);
1441 }
1442
1443 return PRIMITIVES_SUCCESS;
1444}
1445
1446static pstatus_t sse41_RGBToAVC444YUVv2(const BYTE* WINPR_RESTRICT pSrc, UINT32 srcFormat,
1447 UINT32 srcStep, BYTE* WINPR_RESTRICT pDst1[],
1448 const UINT32 dst1Step[], BYTE* WINPR_RESTRICT pDst2[],
1449 const UINT32 dst2Step[],
1450 const prim_size_t* WINPR_RESTRICT roi)
1451{
1452 switch (srcFormat)
1453 {
1454 case PIXEL_FORMAT_BGRX32:
1455 case PIXEL_FORMAT_BGRA32:
1456 return sse41_RGBToAVC444YUVv2_BGRX(pSrc, srcFormat, srcStep, pDst1, dst1Step, pDst2,
1457 dst2Step, roi);
1458
1459 default:
1460 return generic->RGBToAVC444YUVv2(pSrc, srcFormat, srcStep, pDst1, dst1Step, pDst2,
1461 dst2Step, roi);
1462 }
1463}
1464
1465static pstatus_t sse41_LumaToYUV444(const BYTE* WINPR_RESTRICT pSrcRaw[], const UINT32 srcStep[],
1466 BYTE* WINPR_RESTRICT pDstRaw[], const UINT32 dstStep[],
1467 const RECTANGLE_16* WINPR_RESTRICT roi)
1468{
1469 const UINT32 nWidth = roi->right - roi->left;
1470 const UINT32 nHeight = roi->bottom - roi->top;
1471 const UINT32 halfWidth = (nWidth + 1) / 2;
1472 const UINT32 halfPad = halfWidth % 16;
1473 const UINT32 halfHeight = (nHeight + 1) / 2;
1474 const UINT32 oddY = 1;
1475 const UINT32 evenY = 0;
1476 const UINT32 oddX = 1;
1477 const UINT32 evenX = 0;
1478 const BYTE* pSrc[3] = { pSrcRaw[0] + 1ULL * roi->top * srcStep[0] + roi->left,
1479 pSrcRaw[1] + 1ULL * roi->top / 2 * srcStep[1] + roi->left / 2,
1480 pSrcRaw[2] + 1ULL * roi->top / 2 * srcStep[2] + roi->left / 2 };
1481 BYTE* pDst[3] = { pDstRaw[0] + 1ULL * roi->top * dstStep[0] + roi->left,
1482 pDstRaw[1] + 1ULL * roi->top * dstStep[1] + roi->left,
1483 pDstRaw[2] + 1ULL * roi->top * dstStep[2] + roi->left };
1484
1485 /* Y data is already here... */
1486 /* B1 */
1487 for (size_t y = 0; y < nHeight; y++)
1488 {
1489 const BYTE* Ym = pSrc[0] + y * srcStep[0];
1490 BYTE* pY = pDst[0] + y * dstStep[0];
1491 memcpy(pY, Ym, nWidth);
1492 }
1493
1494 /* The first half of U, V are already here part of this frame. */
1495 /* B2 and B3 */
1496 for (size_t y = 0; y < halfHeight; y++)
1497 {
1498 const size_t val2y = (2 * y + evenY);
1499 const size_t val2y1 = val2y + oddY;
1500 const BYTE* Um = pSrc[1] + 1ULL * srcStep[1] * y;
1501 const BYTE* Vm = pSrc[2] + 1ULL * srcStep[2] * y;
1502 BYTE* pU = pDst[1] + 1ULL * dstStep[1] * val2y;
1503 BYTE* pV = pDst[2] + 1ULL * dstStep[2] * val2y;
1504 BYTE* pU1 = pDst[1] + 1ULL * dstStep[1] * val2y1;
1505 BYTE* pV1 = pDst[2] + 1ULL * dstStep[2] * val2y1;
1506
1507 size_t x = 0;
1508 for (; x < halfWidth - halfPad; x += 16)
1509 {
1510 const __m128i unpackHigh = _mm_set_epi8(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
1511 const __m128i unpackLow =
1512 _mm_set_epi8(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8);
1513 {
1514 const __m128i u = LOAD_SI128(&Um[x]);
1515 const __m128i uHigh = _mm_shuffle_epi8(u, unpackHigh);
1516 const __m128i uLow = _mm_shuffle_epi8(u, unpackLow);
1517 STORE_SI128(&pU[2ULL * x], uHigh);
1518 STORE_SI128(&pU[2ULL * x + 16], uLow);
1519 STORE_SI128(&pU1[2ULL * x], uHigh);
1520 STORE_SI128(&pU1[2ULL * x + 16], uLow);
1521 }
1522 {
1523 const __m128i u = LOAD_SI128(&Vm[x]);
1524 const __m128i uHigh = _mm_shuffle_epi8(u, unpackHigh);
1525 const __m128i uLow = _mm_shuffle_epi8(u, unpackLow);
1526 STORE_SI128(&pV[2 * x], uHigh);
1527 STORE_SI128(&pV[2 * x + 16], uLow);
1528 STORE_SI128(&pV1[2 * x], uHigh);
1529 STORE_SI128(&pV1[2 * x + 16], uLow);
1530 }
1531 }
1532
1533 for (; x < halfWidth; x++)
1534 {
1535 const size_t val2x = 2 * x + evenX;
1536 const size_t val2x1 = val2x + oddX;
1537 pU[val2x] = Um[x];
1538 pV[val2x] = Vm[x];
1539 pU[val2x1] = Um[x];
1540 pV[val2x1] = Vm[x];
1541 pU1[val2x] = Um[x];
1542 pV1[val2x] = Vm[x];
1543 pU1[val2x1] = Um[x];
1544 pV1[val2x1] = Vm[x];
1545 }
1546 }
1547
1548 return PRIMITIVES_SUCCESS;
1549}
1550
1551static pstatus_t sse41_ChromaV1ToYUV444(const BYTE* WINPR_RESTRICT pSrcRaw[3],
1552 const UINT32 srcStep[3], BYTE* WINPR_RESTRICT pDstRaw[3],
1553 const UINT32 dstStep[3],
1554 const RECTANGLE_16* WINPR_RESTRICT roi)
1555{
1556 const UINT32 mod = 16;
1557 UINT32 uY = 0;
1558 UINT32 vY = 0;
1559 const UINT32 nWidth = roi->right - roi->left;
1560 const UINT32 nHeight = roi->bottom - roi->top;
1561 const UINT32 halfWidth = (nWidth + 1) / 2;
1562 const UINT32 halfPad = halfWidth % 16;
1563 const UINT32 halfHeight = (nHeight + 1) / 2;
1564 const UINT32 oddY = 1;
1565 const UINT32 evenY = 0;
1566 const UINT32 oddX = 1;
1567 /* The auxiliary frame is aligned to multiples of 16x16.
1568 * We need the padded height for B4 and B5 conversion. */
1569 const UINT32 padHeight = nHeight + 16 - nHeight % 16;
1570 const BYTE* pSrc[3] = { pSrcRaw[0] + 1ULL * roi->top * srcStep[0] + roi->left,
1571 pSrcRaw[1] + 1ULL * roi->top / 2 * srcStep[1] + roi->left / 2,
1572 pSrcRaw[2] + 1ULL * roi->top / 2 * srcStep[2] + roi->left / 2 };
1573 BYTE* pDst[3] = { pDstRaw[0] + 1ULL * roi->top * dstStep[0] + roi->left,
1574 pDstRaw[1] + 1ULL * roi->top * dstStep[1] + roi->left,
1575 pDstRaw[2] + 1ULL * roi->top * dstStep[2] + roi->left };
1576 const __m128i zero = _mm_setzero_si128();
1577 const __m128i mask = _mm_set_epi8(0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0,
1578 (char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80);
1579
1580 /* The second half of U and V is a bit more tricky... */
1581 /* B4 and B5 */
1582 for (size_t y = 0; y < padHeight; y++)
1583 {
1584 const BYTE* Ya = pSrc[0] + 1ULL * srcStep[0] * y;
1585 BYTE* pX = nullptr;
1586
1587 if ((y) % mod < (mod + 1) / 2)
1588 {
1589 const UINT32 pos = (2 * uY++ + oddY);
1590
1591 if (pos >= nHeight)
1592 continue;
1593
1594 pX = pDst[1] + 1ULL * dstStep[1] * pos;
1595 }
1596 else
1597 {
1598 const UINT32 pos = (2 * vY++ + oddY);
1599
1600 if (pos >= nHeight)
1601 continue;
1602
1603 pX = pDst[2] + 1ULL * dstStep[2] * pos;
1604 }
1605
1606 if (y < nHeight)
1607 memcpy(pX, Ya, nWidth);
1608 }
1609
1610 /* B6 and B7 */
1611 for (size_t y = 0; y < halfHeight; y++)
1612 {
1613 const size_t val2y = (y * 2 + evenY);
1614 const BYTE* Ua = pSrc[1] + srcStep[1] * y;
1615 const BYTE* Va = pSrc[2] + srcStep[2] * y;
1616 BYTE* pU = pDst[1] + dstStep[1] * val2y;
1617 BYTE* pV = pDst[2] + dstStep[2] * val2y;
1618
1619 size_t x = 0;
1620 for (; x < halfWidth - halfPad; x += 16)
1621 {
1622 {
1623 const __m128i u = LOAD_SI128(&Ua[x]);
1624 const __m128i u2 = _mm_unpackhi_epi8(u, zero);
1625 const __m128i u1 = _mm_unpacklo_epi8(u, zero);
1626 /* the mask selects the even bytes of u1/u2, which carry the
1627 * samples; B6 goes to the odd columns 2x + 1 */
1628 _mm_maskmoveu_si128(u1, mask, (char*)&pU[2 * x + 1]);
1629 _mm_maskmoveu_si128(u2, mask, (char*)&pU[2 * x + 17]);
1630 }
1631 {
1632 const __m128i u = LOAD_SI128(&Va[x]);
1633 const __m128i u2 = _mm_unpackhi_epi8(u, zero);
1634 const __m128i u1 = _mm_unpacklo_epi8(u, zero);
1635 /* B7 likewise to the odd columns */
1636 _mm_maskmoveu_si128(u1, mask, (char*)&pV[2 * x + 1]);
1637 _mm_maskmoveu_si128(u2, mask, (char*)&pV[2 * x + 17]);
1638 }
1639 }
1640
1641 for (; x < halfWidth; x++)
1642 {
1643 const size_t val2x1 = (x * 2ULL + oddX);
1644 pU[val2x1] = Ua[x];
1645 pV[val2x1] = Va[x];
1646 }
1647 }
1648
1649 return PRIMITIVES_SUCCESS;
1650}
1651
1652static pstatus_t sse41_ChromaV2ToYUV444(const BYTE* WINPR_RESTRICT pSrc[3], const UINT32 srcStep[3],
1653 UINT32 nTotalWidth, WINPR_ATTR_UNUSED UINT32 nTotalHeight,
1654 BYTE* WINPR_RESTRICT pDst[3], const UINT32 dstStep[3],
1655 const RECTANGLE_16* WINPR_RESTRICT roi)
1656{
1657 const UINT32 nWidth = roi->right - roi->left;
1658 const UINT32 nHeight = roi->bottom - roi->top;
1659 const UINT32 halfWidth = (nWidth + 1) / 2;
1660 const UINT32 halfPad = halfWidth % 16;
1661 const UINT32 halfHeight = (nHeight + 1) / 2;
1662 const UINT32 quaterWidth = (nWidth + 3) / 4;
1663 const UINT32 quaterPad = quaterWidth % 16;
1664 const __m128i zero = _mm_setzero_si128();
1665 const __m128i mask = _mm_set_epi8((char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0,
1666 (char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0);
1667 const __m128i mask2 = _mm_set_epi8(0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80,
1668 0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80);
1669 const __m128i shuffle1 =
1670 _mm_set_epi8((char)0x80, 15, (char)0x80, 14, (char)0x80, 13, (char)0x80, 12, (char)0x80, 11,
1671 (char)0x80, 10, (char)0x80, 9, (char)0x80, 8);
1672 const __m128i shuffle2 =
1673 _mm_set_epi8((char)0x80, 7, (char)0x80, 6, (char)0x80, 5, (char)0x80, 4, (char)0x80, 3,
1674 (char)0x80, 2, (char)0x80, 1, (char)0x80, 0);
1675
1676 /* B4 and B5: odd UV values for width/2, height */
1677 for (size_t y = 0; y < nHeight; y++)
1678 {
1679 const size_t yTop = y + roi->top;
1680 const BYTE* pYaU = pSrc[0] + srcStep[0] * yTop + roi->left / 2;
1681 const BYTE* pYaV = pYaU + nTotalWidth / 2;
1682 BYTE* pU = pDst[1] + 1ULL * dstStep[1] * yTop + roi->left;
1683 BYTE* pV = pDst[2] + 1ULL * dstStep[2] * yTop + roi->left;
1684
1685 size_t x = 0;
1686 for (; x < halfWidth - halfPad; x += 16)
1687 {
1688 {
1689 const __m128i u = LOAD_SI128(&pYaU[x]);
1690 const __m128i u2 = _mm_unpackhi_epi8(zero, u);
1691 const __m128i u1 = _mm_unpacklo_epi8(zero, u);
1692 _mm_maskmoveu_si128(u1, mask, (char*)&pU[2 * x]);
1693 _mm_maskmoveu_si128(u2, mask, (char*)&pU[2 * x + 16]);
1694 }
1695 {
1696 const __m128i v = LOAD_SI128(&pYaV[x]);
1697 const __m128i v2 = _mm_unpackhi_epi8(zero, v);
1698 const __m128i v1 = _mm_unpacklo_epi8(zero, v);
1699 _mm_maskmoveu_si128(v1, mask, (char*)&pV[2 * x]);
1700 _mm_maskmoveu_si128(v2, mask, (char*)&pV[2 * x + 16]);
1701 }
1702 }
1703
1704 for (; x < halfWidth; x++)
1705 {
1706 const size_t odd = 2ULL * x + 1;
1707 pU[odd] = pYaU[x];
1708 pV[odd] = pYaV[x];
1709 }
1710 }
1711
1712 /* B6 - B9 */
1713 for (size_t y = 0; y < halfHeight; y++)
1714 {
1715 const BYTE* pUaU = pSrc[1] + srcStep[1] * (y + roi->top / 2) + roi->left / 4;
1716 const BYTE* pUaV = pUaU + nTotalWidth / 4;
1717 const BYTE* pVaU = pSrc[2] + srcStep[2] * (y + roi->top / 2) + roi->left / 4;
1718 const BYTE* pVaV = pVaU + nTotalWidth / 4;
1719 BYTE* pU = pDst[1] + dstStep[1] * (2 * y + 1 + roi->top) + roi->left;
1720 BYTE* pV = pDst[2] + dstStep[2] * (2 * y + 1 + roi->top) + roi->left;
1721
1722 UINT32 x = 0;
1723 for (; x < quaterWidth - quaterPad; x += 16)
1724 {
1725 {
1726 const __m128i uU = LOAD_SI128(&pUaU[x]);
1727 const __m128i uV = LOAD_SI128(&pVaU[x]);
1728 const __m128i uHigh = _mm_unpackhi_epi8(uU, uV);
1729 const __m128i uLow = _mm_unpacklo_epi8(uU, uV);
1730 const __m128i u1 = _mm_shuffle_epi8(uLow, shuffle2);
1731 const __m128i u2 = _mm_shuffle_epi8(uLow, shuffle1);
1732 const __m128i u3 = _mm_shuffle_epi8(uHigh, shuffle2);
1733 const __m128i u4 = _mm_shuffle_epi8(uHigh, shuffle1);
1734 _mm_maskmoveu_si128(u1, mask2, (char*)&pU[4 * x + 0]);
1735 _mm_maskmoveu_si128(u2, mask2, (char*)&pU[4 * x + 16]);
1736 _mm_maskmoveu_si128(u3, mask2, (char*)&pU[4 * x + 32]);
1737 _mm_maskmoveu_si128(u4, mask2, (char*)&pU[4 * x + 48]);
1738 }
1739 {
1740 const __m128i vU = LOAD_SI128(&pUaV[x]);
1741 const __m128i vV = LOAD_SI128(&pVaV[x]);
1742 const __m128i vHigh = _mm_unpackhi_epi8(vU, vV);
1743 const __m128i vLow = _mm_unpacklo_epi8(vU, vV);
1744 const __m128i v1 = _mm_shuffle_epi8(vLow, shuffle2);
1745 const __m128i v2 = _mm_shuffle_epi8(vLow, shuffle1);
1746 const __m128i v3 = _mm_shuffle_epi8(vHigh, shuffle2);
1747 const __m128i v4 = _mm_shuffle_epi8(vHigh, shuffle1);
1748 _mm_maskmoveu_si128(v1, mask2, (char*)&pV[4 * x + 0]);
1749 _mm_maskmoveu_si128(v2, mask2, (char*)&pV[4 * x + 16]);
1750 _mm_maskmoveu_si128(v3, mask2, (char*)&pV[4 * x + 32]);
1751 _mm_maskmoveu_si128(v4, mask2, (char*)&pV[4 * x + 48]);
1752 }
1753 }
1754
1755 for (; x < quaterWidth; x++)
1756 {
1757 pU[4 * x + 0] = pUaU[x];
1758 pV[4 * x + 0] = pUaV[x];
1759 pU[4 * x + 2] = pVaU[x];
1760 pV[4 * x + 2] = pVaV[x];
1761 }
1762 }
1763
1764 return PRIMITIVES_SUCCESS;
1765}
1766
1767static pstatus_t sse41_YUV420CombineToYUV444(avc444_frame_type type,
1768 const BYTE* WINPR_RESTRICT pSrc[3],
1769 const UINT32 srcStep[3], UINT32 nWidth, UINT32 nHeight,
1770 BYTE* WINPR_RESTRICT pDst[3], const UINT32 dstStep[3],
1771 const RECTANGLE_16* WINPR_RESTRICT roi)
1772{
1773 if (!pSrc || !pSrc[0] || !pSrc[1] || !pSrc[2])
1774 return -1;
1775
1776 if (!pDst || !pDst[0] || !pDst[1] || !pDst[2])
1777 return -1;
1778
1779 if (!roi)
1780 return -1;
1781
1782 switch (type)
1783 {
1784 case AVC444_LUMA:
1785 return sse41_LumaToYUV444(pSrc, srcStep, pDst, dstStep, roi);
1786
1787 case AVC444_CHROMAv1:
1788 return sse41_ChromaV1ToYUV444(pSrc, srcStep, pDst, dstStep, roi);
1789
1790 case AVC444_CHROMAv2:
1791 return sse41_ChromaV2ToYUV444(pSrc, srcStep, nWidth, nHeight, pDst, dstStep, roi);
1792
1793 default:
1794 return -1;
1795 }
1796}
1797
1798/* RGBToI444_8u: one Y, U and V sample per pixel, with the exact integer math of
1799 * RGB2Y/RGB2U/RGB2V (16 bit lanes are wide enough for every intermediate value),
1800 * so the result matches the generic routine byte for byte. */
1801
1802/* a pshufb mask that moves byte off of each of four 32 bit pixels into the low
1803 * byte of 16 bit lane first..first+3, and zeroes every other byte */
1804static inline __m128i sse41_I444_mask(size_t off, size_t first)
1805{
1806 BYTE m[16];
1807 memset(m, 0x80, sizeof(m));
1808 for (size_t x = 0; x < 4; x++)
1809 m[2 * (first + x)] = (BYTE)(4 * x + off);
1810 return _mm_loadu_si128((const __m128i*)m);
1811}
1812
1813/* Y, U and V of eight pixels given as 16 bit R, G and B lanes */
1814static inline void sse41_I444_YUV(__m128i r, __m128i g, __m128i b, __m128i* WINPR_RESTRICT y,
1815 __m128i* WINPR_RESTRICT u, __m128i* WINPR_RESTRICT v)
1816{
1817 const __m128i c128 = _mm_set1_epi16(128);
1818
1819 /* (54 R + 183 G + 18 B) >> 8: at most 65025, exact as unsigned 16 bit */
1820 __m128i t = _mm_add_epi16(_mm_mullo_epi16(r, _mm_set1_epi16(54)),
1821 _mm_mullo_epi16(g, _mm_set1_epi16(183)));
1822 t = _mm_add_epi16(t, _mm_mullo_epi16(b, _mm_set1_epi16(18)));
1823 *y = _mm_srli_epi16(t, 8);
1824
1825 /* ((128 B - (29 R + 99 G)) >> 8) + 128: both terms are at most 32640 */
1826 t = _mm_add_epi16(_mm_mullo_epi16(r, _mm_set1_epi16(29)),
1827 _mm_mullo_epi16(g, _mm_set1_epi16(99)));
1828 t = _mm_sub_epi16(_mm_slli_epi16(b, 7), t);
1829 *u = _mm_add_epi16(_mm_srai_epi16(t, 8), c128);
1830
1831 /* ((128 R - (116 G + 12 B)) >> 8) + 128 */
1832 t = _mm_add_epi16(_mm_mullo_epi16(g, _mm_set1_epi16(116)),
1833 _mm_mullo_epi16(b, _mm_set1_epi16(12)));
1834 t = _mm_sub_epi16(_mm_slli_epi16(r, 7), t);
1835 *v = _mm_add_epi16(_mm_srai_epi16(t, 8), c128);
1836}
1837
1838/* eight pixels from two loads of four: lo picks a channel of the first four
1839 * pixels into lanes 0-3, hi the same channel of the next four into lanes 4-7 */
1840static inline void sse41_I444_8px(const BYTE* WINPR_RESTRICT src, const __m128i masks[6],
1841 __m128i* WINPR_RESTRICT y, __m128i* WINPR_RESTRICT u,
1842 __m128i* WINPR_RESTRICT v)
1843{
1844 const __m128i p0 = LOAD_SI128(src);
1845 const __m128i p1 = LOAD_SI128(src + 16);
1846 const __m128i r = _mm_or_si128(_mm_shuffle_epi8(p0, masks[0]), _mm_shuffle_epi8(p1, masks[1]));
1847 const __m128i g = _mm_or_si128(_mm_shuffle_epi8(p0, masks[2]), _mm_shuffle_epi8(p1, masks[3]));
1848 const __m128i b = _mm_or_si128(_mm_shuffle_epi8(p0, masks[4]), _mm_shuffle_epi8(p1, masks[5]));
1849 sse41_I444_YUV(r, g, b, y, u, v);
1850}
1851
1852static pstatus_t sse41_RGBToI444_32bpp(const BYTE* WINPR_RESTRICT pSrc, UINT32 srcStep,
1853 BYTE* WINPR_RESTRICT pDst[3], const UINT32 dstStep[3],
1854 const prim_size_t* WINPR_RESTRICT roi, size_t rOff,
1855 size_t gOff, size_t bOff)
1856{
1857 const __m128i masks[6] = { sse41_I444_mask(rOff, 0), sse41_I444_mask(rOff, 4),
1858 sse41_I444_mask(gOff, 0), sse41_I444_mask(gOff, 4),
1859 sse41_I444_mask(bOff, 0), sse41_I444_mask(bOff, 4) };
1860 const size_t width = roi->width;
1861
1862 for (size_t row = 0; row < roi->height; row++)
1863 {
1864 const BYTE* src = &pSrc[row * srcStep];
1865 BYTE* yd = &pDst[0][row * dstStep[0]];
1866 BYTE* ud = &pDst[1][row * dstStep[1]];
1867 BYTE* vd = &pDst[2][row * dstStep[2]];
1868
1869 size_t x = 0;
1870 for (; x + 16 <= width; x += 16)
1871 {
1872 __m128i y0 = _mm_setzero_si128();
1873 __m128i u0 = _mm_setzero_si128();
1874 __m128i v0 = _mm_setzero_si128();
1875 __m128i y1 = _mm_setzero_si128();
1876 __m128i u1 = _mm_setzero_si128();
1877 __m128i v1 = _mm_setzero_si128();
1878 sse41_I444_8px(&src[4 * x], masks, &y0, &u0, &v0);
1879 sse41_I444_8px(&src[4 * x + 32], masks, &y1, &u1, &v1);
1880 STORE_SI128(&yd[x], _mm_packus_epi16(y0, y1));
1881 STORE_SI128(&ud[x], _mm_packus_epi16(u0, u1));
1882 STORE_SI128(&vd[x], _mm_packus_epi16(v0, v1));
1883 }
1884
1885 for (; x < width; x++)
1886 {
1887 const BYTE* px = &src[4 * x];
1888 const BYTE R = px[rOff];
1889 const BYTE G = px[gOff];
1890 const BYTE B = px[bOff];
1891 yd[x] = RGB2Y(R, G, B);
1892 ud[x] = RGB2U(R, G, B);
1893 vd[x] = RGB2V(R, G, B);
1894 }
1895 }
1896
1897 return PRIMITIVES_SUCCESS;
1898}
1899
1900static pstatus_t sse41_RGBToI444_8u(const BYTE* WINPR_RESTRICT pSrc, UINT32 SrcFormat,
1901 const UINT32 srcStep, BYTE* WINPR_RESTRICT pDst[3],
1902 const UINT32 dstStep[3], const prim_size_t* WINPR_RESTRICT roi)
1903{
1904 /* byte offsets of R, G and B in memory, as in general_RGBToI444_8u */
1905 switch (SrcFormat)
1906 {
1907 case PIXEL_FORMAT_BGRA32:
1908 case PIXEL_FORMAT_BGRX32:
1909 return sse41_RGBToI444_32bpp(pSrc, srcStep, pDst, dstStep, roi, 2, 1, 0);
1910 case PIXEL_FORMAT_RGBA32:
1911 case PIXEL_FORMAT_RGBX32:
1912 return sse41_RGBToI444_32bpp(pSrc, srcStep, pDst, dstStep, roi, 0, 1, 2);
1913 case PIXEL_FORMAT_ARGB32:
1914 case PIXEL_FORMAT_XRGB32:
1915 return sse41_RGBToI444_32bpp(pSrc, srcStep, pDst, dstStep, roi, 1, 2, 3);
1916 case PIXEL_FORMAT_ABGR32:
1917 case PIXEL_FORMAT_XBGR32:
1918 return sse41_RGBToI444_32bpp(pSrc, srcStep, pDst, dstStep, roi, 3, 2, 1);
1919 default:
1920 return generic->RGBToI444_8u(pSrc, SrcFormat, srcStep, pDst, dstStep, roi);
1921 }
1922}
1923#endif
1924
1925void primitives_init_YUV_sse41_int(primitives_t* WINPR_RESTRICT prims)
1926{
1927#if defined(SSE_AVX_INTRINSICS_ENABLED)
1928 generic = primitives_get_generic();
1929
1930 WLog_VRB(PRIM_TAG, "SSE3/sse41 optimizations");
1931 prims->RGBToYUV420_8u_P3AC4R = sse41_RGBToYUV420;
1932 prims->RGBToI444_8u = sse41_RGBToI444_8u;
1933 prims->RGBToAVC444YUV = sse41_RGBToAVC444YUV;
1934 prims->RGBToAVC444YUVv2 = sse41_RGBToAVC444YUVv2;
1935 prims->YUV420ToRGB_8u_P3AC4R = sse41_YUV420ToRGB;
1936 prims->YUV444ToRGB_8u_P3AC4R = sse41_YUV444ToRGB_8u_P3AC4R;
1937 prims->I444ToRGB_8u = sse41_I444ToRGB_8u;
1938 prims->YUV420CombineToYUV444 = sse41_YUV420CombineToYUV444;
1939#else
1940 WLog_VRB(PRIM_TAG, "undefined WITH_SIMD or sse41 intrinsics not available");
1941 WINPR_UNUSED(prims);
1942#endif
1943}
RECTANGLE_16
Definition types.h:109
prim_size_t
Definition primitives.h:74
primitives_t
Definition primitives.h:255