FreeRDP
rfx_sse2.c
1 
21 #include <winpr/assert.h>
22 #include <winpr/cast.h>
23 #include <winpr/platform.h>
24 #include <freerdp/config.h>
25 
26 #include "../rfx_types.h"
27 #include "rfx_sse2.h"
28 
29 #include "../../core/simd.h"
30 
31 #if defined(SSE_AVX_INTRINSICS_ENABLED)
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <string.h>
35 #include <winpr/sysinfo.h>
36 
37 #include <xmmintrin.h>
38 #include <emmintrin.h>
39 
40 #ifdef _MSC_VER
41 #define __attribute__(...)
42 #endif
43 
44 #define CACHE_LINE_BYTES 64
45 
46 #ifndef __clang__
47 #define ATTRIBUTES __gnu_inline__, __always_inline__, __artificial__
48 #else
49 #define ATTRIBUTES __gnu_inline__, __always_inline__
50 #endif
51 
52 static __inline void __attribute__((ATTRIBUTES))
53 mm_prefetch_buffer(char* WINPR_RESTRICT buffer, size_t num_bytes)
54 {
55  __m128i* buf = (__m128i*)buffer;
56 
57  for (size_t i = 0; i < (num_bytes / sizeof(__m128i)); i += (CACHE_LINE_BYTES / sizeof(__m128i)))
58  {
59  _mm_prefetch((char*)(&buf[i]), _MM_HINT_NTA);
60  }
61 }
62 
63 /* rfx_decode_ycbcr_to_rgb_sse2 code now resides in the primitives library. */
64 /* rfx_encode_rgb_to_ycbcr_sse2 code now resides in the primitives library. */
65 
66 static __inline void __attribute__((ATTRIBUTES))
67 rfx_quantization_decode_block_sse2(INT16* WINPR_RESTRICT buffer, const size_t buffer_size,
68  const UINT32 factor)
69 {
70  __m128i* ptr = (__m128i*)buffer;
71  const __m128i* buf_end = (__m128i*)(buffer + buffer_size);
72 
73  if (factor == 0)
74  return;
75 
76  do
77  {
78  const __m128i la = _mm_load_si128(ptr);
79  const __m128i a = _mm_slli_epi16(la, WINPR_ASSERTING_INT_CAST(int, factor));
80 
81  _mm_store_si128(ptr, a);
82  ptr++;
83  } while (ptr < buf_end);
84 }
85 
86 static void rfx_quantization_decode_sse2(INT16* WINPR_RESTRICT buffer,
87  const UINT32* WINPR_RESTRICT quantVals)
88 {
89  WINPR_ASSERT(buffer);
90  WINPR_ASSERT(quantVals);
91 
92  mm_prefetch_buffer((char*)buffer, 4096 * sizeof(INT16));
93  rfx_quantization_decode_block_sse2(&buffer[0], 1024, quantVals[8] - 1); /* HL1 */
94  rfx_quantization_decode_block_sse2(&buffer[1024], 1024, quantVals[7] - 1); /* LH1 */
95  rfx_quantization_decode_block_sse2(&buffer[2048], 1024, quantVals[9] - 1); /* HH1 */
96  rfx_quantization_decode_block_sse2(&buffer[3072], 256, quantVals[5] - 1); /* HL2 */
97  rfx_quantization_decode_block_sse2(&buffer[3328], 256, quantVals[4] - 1); /* LH2 */
98  rfx_quantization_decode_block_sse2(&buffer[3584], 256, quantVals[6] - 1); /* HH2 */
99  rfx_quantization_decode_block_sse2(&buffer[3840], 64, quantVals[2] - 1); /* HL3 */
100  rfx_quantization_decode_block_sse2(&buffer[3904], 64, quantVals[1] - 1); /* LH3 */
101  rfx_quantization_decode_block_sse2(&buffer[3968], 64, quantVals[3] - 1); /* HH3 */
102  rfx_quantization_decode_block_sse2(&buffer[4032], 64, quantVals[0] - 1); /* LL3 */
103 }
104 
105 static __inline void __attribute__((ATTRIBUTES))
106 rfx_quantization_encode_block_sse2(INT16* WINPR_RESTRICT buffer, const unsigned buffer_size,
107  const INT16 factor)
108 {
109  __m128i* ptr = (__m128i*)buffer;
110  const __m128i* buf_end = (const __m128i*)(buffer + buffer_size);
111 
112  if (factor == 0)
113  return;
114 
115  const __m128i half = _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(INT16, 1 << (factor - 1)));
116 
117  do
118  {
119  const __m128i la = _mm_load_si128(ptr);
120  __m128i a = _mm_add_epi16(la, half);
121  a = _mm_srai_epi16(a, factor);
122  _mm_store_si128(ptr, a);
123  ptr++;
124  } while (ptr < buf_end);
125 }
126 
127 static void rfx_quantization_encode_sse2(INT16* WINPR_RESTRICT buffer,
128  const UINT32* WINPR_RESTRICT quantization_values)
129 {
130  WINPR_ASSERT(buffer);
131  WINPR_ASSERT(quantization_values);
132  for (size_t x = 0; x < 10; x++)
133  {
134  WINPR_ASSERT(quantization_values[x] >= 6);
135  WINPR_ASSERT(quantization_values[x] <= INT16_MAX + 6);
136  }
137 
138  mm_prefetch_buffer((char*)buffer, 4096 * sizeof(INT16));
139  rfx_quantization_encode_block_sse2(
140  buffer, 1024, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[8] - 6)); /* HL1 */
141  rfx_quantization_encode_block_sse2(
142  buffer + 1024, 1024, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[7] - 6)); /* LH1 */
143  rfx_quantization_encode_block_sse2(
144  buffer + 2048, 1024, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[9] - 6)); /* HH1 */
145  rfx_quantization_encode_block_sse2(
146  buffer + 3072, 256, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[5] - 6)); /* HL2 */
147  rfx_quantization_encode_block_sse2(
148  buffer + 3328, 256, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[4] - 6)); /* LH2 */
149  rfx_quantization_encode_block_sse2(
150  buffer + 3584, 256, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[6] - 6)); /* HH2 */
151  rfx_quantization_encode_block_sse2(
152  buffer + 3840, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[2] - 6)); /* HL3 */
153  rfx_quantization_encode_block_sse2(
154  buffer + 3904, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[1] - 6)); /* LH3 */
155  rfx_quantization_encode_block_sse2(
156  buffer + 3968, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[3] - 6)); /* HH3 */
157  rfx_quantization_encode_block_sse2(
158  buffer + 4032, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[0] - 6)); /* LL3 */
159  rfx_quantization_encode_block_sse2(buffer, 4096, 5);
160 }
161 
162 static __inline void __attribute__((ATTRIBUTES))
163 rfx_dwt_2d_decode_block_horiz_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRICT h,
164  INT16* WINPR_RESTRICT dst, size_t subband_width)
165 {
166  INT16* l_ptr = l;
167  INT16* h_ptr = h;
168  INT16* dst_ptr = dst;
169  int first = 0;
170  int last = 0;
171  __m128i dst1;
172  __m128i dst2;
173 
174  for (size_t y = 0; y < subband_width; y++)
175  {
176  /* Even coefficients */
177  for (size_t n = 0; n < subband_width; n += 8)
178  {
179  /* dst[2n] = l[n] - ((h[n-1] + h[n] + 1) >> 1); */
180  __m128i l_n = _mm_load_si128((__m128i*)l_ptr);
181  __m128i h_n = _mm_load_si128((__m128i*)h_ptr);
182  __m128i h_n_m = _mm_loadu_si128((__m128i*)(h_ptr - 1));
183 
184  if (n == 0)
185  {
186  first = _mm_extract_epi16(h_n_m, 1);
187  h_n_m = _mm_insert_epi16(h_n_m, first, 0);
188  }
189 
190  __m128i tmp_n = _mm_add_epi16(h_n, h_n_m);
191  tmp_n = _mm_add_epi16(tmp_n, _mm_set1_epi16(1));
192  tmp_n = _mm_srai_epi16(tmp_n, 1);
193  const __m128i dst_n = _mm_sub_epi16(l_n, tmp_n);
194  _mm_store_si128((__m128i*)l_ptr, dst_n);
195  l_ptr += 8;
196  h_ptr += 8;
197  }
198 
199  l_ptr -= subband_width;
200  h_ptr -= subband_width;
201 
202  /* Odd coefficients */
203  for (size_t n = 0; n < subband_width; n += 8)
204  {
205  /* dst[2n + 1] = (h[n] << 1) + ((dst[2n] + dst[2n + 2]) >> 1); */
206  __m128i h_n = _mm_load_si128((__m128i*)h_ptr);
207  h_n = _mm_slli_epi16(h_n, 1);
208  __m128i dst_n = _mm_load_si128((__m128i*)(l_ptr));
209  __m128i dst_n_p = _mm_loadu_si128((__m128i*)(l_ptr + 1));
210 
211  if (n == subband_width - 8)
212  {
213  last = _mm_extract_epi16(dst_n_p, 6);
214  dst_n_p = _mm_insert_epi16(dst_n_p, last, 7);
215  }
216 
217  __m128i tmp_n = _mm_add_epi16(dst_n_p, dst_n);
218  tmp_n = _mm_srai_epi16(tmp_n, 1);
219  tmp_n = _mm_add_epi16(tmp_n, h_n);
220  dst1 = _mm_unpacklo_epi16(dst_n, tmp_n);
221  dst2 = _mm_unpackhi_epi16(dst_n, tmp_n);
222  _mm_store_si128((__m128i*)dst_ptr, dst1);
223  _mm_store_si128((__m128i*)(dst_ptr + 8), dst2);
224  l_ptr += 8;
225  h_ptr += 8;
226  dst_ptr += 16;
227  }
228  }
229 }
230 
231 static __inline void __attribute__((ATTRIBUTES))
232 rfx_dwt_2d_decode_block_vert_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRICT h,
233  INT16* WINPR_RESTRICT dst, size_t subband_width)
234 {
235  INT16* l_ptr = l;
236  INT16* h_ptr = h;
237  INT16* dst_ptr = dst;
238  const size_t total_width = subband_width + subband_width;
239 
240  /* Even coefficients */
241  for (size_t n = 0; n < subband_width; n++)
242  {
243  for (size_t x = 0; x < total_width; x += 8)
244  {
245  /* dst[2n] = l[n] - ((h[n-1] + h[n] + 1) >> 1); */
246  const __m128i l_n = _mm_load_si128((__m128i*)l_ptr);
247  const __m128i h_n = _mm_load_si128((__m128i*)h_ptr);
248  __m128i tmp_n = _mm_add_epi16(h_n, _mm_set1_epi16(1));
249 
250  if (n == 0)
251  tmp_n = _mm_add_epi16(tmp_n, h_n);
252  else
253  {
254  const __m128i h_n_m = _mm_loadu_si128((__m128i*)(h_ptr - total_width));
255  tmp_n = _mm_add_epi16(tmp_n, h_n_m);
256  }
257 
258  tmp_n = _mm_srai_epi16(tmp_n, 1);
259  const __m128i dst_n = _mm_sub_epi16(l_n, tmp_n);
260  _mm_store_si128((__m128i*)dst_ptr, dst_n);
261  l_ptr += 8;
262  h_ptr += 8;
263  dst_ptr += 8;
264  }
265 
266  dst_ptr += total_width;
267  }
268 
269  h_ptr = h;
270  dst_ptr = dst + total_width;
271 
272  /* Odd coefficients */
273  for (size_t n = 0; n < subband_width; n++)
274  {
275  for (size_t x = 0; x < total_width; x += 8)
276  {
277  /* dst[2n + 1] = (h[n] << 1) + ((dst[2n] + dst[2n + 2]) >> 1); */
278  __m128i h_n = _mm_load_si128((__m128i*)h_ptr);
279  __m128i dst_n_m = _mm_load_si128((__m128i*)(dst_ptr - total_width));
280  h_n = _mm_slli_epi16(h_n, 1);
281  __m128i tmp_n = dst_n_m;
282 
283  if (n == subband_width - 1)
284  tmp_n = _mm_add_epi16(tmp_n, dst_n_m);
285  else
286  {
287  const __m128i dst_n_p = _mm_loadu_si128((__m128i*)(dst_ptr + total_width));
288  tmp_n = _mm_add_epi16(tmp_n, dst_n_p);
289  }
290 
291  tmp_n = _mm_srai_epi16(tmp_n, 1);
292  const __m128i dst_n = _mm_add_epi16(tmp_n, h_n);
293  _mm_store_si128((__m128i*)dst_ptr, dst_n);
294  h_ptr += 8;
295  dst_ptr += 8;
296  }
297 
298  dst_ptr += total_width;
299  }
300 }
301 
302 static __inline void __attribute__((ATTRIBUTES))
303 rfx_dwt_2d_decode_block_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RESTRICT idwt,
304  size_t subband_width)
305 {
306  mm_prefetch_buffer((char*)idwt, 4ULL * subband_width * sizeof(INT16));
307  /* Inverse DWT in horizontal direction, results in 2 sub-bands in L, H order in tmp buffer idwt.
308  */
309  /* The 4 sub-bands are stored in HL(0), LH(1), HH(2), LL(3) order. */
310  /* The lower part L uses LL(3) and HL(0). */
311  /* The higher part H uses LH(1) and HH(2). */
312  INT16* ll = buffer + 3ULL * subband_width * subband_width;
313  INT16* hl = buffer;
314  INT16* l_dst = idwt;
315  rfx_dwt_2d_decode_block_horiz_sse2(ll, hl, l_dst, subband_width);
316  INT16* lh = buffer + 1ULL * subband_width * subband_width;
317  INT16* hh = buffer + 2ULL * subband_width * subband_width;
318  INT16* h_dst = idwt + 2ULL * subband_width * subband_width;
319  rfx_dwt_2d_decode_block_horiz_sse2(lh, hh, h_dst, subband_width);
320  /* Inverse DWT in vertical direction, results are stored in original buffer. */
321  rfx_dwt_2d_decode_block_vert_sse2(l_dst, h_dst, buffer, subband_width);
322 }
323 
324 static void rfx_dwt_2d_decode_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RESTRICT dwt_buffer)
325 {
326  WINPR_ASSERT(buffer);
327  WINPR_ASSERT(dwt_buffer);
328 
329  mm_prefetch_buffer((char*)buffer, 4096 * sizeof(INT16));
330  rfx_dwt_2d_decode_block_sse2(&buffer[3840], dwt_buffer, 8);
331  rfx_dwt_2d_decode_block_sse2(&buffer[3072], dwt_buffer, 16);
332  rfx_dwt_2d_decode_block_sse2(&buffer[0], dwt_buffer, 32);
333 }
334 
335 static __inline void __attribute__((ATTRIBUTES))
336 rfx_dwt_2d_encode_block_vert_sse2(INT16* WINPR_RESTRICT src, INT16* WINPR_RESTRICT l,
337  INT16* WINPR_RESTRICT h, size_t subband_width)
338 {
339  const size_t total_width = subband_width << 1;
340 
341  for (size_t n = 0; n < subband_width; n++)
342  {
343  for (size_t x = 0; x < total_width; x += 8)
344  {
345  __m128i src_2n = _mm_load_si128((__m128i*)src);
346  __m128i src_2n_1 = _mm_load_si128((__m128i*)(src + total_width));
347  __m128i src_2n_2 = src_2n;
348 
349  if (n < subband_width - 1)
350  src_2n_2 = _mm_load_si128((__m128i*)(src + 2ULL * total_width));
351 
352  /* h[n] = (src[2n + 1] - ((src[2n] + src[2n + 2]) >> 1)) >> 1 */
353  __m128i h_n = _mm_add_epi16(src_2n, src_2n_2);
354  h_n = _mm_srai_epi16(h_n, 1);
355  h_n = _mm_sub_epi16(src_2n_1, h_n);
356  h_n = _mm_srai_epi16(h_n, 1);
357  _mm_store_si128((__m128i*)h, h_n);
358 
359  __m128i h_n_m = h_n;
360  if (n != 0)
361  h_n_m = _mm_load_si128((__m128i*)(h - total_width));
362 
363  /* l[n] = src[2n] + ((h[n - 1] + h[n]) >> 1) */
364  __m128i l_n = _mm_add_epi16(h_n_m, h_n);
365  l_n = _mm_srai_epi16(l_n, 1);
366  l_n = _mm_add_epi16(l_n, src_2n);
367  _mm_store_si128((__m128i*)l, l_n);
368  src += 8;
369  l += 8;
370  h += 8;
371  }
372 
373  src += total_width;
374  }
375 }
376 
377 static __inline void __attribute__((ATTRIBUTES))
378 rfx_dwt_2d_encode_block_horiz_sse2(INT16* WINPR_RESTRICT src, INT16* WINPR_RESTRICT l,
379  INT16* WINPR_RESTRICT h, size_t subband_width)
380 {
381  for (size_t y = 0; y < subband_width; y++)
382  {
383  for (size_t n = 0; n < subband_width; n += 8)
384  {
385  /* The following 3 Set operations consumes more than half of the total DWT processing
386  * time! */
387  __m128i src_2n =
388  _mm_set_epi16(src[14], src[12], src[10], src[8], src[6], src[4], src[2], src[0]);
389  __m128i src_2n_1 =
390  _mm_set_epi16(src[15], src[13], src[11], src[9], src[7], src[5], src[3], src[1]);
391  __m128i src_2n_2 =
392  _mm_set_epi16(((n + 8) == subband_width) ? src[14] : src[16], src[14], src[12],
393  src[10], src[8], src[6], src[4], src[2]);
394  /* h[n] = (src[2n + 1] - ((src[2n] + src[2n + 2]) >> 1)) >> 1 */
395  __m128i h_n = _mm_add_epi16(src_2n, src_2n_2);
396  h_n = _mm_srai_epi16(h_n, 1);
397  h_n = _mm_sub_epi16(src_2n_1, h_n);
398  h_n = _mm_srai_epi16(h_n, 1);
399  _mm_store_si128((__m128i*)h, h_n);
400  __m128i h_n_m = _mm_loadu_si128((__m128i*)(h - 1));
401 
402  if (n == 0)
403  {
404  int first = _mm_extract_epi16(h_n_m, 1);
405  h_n_m = _mm_insert_epi16(h_n_m, first, 0);
406  }
407 
408  /* l[n] = src[2n] + ((h[n - 1] + h[n]) >> 1) */
409  __m128i l_n = _mm_add_epi16(h_n_m, h_n);
410  l_n = _mm_srai_epi16(l_n, 1);
411  l_n = _mm_add_epi16(l_n, src_2n);
412  _mm_store_si128((__m128i*)l, l_n);
413  src += 16;
414  l += 8;
415  h += 8;
416  }
417  }
418 }
419 
420 static __inline void __attribute__((ATTRIBUTES))
421 rfx_dwt_2d_encode_block_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RESTRICT dwt,
422  size_t subband_width)
423 {
424  mm_prefetch_buffer((char*)dwt, 4ULL * subband_width * sizeof(INT16));
425  /* DWT in vertical direction, results in 2 sub-bands in L, H order in tmp buffer dwt. */
426  INT16* l_src = dwt;
427  INT16* h_src = dwt + 2ULL * subband_width * subband_width;
428  rfx_dwt_2d_encode_block_vert_sse2(buffer, l_src, h_src, subband_width);
429  /* DWT in horizontal direction, results in 4 sub-bands in HL(0), LH(1), HH(2), LL(3) order,
430  * stored in original buffer. */
431  /* The lower part L generates LL(3) and HL(0). */
432  /* The higher part H generates LH(1) and HH(2). */
433  INT16* ll = buffer + 3ULL * subband_width * subband_width;
434  INT16* hl = buffer;
435  INT16* lh = buffer + 1ULL * subband_width * subband_width;
436  INT16* hh = buffer + 2ULL * subband_width * subband_width;
437  rfx_dwt_2d_encode_block_horiz_sse2(l_src, ll, hl, subband_width);
438  rfx_dwt_2d_encode_block_horiz_sse2(h_src, lh, hh, subband_width);
439 }
440 
441 static void rfx_dwt_2d_encode_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RESTRICT dwt_buffer)
442 {
443  WINPR_ASSERT(buffer);
444  WINPR_ASSERT(dwt_buffer);
445 
446  mm_prefetch_buffer((char*)buffer, 4096 * sizeof(INT16));
447  rfx_dwt_2d_encode_block_sse2(buffer, dwt_buffer, 32);
448  rfx_dwt_2d_encode_block_sse2(buffer + 3072, dwt_buffer, 16);
449  rfx_dwt_2d_encode_block_sse2(buffer + 3840, dwt_buffer, 8);
450 }
451 #endif
452 
453 void rfx_init_sse2(RFX_CONTEXT* context)
454 {
455 #if defined(SSE_AVX_INTRINSICS_ENABLED)
456  if (!IsProcessorFeaturePresent(PF_XMMI64_INSTRUCTIONS_AVAILABLE))
457  return;
458 
459  PROFILER_RENAME(context->priv->prof_rfx_quantization_decode, "rfx_quantization_decode_sse2")
460  PROFILER_RENAME(context->priv->prof_rfx_quantization_encode, "rfx_quantization_encode_sse2")
461  PROFILER_RENAME(context->priv->prof_rfx_dwt_2d_decode, "rfx_dwt_2d_decode_sse2")
462  PROFILER_RENAME(context->priv->prof_rfx_dwt_2d_encode, "rfx_dwt_2d_encode_sse2")
463  context->quantization_decode = rfx_quantization_decode_sse2;
464  context->quantization_encode = rfx_quantization_encode_sse2;
465  context->dwt_2d_decode = rfx_dwt_2d_decode_sse2;
466  context->dwt_2d_encode = rfx_dwt_2d_encode_sse2;
467 #else
468  WINPR_UNUSED(context);
469 #endif
470 }