Ocean
Loading...
Searching...
No Matches
ZeroMeanSumSquareDifferencesSSE.h
Go to the documentation of this file.
1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8#ifndef META_OCEAN_CV_ZERO_MEAN_SUM_SQUARE_DIFFERENCES_SSE_H
9#define META_OCEAN_CV_ZERO_MEAN_SUM_SQUARE_DIFFERENCES_SSE_H
10
11#include "ocean/cv/CV.h"
12
13#include "ocean/base/Utilities.h"
14
15#if defined(OCEAN_HARDWARE_SSE_VERSION) && OCEAN_HARDWARE_SSE_VERSION >= 41
16
17#include "ocean/cv/SSE.h"
18
19namespace Ocean
20{
21
22namespace CV
23{
24
25/**
26 * This class implements functions to calculate zero-mean sum square differences using SSE instructions.
27 * @ingroup cv
28 */
29class ZeroMeanSumSquareDifferencesSSE
30{
31 protected:
32
33 /**
34 * This class allows to specialize functions for individual channels.
35 * @tparam tChannels Specifies the number of channels for the given frames, with range [1, infinity)
36 */
37 template <unsigned int tChannels>
38 class SpecializedForChannels
39 {
40 public:
41
42 /**
43 * Determines the mean value for a buffer, one value for each channel.
44 * @param buffer The memory buffer to be handled, must be valid
45 * @param meanValues The resulting mean values, one for each channel
46 * @tparam tChannels Specifies the number of channels for the given frames, with range [1, infinity)
47 * @tparam tPixels The number of pixels in the buffer, in pixels, with range [8, infinity)
48 */
49 template <unsigned int tPixels>
50 static inline void mean8BitPerChannel(const uint8_t* const buffer, uint8_t* const meanValues);
51
52 /**
53 * Determines the mean value for an image patch, one value for each channel.
54 * @param patch The top left start position of the image patch, must be valid
55 * @param patchStrideElements The number of elements between two rows for the image patch, in elements, with range [tChannels, tPatchSize, infinity)
56 * @param meanValues The resulting mean values, one for each channel
57 * @tparam tPatchSize The size of the square patch (the edge length) in pixel, with range [1, infinity), must be odd
58 */
59 template <unsigned int tPatchSize>
60 static inline void mean8BitPerChannel(const uint8_t* patch, const unsigned int patchStrideElements, uint8_t* const meanValues);
61
62 /**
63 * Returns the zero-mean sum of square differences between two memory buffers.
64 * @param buffer0 The first memory buffer, must be valid
65 * @param buffer1 The second memory buffer, must be valid
66 * @param meanValues0 The mean values of the first patch, one for each channel, must be valid
67 * @param meanValues1 The mean values of the second patch, one for each channel, must be valid
68 * @return The resulting sum of square differences
69 * @tparam tPixels The number of pixels in the buffer, in pixels, with range [8, infinity)
70 */
71 template <unsigned int tPixels>
72 static inline uint32_t buffer8BitPerChannel(const uint8_t* const buffer0, const uint8_t* const buffer1, const uint8_t* const meanValues0, const uint8_t* const meanValues1);
73
74 /**
75 * Returns the zero-mean sum of square differences between two patches within an image.
76 * @param patch0 The top left start position of the first image patch, must be valid
77 * @param patch1 The top left start position of the second image patch, must be valid
78 * @param patch0StrideElements The number of elements between two rows for the first patch, in elements, with range [tChannels, tPatchSize, infinity)
79 * @param patch1StrideElements The number of elements between two rows for the second patch, in elements, with range [tChannels, tPatchSize, infinity)
80 * @param meanValues0 The mean values of the first patch, one for each channel, must be valid
81 * @param meanValues1 The mean values of the second patch, one for each channel, must be valid
82 * @return The resulting sum of square differences
83 * @tparam tPatchSize The size of the square patch (the edge length) in pixel, with range [1, infinity), must be odd
84 */
85 template <unsigned int tPatchSize>
86 static inline uint32_t patch8BitPerChannel(const uint8_t* patch0, const uint8_t* patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements, const uint8_t* const meanValues0, const uint8_t* const meanValues1);
87 };
88
89 public:
90
91 /**
92 * Returns the zero-mean sum of square differences between two memory buffers.
93 * @param buffer0 The first memory buffer, must be valid
94 * @param buffer1 The second memory buffer, must be valid
95 * @return The resulting sum of square differences
96 * @tparam tChannels Specifies the number of channels for the given buffers, with range [1, infinity)
97 * @tparam tPixels The number of pixels in the buffer, in pixels, with range [8, infinity)
98 */
99 template <unsigned int tChannels, unsigned int tPixels>
100 static inline uint32_t buffer8BitPerChannel(const uint8_t* const buffer0, const uint8_t* const buffer1);
101
102 /**
103 * Returns the zero-mean sum of square differences between two patches within an image.
104 * @param patch0 The top left start position of the first image patch, must be valid
105 * @param patch1 The top left start position of the second image patch, must be valid
106 * @param patch0StrideElements The number of elements between two rows for the first patch, in elements, with range [tChannels, tPatchSize, infinity)
107 * @param patch1StrideElements The number of elements between two rows for the second patch, in elements, with range [tChannels, tPatchSize, infinity)
108 * @return The resulting sum of square differences
109 * @tparam tChannels Specifies the number of channels for the given frames, with range [1, infinity)
110 * @tparam tPatchSize The size of the square patch (the edge length) in pixel, with range [5, infinity), must be odd
111 */
112 template <unsigned int tChannels, unsigned int tPatchSize>
113 static inline uint32_t patch8BitPerChannel(const uint8_t* const patch0, const uint8_t* const patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements);
114
115 /**
116 * Returns the zero-mean sum of square differences between an image patch and a buffer.
117 * @param patch0 The top left start position of the image patch, must be valid
118 * @param buffer1 The memory buffer, must be valid
119 * @param patch0StrideElements The number of elements between two rows for the image patch, in elements, with range [tChannels, tPatchSize, infinity)
120 * @return The resulting sum of square differences
121 * @tparam tChannels Specifies the number of channels for the given frames, with range [1, infinity)
122 * @tparam tPatchSize The size of the square patch (the edge length) in pixel, with range [5, infinity), must be odd
123 */
124 template <unsigned int tChannels, unsigned int tPatchSize>
125 static inline uint32_t patchBuffer8BitPerChannel(const uint8_t* const patch0, const uint8_t* const buffer1, const unsigned int patch0StrideElements);
126
127 /**
128 * Determines the mean value for a buffer, one value for each channel.
129 * @param buffer The memory buffer to be handled, must be valid
130 * @param meanValues The resulting mean values, one for each channel
131 * @tparam tChannels Specifies the number of channels for the given frames, with range [1, infinity)
132 * @tparam tPixels The number of pixels in the buffer, in pixels, with range [8, infinity)
133 */
134 template <unsigned int tChannels, unsigned int tPixels>
135 static OCEAN_FORCE_INLINE void mean8BitPerChannel(const uint8_t* const buffer, uint8_t* const meanValues);
136
137 /**
138 * Determines the mean value for an image patch, one value for each channel.
139 * @param patch The top left start position of the image patch, must be valid
140 * @param patchStrideElements The number of elements between two rows for the image patch, in elements, with range [tChannels, tPatchSize, infinity)
141 * @param meanValues The resulting mean values, one for each channel
142 * @tparam tChannels Specifies the number of channels for the given frames, with range [1, infinity)
143 * @tparam tPatchSize The size of the square patch (the edge length) in pixel, with range [5, infinity), must be odd
144 */
145 template <unsigned int tChannels, unsigned int tPatchSize>
146 static OCEAN_FORCE_INLINE void mean8BitPerChannel(const uint8_t* const patch, const unsigned int patchStrideElements, uint8_t* const meanValues);
147};
148
149template <>
150template <unsigned int tPixels>
151inline void ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<1u>::mean8BitPerChannel(const uint8_t* buffer, uint8_t* const meanValues)
152{
153 static_assert(tPixels >= 8u, "Invalid buffer size!");
154
155 constexpr unsigned int tChannels = 1u;
156
157 ocean_assert(buffer != nullptr && meanValues != nullptr);
158
159 constexpr unsigned int bufferElements = tChannels * tPixels;
160
161 constexpr unsigned int blocks16 = bufferElements / 16u;
162 constexpr unsigned int remainingAfterBlocks16 = bufferElements % 16u;
163
164 // the partial 16 block loads backwards beyond the buffer's start unless a full block precedes it
165 constexpr bool partialBlock16 = blocks16 >= 1u && remainingAfterBlocks16 > 8u;
166
167 constexpr bool fullBlock8 = !partialBlock16 && remainingAfterBlocks16 >= 8u;
168
169 constexpr unsigned int remainingAfterBlocks8 = fullBlock8 ? remainingAfterBlocks16 - 8u : remainingAfterBlocks16;
170
171 constexpr bool partialBlock8 = !partialBlock16 && remainingAfterBlocks8 >= 3u;
172
173 constexpr unsigned int blocks1 = (!partialBlock16 && !partialBlock8) ? remainingAfterBlocks8 : 0u;
174
175 static_assert(blocks1 <= 2u, "Invalid block size!");
176
177 __m128i sum_128i = _mm_setzero_si128();
178
179 uint32_t sumIndividual = 0u;
180
181 for (unsigned int n = 0u; n < blocks16; ++n)
182 {
183 const __m128i buffer_128i = _mm_lddqu_si128((const __m128i*)buffer);
184
185 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
186
187 buffer += 16;
188 }
189
190 if constexpr (partialBlock16)
191 {
192 constexpr unsigned int overlapElements = partialBlock16 ? 16u - remainingAfterBlocks16 : 0u;
193
194 static_assert(overlapElements < 8u, "Invalid value!");
195
196 const __m128i buffer_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(buffer - overlapElements)), overlapElements); // loading 16 elements, but shifting `overlapElements` zeros to the right
197
198 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
199
200 buffer += remainingAfterBlocks16;
201 }
202
203 if constexpr (fullBlock8)
204 {
205 const __m128i buffer_128i = _mm_loadl_epi64((const __m128i*)buffer); // load for unaligned 64 bit memory
206
207 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
208
209 buffer += 8;
210 }
211
212 if constexpr (partialBlock8)
213 {
214 constexpr unsigned int overlapElements = partialBlock8 ? 8u - remainingAfterBlocks8 : 0u;
215
216 static_assert(overlapElements < 8u, "Invalid value!");
217
218 const __m128i buffer_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(buffer - overlapElements)), overlapElements); // loading 8 elements, but shifting `overlapElements` zeros to the right
219
220 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
221
222 buffer += remainingAfterBlocks8;
223 }
224
225 if constexpr (blocks1 != 0u)
226 {
227 for (unsigned int n = 0u; n < blocks1; ++n)
228 {
229 sumIndividual += buffer[n];
230 }
231 }
232
233 const uint32_t sum = SSE::sum_u32_first_third(sum_128i) + sumIndividual;
234
235 meanValues[0] = uint8_t((sum + tPixels / 2u) / tPixels);
236}
237
238template <>
239template <unsigned int tPixels>
240inline void ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<3u>::mean8BitPerChannel(const uint8_t* buffer, uint8_t* const meanValues)
241{
242 static_assert(tPixels >= 8u, "Invalid buffer size!");
243
244 constexpr unsigned int tChannels = 3u;
245
246 ocean_assert(buffer != nullptr && meanValues != nullptr);
247
248 constexpr unsigned int bufferElements = tChannels * tPixels;
249
250 constexpr unsigned int blocks48 = bufferElements / 48u;
251 constexpr unsigned int remainingAfterFullBlocks48 = bufferElements % 48u;
252
253 constexpr bool partialBlock48 = remainingAfterFullBlocks48 > 2u * 16u;
254
255 constexpr unsigned int remainingAfterPartialBlock48 = partialBlock48 ? 0u : remainingAfterFullBlocks48;
256
257 constexpr unsigned int blocks24 = remainingAfterPartialBlock48 / 24u;
258
259 constexpr unsigned int remainingAfterPartialBlock24 = remainingAfterPartialBlock48 % 24u;
260
261 constexpr unsigned int blocks21 = remainingAfterPartialBlock24 / 21u;
262
263 constexpr unsigned int remainingAfterPartialBlock21 = remainingAfterPartialBlock24 % 21u;
264
265 constexpr unsigned int blocks15 = remainingAfterPartialBlock21 / 15u;
266
267 constexpr unsigned int remainingAfterPartialBlock15 = remainingAfterPartialBlock21 % 15u;
268
269 constexpr unsigned int blocks1 = remainingAfterPartialBlock15;
270
271 static_assert(blocks1 % 3u == 0u, "Invalid number of single blocks");
272
273 __m128i sumChannel0_128i = _mm_setzero_si128();
274 __m128i sumChannel1_128i = _mm_setzero_si128();
275 __m128i sumChannel2_128i = _mm_setzero_si128();
276
277 uint32_t sumIndividual[3] = {0u};
278
279 for (unsigned int n = 0u; n < blocks48; ++n)
280 {
281 const __m128i bufferA_128i = _mm_lddqu_si128((const __m128i*)(buffer + 0));
282 const __m128i bufferB_128i = _mm_lddqu_si128((const __m128i*)(buffer + 16));
283 const __m128i bufferC_128i = _mm_lddqu_si128((const __m128i*)(buffer + 32));
284
285 __m128i channel0;
286 __m128i channel1;
287 __m128i channel2;
288 SSE::deInterleave3Channel8Bit48Elements(bufferA_128i, bufferB_128i, bufferC_128i, channel0, channel1, channel2);
289
290 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_sad_epu8(channel0, _mm_setzero_si128()));
291 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_sad_epu8(channel1, _mm_setzero_si128()));
292 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2, _mm_setzero_si128()));
293
294 buffer += 48;
295 }
296
297 if constexpr (partialBlock48)
298 {
299 constexpr int overlappingElements = int(48u - remainingAfterFullBlocks48);
300
301 const __m128i bufferA_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)(buffer)), overlappingElements);
302 const __m128i bufferB_128i = _mm_lddqu_si128((const __m128i*)(buffer - overlappingElements + 16));
303 const __m128i bufferC_128i = _mm_lddqu_si128((const __m128i*)(buffer - overlappingElements + 32));
304
305 __m128i channel0;
306 __m128i channel1;
307 __m128i channel2;
308 SSE::deInterleave3Channel8Bit48Elements(bufferA_128i, bufferB_128i, bufferC_128i, channel0, channel1, channel2);
309
310 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_sad_epu8(channel0, _mm_setzero_si128()));
311 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_sad_epu8(channel1, _mm_setzero_si128()));
312 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2, _mm_setzero_si128()));
313
314 buffer += remainingAfterFullBlocks48;
315 }
316
317 for (unsigned int n = 0u; n < blocks24; ++n)
318 {
319 const __m128i bufferA_128i = _mm_lddqu_si128((const __m128i*)(buffer + 0));
320 const __m128i bufferB_128i = _mm_loadl_epi64((const __m128i*)(buffer + 16)); // load for unaligned 64 bit memory
321
322 __m128i channel01_128i;
323 __m128i channel2_128i;
324 SSE::deInterleave3Channel8Bit24Elements(bufferA_128i, bufferB_128i, channel01_128i, channel2_128i);
325
326 const __m128i sumChannel01_128i = _mm_sad_epu8(channel01_128i, _mm_setzero_si128());
327
328 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_slli_si128(sumChannel01_128i, 8));
329 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_srli_si128(sumChannel01_128i, 8));
330 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2_128i, _mm_setzero_si128()));
331
332 buffer += 24;
333 }
334
335 for (unsigned int n = 0u; n < blocks21; ++n)
336 {
337 const __m128i bufferA_128i = _mm_lddqu_si128((const __m128i*)(buffer + 0));
338 const __m128i bufferB_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(buffer + 16 - 3)), 3); // load for unaligned 64 bit memory
339
340 __m128i channel01_128i;
341 __m128i channel2_128i;
342 SSE::deInterleave3Channel8Bit24Elements(bufferA_128i, bufferB_128i, channel01_128i, channel2_128i);
343
344 const __m128i sumChannel01_128i = _mm_sad_epu8(channel01_128i, _mm_setzero_si128());
345
346 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_slli_si128(sumChannel01_128i, 8));
347 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_srli_si128(sumChannel01_128i, 8));
348 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2_128i, _mm_setzero_si128()));
349
350 buffer += 21;
351 }
352
353 for (unsigned int n = 0u; n < blocks15; ++n)
354 {
355 const __m128i buffer_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(buffer - 1)), 1);
356
357 __m128i channel01_128i;
358 __m128i channel2_128i;
359 SSE::deInterleave3Channel8Bit15Elements(buffer_128i, channel01_128i, channel2_128i);
360
361 const __m128i sumChannel01_128i = _mm_sad_epu8(channel01_128i, _mm_setzero_si128());
362
363 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_slli_si128(sumChannel01_128i, 8));
364 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_srli_si128(sumChannel01_128i, 8));
365 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2_128i, _mm_setzero_si128()));
366
367 buffer += 15;
368 }
369
370 if constexpr (blocks1 != 0u)
371 {
372 constexpr unsigned int pixels = blocks1 / 3u;
373
374 for (unsigned int x = 0u; x < pixels; ++x)
375 {
376 for (unsigned int n = 0u; n < 3u; ++n)
377 {
378 sumIndividual[n] += buffer[x * 3u + n];
379 }
380 }
381
382 buffer += blocks1;
383 }
384
385 meanValues[0] = uint8_t((SSE::sum_u32_first_third(sumChannel0_128i) + sumIndividual[0] + tPixels / 2u) / tPixels);
386 meanValues[1] = uint8_t((SSE::sum_u32_first_third(sumChannel1_128i) + sumIndividual[1] + tPixels / 2u) / tPixels);
387 meanValues[2] = uint8_t((SSE::sum_u32_first_third(sumChannel2_128i) + sumIndividual[2] + tPixels / 2u) / tPixels);
388}
389
390template <unsigned int tChannels>
391template <unsigned int tPixels>
392inline void ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<tChannels>::mean8BitPerChannel(const uint8_t* const buffer, uint8_t* const meanValues)
393{
394 static_assert(tChannels >= 1u, "Invalid channel number!");
395 static_assert(tPixels >= 1u, "Invalid buffer size!");
396
397 ocean_assert(buffer != nullptr && meanValues != nullptr);
398
399 uint32_t sum[tChannels] = {0u};
400
401 for (unsigned int n = 0u; n < tPixels; ++n)
402 {
403 for (unsigned int c = 0u; c < tChannels; ++c)
404 {
405 sum[c] += buffer[n * tChannels + c];
406 }
407 }
408
409 for (unsigned int c = 0u; c < tChannels; ++c)
410 {
411 meanValues[c] = uint8_t((sum[c] + tPixels / 2u) / tPixels);
412 }
413}
414
415template <>
416template <unsigned int tPatchSize>
417inline void ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<1u>::mean8BitPerChannel(const uint8_t* patch, const unsigned int patchStrideElements, uint8_t* const meanValues)
418{
419 static_assert(tPatchSize >= 5u, "Invalid patch size!");
420
421 constexpr unsigned int tChannels = 1u;
422
423 ocean_assert(patch != nullptr && meanValues != nullptr);
424
425 ocean_assert(patchStrideElements >= tChannels * tPatchSize);
426
427 constexpr unsigned int patchWidthElements = tChannels * tPatchSize;
428
429 constexpr unsigned int blocks16 = patchWidthElements / 16u;
430 constexpr unsigned int remainingAfterBlocks16 = patchWidthElements % 16u;
431
432 constexpr bool partialBlock16 = remainingAfterBlocks16 > 8u;
433
434 constexpr bool fullBlock8 = !partialBlock16 && remainingAfterBlocks16 == 8u;
435
436 constexpr bool partialBlock8 = !partialBlock16 && !fullBlock8 && remainingAfterBlocks16 >= 3u;
437
438 constexpr unsigned int blocks1 = (!partialBlock16 && !fullBlock8 && !partialBlock8) ? remainingAfterBlocks16 : 0u;
439
440 static_assert(blocks1 <= 2u, "Invalid block size!");
441
442 __m128i sum_128i = _mm_setzero_si128();
443
444 uint32_t sumIndividual = 0u;
445
446 for (unsigned int y = 0u; y < tPatchSize; ++y)
447 {
448 SSE::prefetchT0(patch + patchStrideElements);
449
450 for (unsigned int n = 0u; n < blocks16; ++n)
451 {
452 const __m128i buffer_128i = _mm_lddqu_si128((const __m128i*)patch);
453
454 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
455
456 patch += 16;
457 }
458
459 if constexpr (fullBlock8)
460 {
461 const __m128i buffer_128i = _mm_loadl_epi64((const __m128i*)patch); // load for unaligned 64 bit memory
462
463 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
464
465 patch += 8;
466 }
467
468 if constexpr (partialBlock16)
469 {
470 constexpr unsigned int overlapElements = partialBlock16 ? 16u - remainingAfterBlocks16 : 0u;
471
472 static_assert(overlapElements < 8u, "Invalid value!");
473
474 if (y < tPatchSize - 1u)
475 {
476 const __m128i buffer_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)patch), overlapElements); // loading 16 elements, but shifting `overlapElements` zeros to the left
477
478 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
479 }
480 else
481 {
482 const __m128i buffer_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(patch - overlapElements)), overlapElements); // loading 16 elements, but shifting `overlapElements` zeros to the right
483
484 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
485 }
486
487 patch += remainingAfterBlocks16;
488 }
489
490 if constexpr (partialBlock8)
491 {
492 constexpr unsigned int overlapElements = partialBlock8 ? 8u - remainingAfterBlocks16 : 0u;
493
494 static_assert(overlapElements < 8u, "Invalid value!");
495
496 if (y < tPatchSize - 1u)
497 {
498 const __m128i buffer_128i = _mm_slli_si128(_mm_loadl_epi64((const __m128i*)patch), overlapElements + 8); // loading 8 elements, but shifting `overlapElements` zeros to the left
499
500 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
501 }
502 else
503 {
504 const __m128i buffer_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(patch - overlapElements)), overlapElements); // loading 8 elements, but shifting `overlapElements` zeros to the right
505
506 sum_128i = _mm_add_epi32(sum_128i, _mm_sad_epu8(buffer_128i, _mm_setzero_si128()));
507 }
508
509 patch += remainingAfterBlocks16;
510 }
511
512 if constexpr (blocks1 != 0u)
513 {
514 for (unsigned int n = 0u; n < blocks1; ++n)
515 {
516 sumIndividual += patch[n];
517 }
518
519 patch += blocks1;
520 }
521
522 patch += patchStrideElements - patchWidthElements;
523 }
524
525 const uint32_t sum = SSE::sum_u32_first_third(sum_128i) + sumIndividual;
526
527 meanValues[0] = uint8_t((sum + tPatchSize * tPatchSize / 2u) / (tPatchSize * tPatchSize));
528}
529
530template <>
531template <unsigned int tPatchSize>
532inline void ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<3u>::mean8BitPerChannel(const uint8_t* patch, const unsigned int patchStrideElements, uint8_t* const meanValues)
533{
534 static_assert(tPatchSize >= 5u, "Invalid patch size!");
535
536 constexpr unsigned int tChannels = 3u;
537
538 ocean_assert(patch != nullptr && meanValues != nullptr);
539
540 ocean_assert(patchStrideElements >= tChannels * tPatchSize);
541
542 constexpr unsigned int patchWidthElements = tChannels * tPatchSize;
543
544 constexpr unsigned int blocks48 = patchWidthElements / 48u;
545 constexpr unsigned int remainingAfterFullBlocks48 = patchWidthElements % 48u;
546
547 constexpr bool partialBlock48 = remainingAfterFullBlocks48 > 2u * 16u;
548
549 constexpr unsigned int remainingAfterPartialBlock48 = partialBlock48 ? 0u : remainingAfterFullBlocks48;
550
551 constexpr unsigned int blocks24 = remainingAfterPartialBlock48 / 24u;
552
553 constexpr unsigned int remainingAfterPartialBlock24 = remainingAfterPartialBlock48 % 24u;
554
555 constexpr unsigned int blocks21 = remainingAfterPartialBlock24 / 21u;
556
557 constexpr unsigned int remainingAfterPartialBlock21 = remainingAfterPartialBlock24 % 21u;
558
559 constexpr unsigned int blocks15 = remainingAfterPartialBlock21 / 15u;
560
561 constexpr unsigned int remainingAfterPartialBlock15 = remainingAfterPartialBlock21 % 15u;
562
563 constexpr unsigned int blocks1 = remainingAfterPartialBlock15;
564
565 static_assert(blocks1 % 3u == 0u, "Invalid number of single blocks");
566
567 __m128i sumChannel0_128i = _mm_setzero_si128();
568 __m128i sumChannel1_128i = _mm_setzero_si128();
569 __m128i sumChannel2_128i = _mm_setzero_si128();
570
571 uint32_t sumIndividual[3] = {0u};
572
573 for (unsigned int y = 0u; y < tPatchSize; ++y)
574 {
575 SSE::prefetchT0(patch + patchStrideElements);
576
577 for (unsigned int n = 0u; n < blocks48; ++n)
578 {
579 const __m128i bufferA_128i = _mm_lddqu_si128((const __m128i*)(patch + 0));
580 const __m128i bufferB_128i = _mm_lddqu_si128((const __m128i*)(patch + 16));
581 const __m128i bufferC_128i = _mm_lddqu_si128((const __m128i*)(patch + 32));
582
583 __m128i channel0;
584 __m128i channel1;
585 __m128i channel2;
586 SSE::deInterleave3Channel8Bit48Elements(bufferA_128i, bufferB_128i, bufferC_128i, channel0, channel1, channel2);
587
588 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_sad_epu8(channel0, _mm_setzero_si128()));
589 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_sad_epu8(channel1, _mm_setzero_si128()));
590 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2, _mm_setzero_si128()));
591
592 patch += 48;
593 }
594
595 if constexpr (partialBlock48)
596 {
597 constexpr int overlappingElements = int(48u - remainingAfterFullBlocks48);
598
599 const __m128i bufferA_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)(patch)), overlappingElements);
600 const __m128i bufferB_128i = _mm_lddqu_si128((const __m128i*)(patch - overlappingElements + 16));
601 const __m128i bufferC_128i = _mm_lddqu_si128((const __m128i*)(patch - overlappingElements + 32));
602
603 __m128i channel0;
604 __m128i channel1;
605 __m128i channel2;
606 SSE::deInterleave3Channel8Bit48Elements(bufferA_128i, bufferB_128i, bufferC_128i, channel0, channel1, channel2);
607
608 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_sad_epu8(channel0, _mm_setzero_si128()));
609 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_sad_epu8(channel1, _mm_setzero_si128()));
610 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2, _mm_setzero_si128()));
611
612 patch += remainingAfterFullBlocks48;
613 }
614
615 for (unsigned int n = 0u; n < blocks24; ++n)
616 {
617 const __m128i bufferA_128i = _mm_lddqu_si128((const __m128i*)(patch + 0));
618 const __m128i bufferB_128i = _mm_loadl_epi64((const __m128i*)(patch + 16)); // load for unaligned 64 bit memory
619
620 __m128i channel01_128i;
621 __m128i channel2_128i;
622 SSE::deInterleave3Channel8Bit24Elements(bufferA_128i, bufferB_128i, channel01_128i, channel2_128i);
623
624 const __m128i sumChannel01_128i = _mm_sad_epu8(channel01_128i, _mm_setzero_si128());
625
626 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_slli_si128(sumChannel01_128i, 8));
627 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_srli_si128(sumChannel01_128i, 8));
628 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2_128i, _mm_setzero_si128()));
629
630 patch += 24;
631 }
632
633 for (unsigned int n = 0u; n < blocks21; ++n)
634 {
635 const __m128i bufferA_128i = _mm_lddqu_si128((const __m128i*)(patch + 0));
636 const __m128i bufferB_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(patch + 16 - 3)), 3); // load for unaligned 64 bit memory
637
638 __m128i channel01_128i;
639 __m128i channel2_128i;
640 SSE::deInterleave3Channel8Bit24Elements(bufferA_128i, bufferB_128i, channel01_128i, channel2_128i);
641
642 const __m128i sumChannel01_128i = _mm_sad_epu8(channel01_128i, _mm_setzero_si128());
643
644 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_slli_si128(sumChannel01_128i, 8));
645 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_srli_si128(sumChannel01_128i, 8));
646 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2_128i, _mm_setzero_si128()));
647
648 patch += 21;
649 }
650
651 for (unsigned int n = 0u; n < blocks15; ++n)
652 {
653 const __m128i buffer_128i = y < tPatchSize - 1u ? _mm_lddqu_si128((const __m128i*)(patch)) : _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(patch - 1)), 1);
654
655 __m128i channel01_128i;
656 __m128i channel2_128i;
657 SSE::deInterleave3Channel8Bit15Elements(buffer_128i, channel01_128i, channel2_128i);
658
659 const __m128i sumChannel01_128i = _mm_sad_epu8(channel01_128i, _mm_setzero_si128());
660
661 sumChannel0_128i = _mm_add_epi32(sumChannel0_128i, _mm_slli_si128(sumChannel01_128i, 8));
662 sumChannel1_128i = _mm_add_epi32(sumChannel1_128i, _mm_srli_si128(sumChannel01_128i, 8));
663 sumChannel2_128i = _mm_add_epi32(sumChannel2_128i, _mm_sad_epu8(channel2_128i, _mm_setzero_si128()));
664
665 patch += 15;
666 }
667
668 if constexpr (blocks1 != 0u)
669 {
670 constexpr unsigned int pixels = blocks1 / 3u;
671
672 for (unsigned int x = 0u; x < pixels; ++x)
673 {
674 for (unsigned int n = 0u; n < 3u; ++n)
675 {
676 sumIndividual[n] += patch[x * 3u + n];
677 }
678 }
679
680 patch += blocks1;
681 }
682
683 patch += patchStrideElements - patchWidthElements;
684 }
685
686 meanValues[0] = uint8_t((SSE::sum_u32_first_third(sumChannel0_128i) + sumIndividual[0] + tPatchSize * tPatchSize / 2u) / (tPatchSize * tPatchSize));
687 meanValues[1] = uint8_t((SSE::sum_u32_first_third(sumChannel1_128i) + sumIndividual[1] + tPatchSize * tPatchSize / 2u) / (tPatchSize * tPatchSize));
688 meanValues[2] = uint8_t((SSE::sum_u32_first_third(sumChannel2_128i) + sumIndividual[2] + tPatchSize * tPatchSize / 2u) / (tPatchSize * tPatchSize));
689}
690
691template <unsigned int tChannels>
692template <unsigned int tPatchSize>
693inline void ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<tChannels>::mean8BitPerChannel(const uint8_t* patch, const unsigned int patchStrideElements, uint8_t* const meanValues)
694{
695 static_assert(tChannels >= 1u, "Invalid channel number!");
696 static_assert(tPatchSize >= 1u, "Invalid patch size!");
697
698 ocean_assert(patch != nullptr && meanValues != nullptr);
699
700 ocean_assert(patchStrideElements >= tChannels * tPatchSize);
701
702 uint32_t sum[tChannels] = {0u};
703
704 for (unsigned int y = 0u; y < tPatchSize; ++y)
705 {
706 for (unsigned int x = 0u; x < tPatchSize; ++x)
707 {
708 for (unsigned int n = 0u; n < tChannels; ++n)
709 {
710 sum[n] += patch[x * tChannels + n];
711 }
712 }
713
714 patch += patchStrideElements;
715 }
716
717 for (unsigned int n = 0u; n < tChannels; ++n)
718 {
719 meanValues[n] = uint8_t((sum[n] + tPatchSize * tPatchSize / 2u) / (tPatchSize * tPatchSize));
720 }
721}
722
723template <>
724template <unsigned int tPixels>
725inline uint32_t ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<1u>::buffer8BitPerChannel(const uint8_t* buffer0, const uint8_t* buffer1, const uint8_t* const meanValues0, const uint8_t* const meanValues1)
726{
727 static_assert(tPixels >= 8u, "Invalid pixel number!");
728
729 constexpr unsigned int tChannels = 1u;
730
731 ocean_assert(buffer0 != nullptr && buffer1 != nullptr);
732 ocean_assert(meanValues0 != nullptr && meanValues1 != nullptr);
733
734 constexpr unsigned int bufferElements = tChannels * tPixels;
735
736 constexpr unsigned int blocks16 = bufferElements / 16u;
737 constexpr unsigned int remainingAfterBlocks16 = bufferElements % 16u;
738
739 // the partial 16 block loads backwards beyond the buffer's start unless a full block precedes it
740 constexpr bool partialBlock16 = blocks16 >= 1u && remainingAfterBlocks16 > 8u;
741
742 constexpr bool fullBlock8 = !partialBlock16 && remainingAfterBlocks16 >= 8u;
743
744 constexpr unsigned int remainingAfterBlocks8 = fullBlock8 ? remainingAfterBlocks16 - 8u : remainingAfterBlocks16;
745
746 constexpr bool partialBlock8 = !partialBlock16 && remainingAfterBlocks8 >= 3u;
747
748 constexpr unsigned int blocks1 = (!partialBlock16 && !partialBlock8) ? remainingAfterBlocks8 : 0u;
749
750 static_assert(blocks1 <= 2u, "Invalid block size!");
751
752 static_assert(std::is_same<short, int16_t>::value, "Invalid data type!");
753
754 const __m128i constant_signs_m128i = _mm_set1_epi16(short(0x1FF)); // -1, 1, -1, 1, -1, 1, -1, 1
755
756 const __m128i mean0_128i = _mm_set1_epi8(meanValues0[0]);
757 const __m128i mean1_128i = _mm_set1_epi8(meanValues1[0]);
758
759 __m128i sum0_128i = _mm_setzero_si128();
760 __m128i sum1_128i = _mm_setzero_si128();
761
762 uint32_t sumIndividual = 0u;
763
764 for (unsigned int n = 0u; n < blocks16; ++n)
765 {
766 const __m128i buffer0_128i = _mm_lddqu_si128((const __m128i*)buffer0);
767 const __m128i buffer1_128i = _mm_lddqu_si128((const __m128i*)buffer1);
768
769 const __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)); // (buffer0 - mean0) - (buffer1 - mean1)
770 const __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_128i, buffer1_128i), constant_signs_m128i));
771
772 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
773 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
774
775 buffer0 += 16;
776 buffer1 += 16;
777 }
778
779 if constexpr (partialBlock16)
780 {
781 constexpr unsigned int overlapElements = partialBlock16 ? 16u - remainingAfterBlocks16 : 0u;
782
783 static_assert(overlapElements < 8u, "Invalid value!");
784
785 const __m128i buffer0_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(buffer0 - overlapElements)), overlapElements); // loading 16 elements, but shifting `overlapElements` zeros to the right
786 const __m128i buffer1_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(buffer1 - overlapElements)), overlapElements);
787
788 const __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)); // (buffer0 - mean0) - (buffer1 - mean1)
789 const __m128i absDifferencesHigh_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)), overlapElements * 2);
790
791 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
792 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
793
794 buffer0 += remainingAfterBlocks16;
795 buffer1 += remainingAfterBlocks16;
796 }
797
798 if constexpr (fullBlock8)
799 {
800 const __m128i buffer0_128i = _mm_loadl_epi64((const __m128i*)buffer0); // load for unaligned 64 bit memory
801 const __m128i buffer1_128i = _mm_loadl_epi64((const __m128i*)buffer1); // load for unaligned 64 bit memory
802
803 const __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)); // (buffer0 - mean0) - (buffer1 - mean1)
804
805 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
806
807 buffer0 += 8;
808 buffer1 += 8;
809 }
810
811 if constexpr (partialBlock8)
812 {
813 constexpr unsigned int overlapElements = partialBlock8 ? 8u - remainingAfterBlocks8 : 0u;
814
815 static_assert(overlapElements < 8u, "Invalid value!");
816
817 const __m128i buffer0_128i = _mm_loadl_epi64((const __m128i*)(buffer0 - overlapElements)); // loading 8 elements
818 const __m128i buffer1_128i = _mm_loadl_epi64((const __m128i*)(buffer1 - overlapElements));
819
820 const __m128i absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)), overlapElements * 2); // (buffer0 - mean0) - (buffer1 - mean1)
821
822 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
823
824 buffer0 += remainingAfterBlocks8;
825 buffer1 += remainingAfterBlocks8;
826 }
827
828 if constexpr (blocks1 != 0u)
829 {
830 for (unsigned int n = 0u; n < blocks1; ++n)
831 {
832 sumIndividual += sqrDistance(buffer0[n] - meanValues0[0], buffer1[n] - meanValues1[0]);
833 }
834
835 buffer0 += blocks1;
836 buffer1 += blocks1;
837 }
838
839 return SSE::sum_u32_4(sum0_128i) + SSE::sum_u32_4(sum1_128i) + sumIndividual;
840}
841
842template <>
843template <unsigned int tPixels>
844inline uint32_t ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<3u>::buffer8BitPerChannel(const uint8_t* buffer0, const uint8_t* buffer1, const uint8_t* const meanValues0, const uint8_t* const meanValues1)
845{
846 static_assert(tPixels >= 5u, "Invalid pixel number!");
847
848 constexpr unsigned int tChannels = 3u;
849
850 ocean_assert(buffer0 != nullptr && buffer1 != nullptr);
851 ocean_assert(meanValues0 != nullptr && meanValues1 != nullptr);
852
853 constexpr unsigned int bufferElements = tChannels * tPixels;
854
855 constexpr unsigned int blocks48 = bufferElements / 48u;
856 constexpr unsigned int remainingAfterFullBlocks48 = bufferElements % 48u;
857
858 constexpr bool partialBlock48 = remainingAfterFullBlocks48 > 2u * 16u;
859
860 constexpr unsigned int remainingAfterPartialBlock48 = partialBlock48 ? 0u : remainingAfterFullBlocks48;
861
862 constexpr unsigned int blocks24 = remainingAfterPartialBlock48 / 24u;
863
864 constexpr unsigned int remainingAfterPartialBlock24 = remainingAfterPartialBlock48 % 24u;
865
866 constexpr unsigned int blocks21 = remainingAfterPartialBlock24 / 21u;
867
868 constexpr unsigned int remainingAfterPartialBlock21 = remainingAfterPartialBlock24 % 21u;
869
870 constexpr unsigned int blocks15 = remainingAfterPartialBlock21 / 15u;
871
872 constexpr unsigned int remainingAfterPartialBlock15 = remainingAfterPartialBlock21 % 15u;
873
874 constexpr unsigned int blocks1 = remainingAfterPartialBlock15;
875
876 static_assert(blocks1 % 3u == 0u, "Invalid number of single blocks");
877
878 static_assert(std::is_same<short, int16_t>::value, "Invalid data type!");
879
880 const __m128i constant_signs_m128i = _mm_set1_epi16(short(0x1FF)); // -1, 1, -1, 1, -1, 1, -1, 1
881
882 const __m128i mean0_0_128i = _mm_set1_epi8(meanValues0[0]);
883 const __m128i mean0_1_128i = _mm_set1_epi8(meanValues0[1]);
884 const __m128i mean0_2_128i = _mm_set1_epi8(meanValues0[2]);
885
886 const __m128i mean1_0_128i = _mm_set1_epi8(meanValues1[0]);
887 const __m128i mean1_1_128i = _mm_set1_epi8(meanValues1[1]);
888 const __m128i mean1_2_128i = _mm_set1_epi8(meanValues1[2]);
889
890 __m128i sum0_128i = _mm_setzero_si128();
891 __m128i sum1_128i = _mm_setzero_si128();
892
893 uint32_t sumIndividual = 0u;
894
895 for (unsigned int n = 0u; n < blocks48; ++n)
896 {
897 const __m128i buffer0A_128i = _mm_lddqu_si128((const __m128i*)(buffer0 + 0));
898 const __m128i buffer0B_128i = _mm_lddqu_si128((const __m128i*)(buffer0 + 16));
899 const __m128i buffer0C_128i = _mm_lddqu_si128((const __m128i*)(buffer0 + 32));
900
901 __m128i channel0_0_128i;
902 __m128i channel0_1_128i;
903 __m128i channel0_2_128i;
904 SSE::deInterleave3Channel8Bit48Elements(buffer0A_128i, buffer0B_128i, buffer0C_128i, channel0_0_128i, channel0_1_128i, channel0_2_128i);
905
906 const __m128i buffer1A_128i = _mm_lddqu_si128((const __m128i*)(buffer1 + 0));
907 const __m128i buffer1B_128i = _mm_lddqu_si128((const __m128i*)(buffer1 + 16));
908 const __m128i buffer1C_128i = _mm_lddqu_si128((const __m128i*)(buffer1 + 32));
909
910 __m128i channel1_0_128i;
911 __m128i channel1_1_128i;
912 __m128i channel1_2_128i;
913 SSE::deInterleave3Channel8Bit48Elements(buffer1A_128i, buffer1B_128i, buffer1C_128i, channel1_0_128i, channel1_1_128i, channel1_2_128i);
914
915 __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i)); // (channel0_0 - mean0) - (channel1_0 - mean1)
916 __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i));
917
918 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
919 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
920
921 absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i));
922 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i));
923
924 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
925 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
926
927 absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
928 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
929
930 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
931 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
932
933 buffer0 += 48;
934 buffer1 += 48;
935 }
936
937 if constexpr (partialBlock48)
938 {
939 constexpr int overlappingElements = int(48u - remainingAfterFullBlocks48);
940 constexpr int overlappingPixels = overlappingElements / int(tChannels);
941
942 const __m128i buffer0A_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)(buffer0)), overlappingElements);
943 const __m128i buffer0B_128i = _mm_lddqu_si128((const __m128i*)(buffer0 - overlappingElements + 16));
944 const __m128i buffer0C_128i = _mm_lddqu_si128((const __m128i*)(buffer0 - overlappingElements + 32));
945
946 __m128i channel0_0_128i;
947 __m128i channel0_1_128i;
948 __m128i channel0_2_128i;
949 SSE::deInterleave3Channel8Bit48Elements(buffer0A_128i, buffer0B_128i, buffer0C_128i, channel0_0_128i, channel0_1_128i, channel0_2_128i);
950
951 const __m128i buffer1A_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)(buffer1)), overlappingElements);
952 const __m128i buffer1B_128i = _mm_lddqu_si128((const __m128i*)(buffer1 - overlappingElements + 16));
953 const __m128i buffer1C_128i = _mm_lddqu_si128((const __m128i*)(buffer1 - overlappingElements + 32));
954
955 __m128i channel1_0_128i;
956 __m128i channel1_1_128i;
957 __m128i channel1_2_128i;
958 SSE::deInterleave3Channel8Bit48Elements(buffer1A_128i, buffer1B_128i, buffer1C_128i, channel1_0_128i, channel1_1_128i, channel1_2_128i);
959
960 __m128i absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i)), overlappingPixels * 2); // (channel0_0 - mean0) - (channel1_0 - mean1)
961 __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i));
962
963 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
964 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
965
966 absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i)), overlappingPixels * 2);
967 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i));
968
969 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
970 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
971
972 absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i)), overlappingPixels * 2);
973 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
974
975 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
976 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
977
978 buffer0 += remainingAfterFullBlocks48;
979 buffer1 += remainingAfterFullBlocks48;
980 }
981
982 for (unsigned int n = 0u; n < blocks24; ++n)
983 {
984 const __m128i buffer0A_128i = _mm_lddqu_si128((const __m128i*)(buffer0 + 0));
985 const __m128i buffer0B_128i = _mm_loadl_epi64((const __m128i*)(buffer0 + 16)); // load for unaligned 64 bit memory
986
987 __m128i channel0_01_128i;
988 __m128i channel0_2_128i;
989 SSE::deInterleave3Channel8Bit24Elements(buffer0A_128i, buffer0B_128i, channel0_01_128i, channel0_2_128i);
990
991 const __m128i buffer1A_128i = _mm_lddqu_si128((const __m128i*)(buffer1 + 0));
992 const __m128i buffer1B_128i = _mm_loadl_epi64((const __m128i*)(buffer1 + 16)); // load for unaligned 64 bit memory
993
994 __m128i channel1_01_128i;
995 __m128i channel1_2_128i;
996 SSE::deInterleave3Channel8Bit24Elements(buffer1A_128i, buffer1B_128i, channel1_01_128i, channel1_2_128i);
997
998 __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_01_128i), constant_signs_m128i)); // (channel0_01 - mean0) - (channel1_01 - mean1)
999 __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_01_128i), constant_signs_m128i));
1000
1001 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1002 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1003
1004 absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
1005
1006 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1007
1008 buffer0 += 24;
1009 buffer1 += 24;
1010 }
1011
1012 for (unsigned int n = 0u; n < blocks21; ++n)
1013 {
1014 const __m128i buffer0A_128i = _mm_lddqu_si128((const __m128i*)(buffer0 + 0));
1015 const __m128i buffer0B_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(buffer0 + 16 - 3)), 3); // load for unaligned 64 bit memory
1016
1017 __m128i channel0_01_128i;
1018 __m128i channel0_2_128i;
1019 SSE::deInterleave3Channel8Bit24Elements(buffer0A_128i, buffer0B_128i, channel0_01_128i, channel0_2_128i);
1020
1021 const __m128i buffer1A_128i = _mm_lddqu_si128((const __m128i*)(buffer1 + 0));
1022 const __m128i buffer1B_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(buffer1 + 16 - 3)), 3); // load for unaligned 64 bit memory
1023
1024 __m128i channel1_01_128i;
1025 __m128i channel1_2_128i;
1026 SSE::deInterleave3Channel8Bit24Elements(buffer1A_128i, buffer1B_128i, channel1_01_128i, channel1_2_128i);
1027
1028 __m128i absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_01_128i), constant_signs_m128i)), 2); // (channel0_01 - mean0) - (channel1_01 - mean1)
1029 __m128i absDifferencesHigh_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_01_128i), constant_signs_m128i)), 2);
1030
1031 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1032 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1033
1034 absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i)), 2);
1035
1036 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1037
1038 buffer0 += 21;
1039 buffer1 += 21;
1040 }
1041
1042 for (unsigned int n = 0u; n < blocks15; ++n)
1043 {
1044 const __m128i buffer0_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(buffer0 - 1)), 1);
1045
1046 __m128i channel0_01_128i;
1047 __m128i channel0_2_128i;
1048 SSE::deInterleave3Channel8Bit15Elements(buffer0_128i, channel0_01_128i, channel0_2_128i);
1049
1050 const __m128i buffer1_128i = _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(buffer1 - 1)), 1);
1051
1052 __m128i channel1_01_128i;
1053 __m128i channel1_2_128i;
1054 SSE::deInterleave3Channel8Bit15Elements(buffer1_128i, channel1_01_128i, channel1_2_128i);
1055
1056 __m128i absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_01_128i), constant_signs_m128i)), 6); // (channel0_01 - mean0) - (channel1_01 - mean1)
1057 __m128i absDifferencesHigh_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_01_128i), constant_signs_m128i)), 6);
1058
1059 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1060 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1061
1062 absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i)), 6);
1063
1064 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1065
1066 buffer0 += 15;
1067 buffer1 += 15;
1068 }
1069
1070 if constexpr (blocks1 != 0u)
1071 {
1072 constexpr unsigned int pixels = blocks1 / 3u;
1073
1074 for (unsigned int x = 0u; x < pixels; ++x)
1075 {
1076 for (unsigned int n = 0u; n < 3u; ++n)
1077 {
1078 sumIndividual += sqrDistance(buffer0[x * 3u + n] - meanValues0[n], buffer1[x * 3u + n] - meanValues1[n]);
1079 }
1080 }
1081
1082 buffer0 += blocks1;
1083 buffer1 += blocks1;
1084 }
1085
1086 return SSE::sum_u32_4(sum0_128i) + SSE::sum_u32_4(sum1_128i) + sumIndividual;
1087}
1088
1089template <unsigned int tChannels>
1090template <unsigned int tPixels>
1091inline uint32_t ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<tChannels>::buffer8BitPerChannel(const uint8_t* const buffer0, const uint8_t* const buffer1, const uint8_t* const meanValues0, const uint8_t* const meanValues1)
1092{
1093 static_assert(tChannels >= 1u, "Invalid channel number!");
1094 static_assert(tPixels >= 1u, "Invalid patch size!");
1095
1096 ocean_assert(buffer0 != nullptr && buffer1 != nullptr);
1097 ocean_assert(meanValues0 != nullptr && meanValues1 != nullptr);
1098
1099 uint32_t ssd = 0u;
1100
1101 for (unsigned int n = 0u; n < tPixels; ++n)
1102 {
1103 for (unsigned int c = 0u; c < tChannels; ++c)
1104 {
1105 ssd += sqrDistance(buffer0[n * tChannels + c] - meanValues0[c], buffer1[n * tChannels + c] - meanValues1[c]);
1106 }
1107 }
1108
1109 return ssd;
1110}
1111
1112template <>
1113template <unsigned int tPatchSize>
1114inline uint32_t ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<1u>::patch8BitPerChannel(const uint8_t* patch0, const uint8_t* patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements, const uint8_t* const meanValues0, const uint8_t* const meanValues1)
1115{
1116 static_assert(tPatchSize >= 1u, "Invalid patch size!");
1117
1118 constexpr unsigned int tChannels = 1u;
1119
1120 ocean_assert(patch0 != nullptr && patch1 != nullptr);
1121 ocean_assert(meanValues0 != nullptr && meanValues1 != nullptr);
1122
1123 ocean_assert(patch0StrideElements >= tChannels * tPatchSize);
1124 ocean_assert(patch1StrideElements >= tChannels * tPatchSize);
1125
1126 constexpr unsigned int patchWidthElements = tChannels * tPatchSize;
1127
1128 constexpr unsigned int blocks16 = patchWidthElements / 16u;
1129 constexpr unsigned int remainingAfterBlocks16 = patchWidthElements % 16u;
1130
1131 constexpr bool partialBlock16 = remainingAfterBlocks16 > 8u;
1132
1133 constexpr bool fullBlock8 = !partialBlock16 && remainingAfterBlocks16 == 8u;
1134
1135 constexpr bool partialBlock8 = !partialBlock16 && !fullBlock8 && remainingAfterBlocks16 >= 3u;
1136
1137 constexpr unsigned int blocks1 = (!partialBlock16 && !fullBlock8 && !partialBlock8) ? remainingAfterBlocks16 : 0u;
1138
1139 static_assert(blocks1 <= 2u, "Invalid block size!");
1140
1141 static_assert(std::is_same<short, int16_t>::value, "Invalid data type!");
1142
1143 const __m128i constant_signs_m128i = _mm_set1_epi16(short(0x1FF)); // -1, 1, -1, 1, -1, 1, -1, 1
1144
1145 const __m128i mean0_128i = _mm_set1_epi8(meanValues0[0]);
1146 const __m128i mean1_128i = _mm_set1_epi8(meanValues1[0]);
1147
1148 __m128i sum0_128i = _mm_setzero_si128();
1149 __m128i sum1_128i = _mm_setzero_si128();
1150
1151 uint32_t sumIndividual = 0u;
1152
1153 for (unsigned int y = 0u; y < tPatchSize; ++y)
1154 {
1155 SSE::prefetchT0(patch0 + patch0StrideElements);
1156 SSE::prefetchT0(patch1 + patch1StrideElements);
1157
1158 for (unsigned int n = 0u; n < blocks16; ++n)
1159 {
1160 const __m128i buffer0_128i = _mm_lddqu_si128((const __m128i*)patch0);
1161 const __m128i buffer1_128i = _mm_lddqu_si128((const __m128i*)patch1);
1162
1163 const __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)); // (buffer0 - mean0) - (buffer1 - mean1)
1164 const __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_128i, buffer1_128i), constant_signs_m128i));
1165
1166 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1167 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1168
1169 patch0 += 16;
1170 patch1 += 16;
1171 }
1172
1173 if constexpr (fullBlock8)
1174 {
1175 const __m128i buffer0_128i = _mm_loadl_epi64((const __m128i*)patch0); // load for unaligned 64 bit memory
1176 const __m128i buffer1_128i = _mm_loadl_epi64((const __m128i*)patch1); // load for unaligned 64 bit memory
1177
1178 const __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)); // (buffer0 - mean0) - (buffer1 - mean1)
1179 const __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_128i, buffer1_128i), constant_signs_m128i));
1180
1181 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1182 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1183
1184 patch0 += 8;
1185 patch1 += 8;
1186 }
1187
1188 if constexpr (partialBlock16)
1189 {
1190 constexpr unsigned int overlapElements = partialBlock16 ? 16u - remainingAfterBlocks16 : 0u;
1191
1192 static_assert(overlapElements < 8u, "Invalid value!");
1193
1194 if (y < tPatchSize - 1u)
1195 {
1196 const __m128i buffer0_128i = _mm_lddqu_si128((const __m128i*)patch0); // loading 16 elements
1197 const __m128i buffer1_128i = _mm_lddqu_si128((const __m128i*)patch1);
1198
1199 const __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)); // (buffer0 - mean0) - (buffer1 - mean1)
1200 const __m128i absDifferencesHigh_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)), overlapElements * 2);
1201
1202 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1203 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1204 }
1205 else
1206 {
1207 const __m128i buffer0_128i = _mm_lddqu_si128((const __m128i*)(patch0 - overlapElements)); // loading 16 elements
1208 const __m128i buffer1_128i = _mm_lddqu_si128((const __m128i*)(patch1 - overlapElements));
1209
1210 const __m128i absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)), overlapElements * 2); // (buffer0 - mean0) - (buffer1 - mean1)
1211 const __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_128i, buffer1_128i), constant_signs_m128i));
1212
1213 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1214 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1215 }
1216
1217 patch0 += remainingAfterBlocks16;
1218 patch1 += remainingAfterBlocks16;
1219 }
1220
1221 if constexpr (partialBlock8)
1222 {
1223 constexpr unsigned int overlapElements = partialBlock8 ? 8u - remainingAfterBlocks16 : 0u;
1224
1225 static_assert(overlapElements < 8u, "Invalid value!");
1226
1227 if (y < tPatchSize - 1u)
1228 {
1229 const __m128i buffer0_128i = _mm_loadl_epi64((const __m128i*)patch0);// loading 8 elements
1230 const __m128i buffer1_128i = _mm_loadl_epi64((const __m128i*)patch1);
1231
1232 const __m128i absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)), overlapElements * 2); // (buffer0 - mean0) - (buffer1 - mean1)
1233
1234 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1235 }
1236 else
1237 {
1238 const __m128i buffer0_128i = _mm_loadl_epi64((const __m128i*)(patch0 - overlapElements)); // loading 8 elements
1239 const __m128i buffer1_128i = _mm_loadl_epi64((const __m128i*)(patch1 - overlapElements));
1240
1241 const __m128i absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_128i, buffer0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_128i, buffer1_128i), constant_signs_m128i)), overlapElements * 2); // (buffer0 - mean0) - (buffer1 - mean1)
1242
1243 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1244 }
1245
1246 patch0 += remainingAfterBlocks16;
1247 patch1 += remainingAfterBlocks16;
1248 }
1249
1250 if constexpr (blocks1 != 0u)
1251 {
1252 for (unsigned int n = 0u; n < blocks1; ++n)
1253 {
1254 sumIndividual += sqrDistance(patch0[n] - meanValues0[0], patch1[n] - meanValues1[0]);
1255 }
1256
1257 patch0 += blocks1;
1258 patch1 += blocks1;
1259 }
1260
1261 patch0 += patch0StrideElements - patchWidthElements;
1262 patch1 += patch1StrideElements - patchWidthElements;
1263 }
1264
1265 return SSE::sum_u32_4(sum0_128i) + SSE::sum_u32_4(sum1_128i) + sumIndividual;
1266}
1267
1268template <>
1269template <unsigned int tPatchSize>
1270inline uint32_t ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<3u>::patch8BitPerChannel(const uint8_t* patch0, const uint8_t* patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements, const uint8_t* const meanValues0, const uint8_t* const meanValues1)
1271{
1272 static_assert(tPatchSize >= 5u, "Invalid patch size!");
1273
1274 constexpr unsigned int tChannels = 3u;
1275
1276 ocean_assert(patch0 != nullptr && patch1 != nullptr);
1277 ocean_assert(meanValues0 != nullptr && meanValues1 != nullptr);
1278
1279 ocean_assert(patch0StrideElements >= tChannels * tPatchSize);
1280 ocean_assert(patch1StrideElements >= tChannels * tPatchSize);
1281
1282 constexpr unsigned int patchWidthElements = tChannels * tPatchSize;
1283
1284 constexpr unsigned int blocks48 = patchWidthElements / 48u;
1285 constexpr unsigned int remainingAfterFullBlocks48 = patchWidthElements % 48u;
1286
1287 constexpr bool partialBlock48 = remainingAfterFullBlocks48 > 2u * 16u;
1288
1289 constexpr unsigned int remainingAfterPartialBlock48 = partialBlock48 ? 0u : remainingAfterFullBlocks48;
1290
1291 constexpr unsigned int blocks24 = remainingAfterPartialBlock48 / 24u;
1292
1293 constexpr unsigned int remainingAfterPartialBlock24 = remainingAfterPartialBlock48 % 24u;
1294
1295 constexpr unsigned int blocks21 = remainingAfterPartialBlock24 / 21u;
1296
1297 constexpr unsigned int remainingAfterPartialBlock21 = remainingAfterPartialBlock24 % 21u;
1298
1299 constexpr unsigned int blocks15 = remainingAfterPartialBlock21 / 15u;
1300
1301 constexpr unsigned int remainingAfterPartialBlock15 = remainingAfterPartialBlock21 % 15u;
1302
1303 constexpr unsigned int blocks1 = remainingAfterPartialBlock15;
1304
1305 static_assert(blocks1 % 3u == 0u, "Invalid number of single blocks");
1306
1307 static_assert(std::is_same<short, int16_t>::value, "Invalid data type!");
1308
1309 const __m128i constant_signs_m128i = _mm_set1_epi16(short(0x1FF)); // -1, 1, -1, 1, -1, 1, -1, 1
1310
1311 const __m128i mean0_0_128i = _mm_set1_epi8(meanValues0[0]);
1312 const __m128i mean0_1_128i = _mm_set1_epi8(meanValues0[1]);
1313 const __m128i mean0_2_128i = _mm_set1_epi8(meanValues0[2]);
1314
1315 const __m128i mean1_0_128i = _mm_set1_epi8(meanValues1[0]);
1316 const __m128i mean1_1_128i = _mm_set1_epi8(meanValues1[1]);
1317 const __m128i mean1_2_128i = _mm_set1_epi8(meanValues1[2]);
1318
1319 __m128i sum0_128i = _mm_setzero_si128();
1320 __m128i sum1_128i = _mm_setzero_si128();
1321
1322 uint32_t sumIndividual = 0u;
1323
1324 for (unsigned int y = 0u; y < tPatchSize; ++y)
1325 {
1326 SSE::prefetchT0(patch0 + patch0StrideElements);
1327 SSE::prefetchT0(patch1 + patch1StrideElements);
1328
1329 for (unsigned int n = 0u; n < blocks48; ++n)
1330 {
1331 const __m128i buffer0A_128i = _mm_lddqu_si128((const __m128i*)(patch0 + 0));
1332 const __m128i buffer0B_128i = _mm_lddqu_si128((const __m128i*)(patch0 + 16));
1333 const __m128i buffer0C_128i = _mm_lddqu_si128((const __m128i*)(patch0 + 32));
1334
1335 __m128i channel0_0_128i;
1336 __m128i channel0_1_128i;
1337 __m128i channel0_2_128i;
1338 SSE::deInterleave3Channel8Bit48Elements(buffer0A_128i, buffer0B_128i, buffer0C_128i, channel0_0_128i, channel0_1_128i, channel0_2_128i);
1339
1340 const __m128i buffer1A_128i = _mm_lddqu_si128((const __m128i*)(patch1 + 0));
1341 const __m128i buffer1B_128i = _mm_lddqu_si128((const __m128i*)(patch1 + 16));
1342 const __m128i buffer1C_128i = _mm_lddqu_si128((const __m128i*)(patch1 + 32));
1343
1344 __m128i channel1_0_128i;
1345 __m128i channel1_1_128i;
1346 __m128i channel1_2_128i;
1347 SSE::deInterleave3Channel8Bit48Elements(buffer1A_128i, buffer1B_128i, buffer1C_128i, channel1_0_128i, channel1_1_128i, channel1_2_128i);
1348
1349 __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i)); // (channel0_0 - mean0) - (channel1_0 - mean1)
1350 __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i));
1351
1352 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1353 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1354
1355 absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i));
1356 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i));
1357
1358 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1359 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1360
1361 absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
1362 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
1363
1364 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1365 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1366
1367 patch0 += 48;
1368 patch1 += 48;
1369 }
1370
1371 if constexpr (partialBlock48)
1372 {
1373 constexpr int overlappingElements = int(48u - remainingAfterFullBlocks48);
1374 constexpr int overlappingPixels = overlappingElements / int(tChannels);
1375
1376 const __m128i buffer0A_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)(patch0)), overlappingElements);
1377 const __m128i buffer0B_128i = _mm_lddqu_si128((const __m128i*)(patch0 - overlappingElements + 16));
1378 const __m128i buffer0C_128i = _mm_lddqu_si128((const __m128i*)(patch0 - overlappingElements + 32));
1379
1380 __m128i channel0_0_128i;
1381 __m128i channel0_1_128i;
1382 __m128i channel0_2_128i;
1383 SSE::deInterleave3Channel8Bit48Elements(buffer0A_128i, buffer0B_128i, buffer0C_128i, channel0_0_128i, channel0_1_128i, channel0_2_128i);
1384
1385 const __m128i buffer1A_128i = _mm_slli_si128(_mm_lddqu_si128((const __m128i*)(patch1)), overlappingElements);
1386 const __m128i buffer1B_128i = _mm_lddqu_si128((const __m128i*)(patch1 - overlappingElements + 16));
1387 const __m128i buffer1C_128i = _mm_lddqu_si128((const __m128i*)(patch1 - overlappingElements + 32));
1388
1389 __m128i channel1_0_128i;
1390 __m128i channel1_1_128i;
1391 __m128i channel1_2_128i;
1392 SSE::deInterleave3Channel8Bit48Elements(buffer1A_128i, buffer1B_128i, buffer1C_128i, channel1_0_128i, channel1_1_128i, channel1_2_128i);
1393
1394 __m128i absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i)), overlappingPixels * 2); // (channel0_0 - mean0) - (channel1_0 - mean1)
1395 __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_0_128i, channel0_0_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_0_128i, channel1_0_128i), constant_signs_m128i));
1396
1397 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1398 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1399
1400 absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i)), overlappingPixels * 2);
1401 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_1_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_1_128i), constant_signs_m128i));
1402
1403 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1404 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1405
1406 absDifferencesLow_128i = _mm_srli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i)), overlappingPixels * 2);
1407 absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
1408
1409 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i));
1410 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1411
1412 patch0 += remainingAfterFullBlocks48;
1413 patch1 += remainingAfterFullBlocks48;
1414 }
1415
1416 for (unsigned int n = 0u; n < blocks24; ++n)
1417 {
1418 const __m128i buffer0A_128i = _mm_lddqu_si128((const __m128i*)(patch0 + 0));
1419 const __m128i buffer0B_128i = _mm_loadl_epi64((const __m128i*)(patch0 + 16)); // load for unaligned 64 bit memory
1420
1421 __m128i channel0_01_128i;
1422 __m128i channel0_2_128i;
1423 SSE::deInterleave3Channel8Bit24Elements(buffer0A_128i, buffer0B_128i, channel0_01_128i, channel0_2_128i);
1424
1425 const __m128i buffer1A_128i = _mm_lddqu_si128((const __m128i*)(patch1 + 0));
1426 const __m128i buffer1B_128i = _mm_loadl_epi64((const __m128i*)(patch1 + 16)); // load for unaligned 64 bit memory
1427
1428 __m128i channel1_01_128i;
1429 __m128i channel1_2_128i;
1430 SSE::deInterleave3Channel8Bit24Elements(buffer1A_128i, buffer1B_128i, channel1_01_128i, channel1_2_128i);
1431
1432 __m128i absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_01_128i), constant_signs_m128i)); // (channel0_01 - mean0) - (channel1_01 - mean1)
1433 __m128i absDifferencesHigh_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_01_128i), constant_signs_m128i));
1434
1435 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1436 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1437
1438 absDifferencesLow_128i = _mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i));
1439
1440 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1441
1442 patch0 += 24;
1443 patch1 += 24;
1444 }
1445
1446 for (unsigned int n = 0u; n < blocks21; ++n)
1447 {
1448 const __m128i buffer0A_128i = _mm_lddqu_si128((const __m128i*)(patch0 + 0));
1449 const __m128i buffer0B_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(patch0 + 16 - 3)), 3); // load for unaligned 64 bit memory
1450
1451 __m128i channel0_01_128i;
1452 __m128i channel0_2_128i;
1453 SSE::deInterleave3Channel8Bit24Elements(buffer0A_128i, buffer0B_128i, channel0_01_128i, channel0_2_128i);
1454
1455 const __m128i buffer1A_128i = _mm_lddqu_si128((const __m128i*)(patch1 + 0));
1456 const __m128i buffer1B_128i = _mm_srli_si128(_mm_loadl_epi64((const __m128i*)(patch1 + 16 - 3)), 3); // load for unaligned 64 bit memory
1457
1458 __m128i channel1_01_128i;
1459 __m128i channel1_2_128i;
1460 SSE::deInterleave3Channel8Bit24Elements(buffer1A_128i, buffer1B_128i, channel1_01_128i, channel1_2_128i);
1461
1462 __m128i absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_01_128i), constant_signs_m128i)), 2); // (channel0_01 - mean0) - (channel1_01 - mean1)
1463 __m128i absDifferencesHigh_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_01_128i), constant_signs_m128i)), 2);
1464
1465 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1466 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1467
1468 absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i)), 2);
1469
1470 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1471
1472 patch0 += 21;
1473 patch1 += 21;
1474 }
1475
1476 for (unsigned int n = 0u; n < blocks15; ++n)
1477 {
1478 const __m128i buffer0_128i = y < tPatchSize - 1u ? _mm_lddqu_si128((const __m128i*)(patch0)) : _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(patch0 - 1)), 1);
1479
1480 __m128i channel0_01_128i;
1481 __m128i channel0_2_128i;
1482 SSE::deInterleave3Channel8Bit15Elements(buffer0_128i, channel0_01_128i, channel0_2_128i);
1483
1484 const __m128i buffer1_128i = y < tPatchSize - 1u ? _mm_lddqu_si128((const __m128i*)(patch1)) : _mm_srli_si128(_mm_lddqu_si128((const __m128i*)(patch1 - 1)), 1);
1485
1486 __m128i channel1_01_128i;
1487 __m128i channel1_2_128i;
1488 SSE::deInterleave3Channel8Bit15Elements(buffer1_128i, channel1_01_128i, channel1_2_128i);
1489
1490 __m128i absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_0_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_0_128i, channel1_01_128i), constant_signs_m128i)), 6); // (channel0_01 - mean0) - (channel1_01 - mean1)
1491 __m128i absDifferencesHigh_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpackhi_epi8(mean0_1_128i, channel0_01_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpackhi_epi8(mean1_1_128i, channel1_01_128i), constant_signs_m128i)), 6);
1492
1493 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1494 sum1_128i = _mm_add_epi32(sum1_128i, _mm_madd_epi16(absDifferencesHigh_128i, absDifferencesHigh_128i));
1495
1496 absDifferencesLow_128i = _mm_slli_si128(_mm_sub_epi16(_mm_maddubs_epi16(_mm_unpacklo_epi8(mean0_2_128i, channel0_2_128i), constant_signs_m128i), _mm_maddubs_epi16(_mm_unpacklo_epi8(mean1_2_128i, channel1_2_128i), constant_signs_m128i)), 6);
1497
1498 sum0_128i = _mm_add_epi32(sum0_128i, _mm_madd_epi16(absDifferencesLow_128i, absDifferencesLow_128i)); // sum0_128i += absDifferencesLow_128i * absDifferencesLow_128i
1499
1500 patch0 += 15;
1501 patch1 += 15;
1502 }
1503
1504 if constexpr (blocks1 != 0u)
1505 {
1506 constexpr unsigned int pixels = blocks1 / 3u;
1507
1508 for (unsigned int x = 0u; x < pixels; ++x)
1509 {
1510 for (unsigned int n = 0u; n < 3u; ++n)
1511 {
1512 sumIndividual += sqrDistance(patch0[x * 3u + n] - meanValues0[n], patch1[x * 3u + n] - meanValues1[n]);
1513 }
1514 }
1515
1516 patch0 += blocks1;
1517 patch1 += blocks1;
1518 }
1519
1520 patch0 += patch0StrideElements - patchWidthElements;
1521 patch1 += patch1StrideElements - patchWidthElements;
1522 }
1523
1524 return SSE::sum_u32_4(sum0_128i) + SSE::sum_u32_4(sum1_128i) + sumIndividual;
1525}
1526
1527template <unsigned int tChannels>
1528template <unsigned int tPatchSize>
1529inline uint32_t ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels<tChannels>::patch8BitPerChannel(const uint8_t* patch0, const uint8_t* patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements, const uint8_t* const meanValues0, const uint8_t* const meanValues1)
1530{
1531 static_assert(tChannels >= 1u, "Invalid channel number!");
1532 static_assert(tPatchSize >= 1u, "Invalid patch size!");
1533
1534 ocean_assert(patch0 != nullptr && patch1 != nullptr);
1535 ocean_assert(meanValues0 != nullptr && meanValues1 != nullptr);
1536
1537 ocean_assert(patch0StrideElements >= tChannels * tPatchSize);
1538 ocean_assert(patch1StrideElements >= tChannels * tPatchSize);
1539
1540 uint32_t ssd = 0u;
1541
1542 for (unsigned int y = 0u; y < tPatchSize; ++y)
1543 {
1544 for (unsigned int x = 0u; x < tPatchSize; ++x)
1545 {
1546 for (unsigned int n = 0u; n < tChannels; ++n)
1547 {
1548 ssd += sqrDistance(patch0[x * tChannels + n] - meanValues0[n], patch1[x * tChannels + n] - meanValues1[n]);
1549 }
1550 }
1551
1552 patch0 += patch0StrideElements;
1553 patch1 += patch1StrideElements;
1554 }
1555
1556 return ssd;
1557}
1558
1559template <unsigned int tChannels, unsigned int tPixels>
1560inline uint32_t ZeroMeanSumSquareDifferencesSSE::buffer8BitPerChannel(const uint8_t* const buffer0, const uint8_t* const buffer1)
1561{
1562 static_assert(tChannels >= 1u, "Invalid channel number!");
1563 static_assert(tPixels >= 8u, "Invalid patch size!");
1564
1565 ocean_assert(buffer0 != nullptr && buffer1 != nullptr);
1566
1567 uint8_t meanValues0[tChannels];
1568 mean8BitPerChannel<tChannels, tPixels>(buffer0, meanValues0);
1569
1570 uint8_t meanValues1[tChannels];
1571 mean8BitPerChannel<tChannels, tPixels>(buffer1, meanValues1);
1572
1573 return SpecializedForChannels<tChannels>::template buffer8BitPerChannel<tPixels>(buffer0, buffer1, meanValues0, meanValues1);
1574}
1575
1576template <unsigned int tChannels, unsigned int tPatchSize>
1577inline uint32_t ZeroMeanSumSquareDifferencesSSE::patch8BitPerChannel(const uint8_t* const patch0, const uint8_t* const patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements)
1578{
1579 static_assert(tChannels >= 1u, "Invalid channel number!");
1580 static_assert(tPatchSize >= 5u, "Invalid patch size!");
1581
1582 ocean_assert(patch0 != nullptr && patch1 != nullptr);
1583
1584 ocean_assert(patch0StrideElements >= tChannels * tPatchSize);
1585 ocean_assert(patch1StrideElements >= tChannels * tPatchSize);
1586
1587 uint8_t meanValues0[tChannels];
1588 mean8BitPerChannel<tChannels, tPatchSize>(patch0, patch0StrideElements, meanValues0);
1589
1590 uint8_t meanValues1[tChannels];
1591 mean8BitPerChannel<tChannels, tPatchSize>(patch1, patch1StrideElements, meanValues1);
1592
1593 return SpecializedForChannels<tChannels>::template patch8BitPerChannel<tPatchSize>(patch0, patch1, patch0StrideElements, patch1StrideElements, meanValues0, meanValues1);
1594}
1595
1596template <unsigned int tChannels, unsigned int tPatchSize>
1597inline uint32_t ZeroMeanSumSquareDifferencesSSE::patchBuffer8BitPerChannel(const uint8_t* const patch0, const uint8_t* const buffer1, const unsigned int patch0StrideElements)
1598{
1599 static_assert(tChannels >= 1u, "Invalid channel number!");
1600 static_assert(tPatchSize >= 5u, "Invalid patch size!");
1601
1602 ocean_assert(patch0 != nullptr && buffer1 != nullptr);
1603
1604 ocean_assert(patch0StrideElements >= tChannels * tPatchSize);
1605
1606 uint8_t meanValues0[tChannels];
1607 mean8BitPerChannel<tChannels, tPatchSize>(patch0, patch0StrideElements, meanValues0);
1608
1609 uint8_t meanValues1[tChannels];
1610 mean8BitPerChannel<tChannels, tPatchSize * tPatchSize>(buffer1, meanValues1);
1611
1612 constexpr unsigned int patch1StrideElements = tChannels * tPatchSize;
1613
1614 return SpecializedForChannels<tChannels>::template patch8BitPerChannel<tPatchSize>(patch0, buffer1, patch0StrideElements, patch1StrideElements, meanValues0, meanValues1);
1615}
1616
1617template <unsigned int tChannels, unsigned int tPixels>
1618OCEAN_FORCE_INLINE void ZeroMeanSumSquareDifferencesSSE::mean8BitPerChannel(const uint8_t* const buffer, uint8_t* const meanValues)
1619{
1620 static_assert(tChannels >= 1u, "Invalid channel number!");
1621 static_assert(tPixels >= 8u, "Invalid patch size!");
1622
1623 SpecializedForChannels<tChannels>::template mean8BitPerChannel<tPixels>(buffer, meanValues);
1624}
1625
1626template <unsigned int tChannels, unsigned int tPatchSize>
1627OCEAN_FORCE_INLINE void ZeroMeanSumSquareDifferencesSSE::mean8BitPerChannel(const uint8_t* const patch, const unsigned int patchStrideElements, uint8_t* const meanValues)
1628{
1629 static_assert(tChannels >= 1u, "Invalid channel number!");
1630 static_assert(tPatchSize >= 5u, "Invalid patch size!");
1631
1632 SpecializedForChannels<tChannels>::template mean8BitPerChannel<tPatchSize>(patch, patchStrideElements, meanValues);
1633}
1634
1635}
1636
1637}
1638
1639#endif // OCEAN_HARDWARE_SSE_VERSION >= 41
1640
1641#endif // META_OCEAN_CV_ZERO_MEAN_SUM_SQUARE_DIFFERENCES_SSE_H
Ocean::CV::SSE::deInterleave3Channel8Bit15Elements
static OCEAN_FORCE_INLINE void deInterleave3Channel8Bit15Elements(const __m128i &interleaved, __m128i &channel01, __m128i &channel2)
Deinterleaves 15 elements of e.g., an image with 3 channels and 8 bit per element.
Definition SSE.h:3341
Ocean::CV::SSE::sum_u32_first_third
static unsigned int sum_u32_first_third(const __m128i &value)
Adds the first and the second 32 bit unsigned integer values of a m128i value and returns the result.
Definition SSE.h:1377
Ocean::CV::SSE::deInterleave3Channel8Bit24Elements
static OCEAN_FORCE_INLINE void deInterleave3Channel8Bit24Elements(const __m128i &interleavedA, const __m128i &interleavedB, __m128i &channel01, __m128i &channel2)
Deinterleaves 24 elements of e.g., an image with 3 channels and 8 bit per element.
Definition SSE.h:3353
Ocean::CV::SSE::prefetchT0
static void prefetchT0(const void *const data)
Prefetches a block of temporal memory into all cache levels.
Definition SSE.h:1292
Ocean::CV::SSE::deInterleave3Channel8Bit48Elements
static OCEAN_FORCE_INLINE void deInterleave3Channel8Bit48Elements(const __m128i &interleavedA, const __m128i &interleavedB, const __m128i &interleavedC, __m128i &channel0, __m128i &channel1, __m128i &channel2)
Deinterleaves 48 elements of e.g., an image with 3 channels and 8 bit per element.
Definition SSE.h:3368
Ocean::CV::SSE::sum_u32_4
static OCEAN_FORCE_INLINE unsigned int sum_u32_4(const __m128i &value)
Adds the four (all four) individual 32 bit unsigned integer values of a m128i value and returns the r...
Definition SSE.h:1359
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels
This class allows to specialize functions for individual channels.
Definition ZeroMeanSumSquareDifferencesSSE.h:39
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels::patch8BitPerChannel
static uint32_t patch8BitPerChannel(const uint8_t *patch0, const uint8_t *patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements, const uint8_t *const meanValues0, const uint8_t *const meanValues1)
Returns the zero-mean sum of square differences between two patches within an image.
Definition ZeroMeanSumSquareDifferencesSSE.h:1529
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels::buffer8BitPerChannel
static uint32_t buffer8BitPerChannel(const uint8_t *const buffer0, const uint8_t *const buffer1, const uint8_t *const meanValues0, const uint8_t *const meanValues1)
Returns the zero-mean sum of square differences between two memory buffers.
Definition ZeroMeanSumSquareDifferencesSSE.h:1091
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::SpecializedForChannels::mean8BitPerChannel
static void mean8BitPerChannel(const uint8_t *const buffer, uint8_t *const meanValues)
Determines the mean value for a buffer, one value for each channel.
Definition ZeroMeanSumSquareDifferencesSSE.h:392
Ocean::CV::ZeroMeanSumSquareDifferencesSSE
This class implements functions to calculate zero-mean sum square differences using SSE instructions.
Definition ZeroMeanSumSquareDifferencesSSE.h:30
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::patchBuffer8BitPerChannel
static uint32_t patchBuffer8BitPerChannel(const uint8_t *const patch0, const uint8_t *const buffer1, const unsigned int patch0StrideElements)
Returns the zero-mean sum of square differences between an image patch and a buffer.
Definition ZeroMeanSumSquareDifferencesSSE.h:1597
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::buffer8BitPerChannel
static uint32_t buffer8BitPerChannel(const uint8_t *const buffer0, const uint8_t *const buffer1)
Returns the zero-mean sum of square differences between two memory buffers.
Definition ZeroMeanSumSquareDifferencesSSE.h:1560
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::mean8BitPerChannel
static OCEAN_FORCE_INLINE void mean8BitPerChannel(const uint8_t *const buffer, uint8_t *const meanValues)
Determines the mean value for a buffer, one value for each channel.
Definition ZeroMeanSumSquareDifferencesSSE.h:1618
Ocean::CV::ZeroMeanSumSquareDifferencesSSE::patch8BitPerChannel
static uint32_t patch8BitPerChannel(const uint8_t *const patch0, const uint8_t *const patch1, const unsigned int patch0StrideElements, const unsigned int patch1StrideElements)
Returns the zero-mean sum of square differences between two patches within an image.
Definition ZeroMeanSumSquareDifferencesSSE.h:1577
Ocean::sqrDistance
unsigned int sqrDistance(const char first, const char second)
Returns the square distance between two values.
Definition base/Utilities.h:1159
Ocean
The namespace covering the entire Ocean framework.
Definition Accessor.h:15