|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#include <assert.h> |
|
|
|
#include <immintrin.h> |
|
|
|
#include <xnnpack/gemm.h> |
|
#include <xnnpack/intrinsics-polyfill.h> |
|
|
|
|
|
void xnn_f32_gemm_minmax_ukernel_1x16__avx512f_broadcast( |
|
size_t mr, |
|
size_t nc, |
|
size_t kc, |
|
const float* restrict a, |
|
size_t a_stride, |
|
const float* restrict w, |
|
float* restrict c, |
|
size_t cm_stride, |
|
size_t cn_stride, |
|
const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) |
|
{ |
|
assert(mr != 0); |
|
assert(mr <= 1); |
|
assert(nc != 0); |
|
assert(kc != 0); |
|
assert(kc % sizeof(float) == 0); |
|
assert(a != NULL); |
|
assert(w != NULL); |
|
assert(c != NULL); |
|
|
|
const float* a0 = a; |
|
float* c0 = c; |
|
|
|
do { |
|
__m512 vacc0x0123456789ABCDEF = _mm512_load_ps(w); |
|
w += 16; |
|
|
|
size_t k = kc; |
|
do { |
|
const __m512 vb0123456789ABCDEF = _mm512_load_ps(w); |
|
w += 16; |
|
|
|
const __m512 va0 = _mm512_set1_ps(*a0); |
|
vacc0x0123456789ABCDEF = _mm512_fmadd_ps(va0, vb0123456789ABCDEF, vacc0x0123456789ABCDEF); |
|
|
|
a0 += 1; |
|
|
|
k -= sizeof(float); |
|
} while (k != 0); |
|
|
|
const __m512 vmin = _mm512_set1_ps(params->scalar.min); |
|
vacc0x0123456789ABCDEF = _mm512_max_ps(vmin, vacc0x0123456789ABCDEF); |
|
|
|
const __m512 vmax = _mm512_set1_ps(params->scalar.max); |
|
vacc0x0123456789ABCDEF = _mm512_min_ps(vmax, vacc0x0123456789ABCDEF); |
|
|
|
if XNN_LIKELY(nc >= 16) { |
|
_mm512_storeu_ps(c0, vacc0x0123456789ABCDEF); |
|
c0 = (float*) ((uintptr_t) c0 + cn_stride); |
|
|
|
a0 = (const float*) ((uintptr_t) a0 - kc); |
|
|
|
nc -= 16; |
|
} else { |
|
if (nc & 15) { |
|
|
|
const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << nc) - UINT32_C(1))); |
|
|
|
_mm512_mask_storeu_ps(c0, vmask, vacc0x0123456789ABCDEF); |
|
} |
|
|
|
nc = 0; |
|
} |
|
} while (nc != 0); |
|
} |
|
|