File size: 6,911 Bytes
8b7c501 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 |
// Copyright 2022 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
$assert BATCH_TILE % 8 == 0
$assert BATCH_TILE >= 8
$SIMD_TILE = BATCH_TILE // 8
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
#include <assert.h>
#include <arm_neon.h>
#include <xnnpack/common.h>
#include <xnnpack/vunary.h>
void xnn_f16_vsqrt_ukernel__neonfp16arith_nr1fma1adj_x${BATCH_TILE}(
size_t batch,
const void* input,
void* output,
const union xnn_f16_sqrt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
{
assert(batch != 0);
assert(batch % sizeof(uint16_t) == 0);
assert(input != NULL);
assert(output != NULL);
const uint16x8_t vpositive_infinity = vmovq_n_u16(UINT16_C(0x7C00));
const float16x8_t vhalf = vreinterpretq_f16_u16(vmovq_n_u16(UINT16_C(0x3800))); // 0.5h
const uint16x8_t vexp4_mask = vmovq_n_u16(UINT16_C(0x7800));
const uint16_t* i = (const uint16_t*) input;
uint16_t* o = (uint16_t*) output;
$if BATCH_TILE > 8:
for (; batch >= ${BATCH_TILE} * sizeof(uint16_t); batch -= ${BATCH_TILE} * sizeof(uint16_t)) {
$for N in range(SIMD_TILE):
float16x8_t vi${ABC[N]} = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
$for N in range(SIMD_TILE):
const float16x8_t vx${N} = vbslq_f16(vexp4_mask, vhalf, vi${N});
const int16x8_t vexp4i${N} = vreinterpretq_s16_u16(vandq_u16(vreinterpretq_u16_f16(vi${N}), vexp4_mask));
$for N in range(SIMD_TILE):
const float16x8_t vrsqrtx${N} = vrsqrteq_f16(vx${N});
const int16x8_t vpostscale${N} = vhsubq_s16(vexp4i${N}, vreinterpretq_s16_f16(vhalf));
$for N in range(SIMD_TILE):
float16x8_t vsqrtx${N} = vmulq_f16(vrsqrtx${N}, vx${N});
const float16x8_t vhalfrsqrtx${N} = vmulq_f16(vrsqrtx${N}, vhalf);
uint16x8_t vspecial_mask${N} = vcgeq_u16(vreinterpretq_u16_f16(vi${N}), vpositive_infinity);
$for N in range(SIMD_TILE):
const float16x8_t vresidual${N} = vfmsq_f16(vhalf, vsqrtx${N}, vhalfrsqrtx${N});
const uint16x8_t vzero_mask${N} = vceqq_f16(vi${N}, vreinterpretq_f16_u16(vmovq_n_u16(0)));
uint16x8_t vspecial_value${N} = vmovq_n_u16(UINT16_C(0x7E00));
$for N in range(SIMD_TILE):
vsqrtx${N} = vfmaq_f16(vsqrtx${N}, vresidual${N}, vsqrtx${N});
vspecial_mask${N} = vorrq_u16(vspecial_mask${N}, vzero_mask${N});
const uint16x8_t vinfinity_mask${N} = vceqq_u16(vreinterpretq_u16_f16(vi${N}), vpositive_infinity);
$for N in range(SIMD_TILE):
const float16x8_t vadjustment${N} = vfmsq_f16(vx${N}, vsqrtx${N}, vsqrtx${N});
const uint16x8_t vinput_mask${N} = vorrq_u16(vinfinity_mask${N}, vzero_mask${N});
$for N in range(SIMD_TILE):
vsqrtx${N} = vfmaq_f16(vsqrtx${N}, vhalfrsqrtx${N}, vadjustment${N});
vspecial_value${N} = vbslq_u16(vinput_mask${N}, vreinterpretq_u16_f16(vi${N}), vspecial_value${N});
$for N in range(SIMD_TILE):
float16x8_t vy${N} = vreinterpretq_f16_s16(vaddq_s16(vreinterpretq_s16_f16(vsqrtx${N}), vpostscale${N}));
$for N in range(SIMD_TILE):
vy${N} = vbslq_f16(vspecial_mask${N}, vreinterpretq_f16_u16(vspecial_value${N}), vy${N});
$for N in range(SIMD_TILE):
vst1q_u16(o, vreinterpretq_u16_f16(vy${ABC[N]})); o += 8;
}
for (; batch >= 8 * sizeof(uint16_t); batch -= 8 * sizeof(uint16_t)) {
const float16x8_t vi = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vx = vbslq_f16(vexp4_mask, vhalf, vi);
const int16x8_t vexp4i = vreinterpretq_s16_u16(vandq_u16(vreinterpretq_u16_f16(vi), vexp4_mask));
const float16x8_t vrsqrtx = vrsqrteq_f16(vx);
const int16x8_t vpostscale = vhsubq_s16(vexp4i, vreinterpretq_s16_f16(vhalf));
float16x8_t vsqrtx = vmulq_f16(vrsqrtx, vx);
const float16x8_t vhalfrsqrtx = vmulq_f16(vrsqrtx, vhalf);
uint16x8_t vspecial_mask = vcgeq_u16(vreinterpretq_u16_f16(vi), vpositive_infinity);
const float16x8_t vresidual = vfmsq_f16(vhalf, vsqrtx, vhalfrsqrtx);
const uint16x8_t vzero_mask = vceqq_f16(vi, vreinterpretq_f16_u16(vmovq_n_u16(0)));
uint16x8_t vspecial_value = vmovq_n_u16(UINT16_C(0x7E00));
vsqrtx = vfmaq_f16(vsqrtx, vresidual, vsqrtx);
vspecial_mask = vorrq_u16(vspecial_mask, vzero_mask);
const uint16x8_t vinfinity_mask = vceqq_u16(vreinterpretq_u16_f16(vi), vpositive_infinity);
const float16x8_t vadjustment = vfmsq_f16(vx, vsqrtx, vsqrtx);
const uint16x8_t vinput_mask = vorrq_u16(vinfinity_mask, vzero_mask);
vsqrtx = vfmaq_f16(vsqrtx, vhalfrsqrtx, vadjustment);
vspecial_value = vbslq_u16(vinput_mask, vreinterpretq_u16_f16(vi), vspecial_value);
float16x8_t vy = vreinterpretq_f16_s16(vaddq_s16(vreinterpretq_s16_f16(vsqrtx), vpostscale));
vy = vbslq_f16(vspecial_mask, vreinterpretq_f16_u16(vspecial_value), vy);
vst1q_u16(o, vreinterpretq_u16_f16(vy)); o += 8;
}
if XNN_UNLIKELY(batch != 0) {
const float16x8_t vi = vreinterpretq_f16_u16(vld1q_u16(i));
const float16x8_t vx = vbslq_f16(vexp4_mask, vhalf, vi);
const int16x8_t vexp4i = vreinterpretq_s16_u16(vandq_u16(vreinterpretq_u16_f16(vi), vexp4_mask));
const float16x8_t vrsqrtx = vrsqrteq_f16(vx);
const int16x8_t vpostscale = vhsubq_s16(vexp4i, vreinterpretq_s16_f16(vhalf));
float16x8_t vsqrtx = vmulq_f16(vrsqrtx, vx);
const float16x8_t vhalfrsqrtx = vmulq_f16(vrsqrtx, vhalf);
uint16x8_t vspecial_mask = vcgeq_u16(vreinterpretq_u16_f16(vi), vpositive_infinity);
const float16x8_t vresidual = vfmsq_f16(vhalf, vsqrtx, vhalfrsqrtx);
const uint16x8_t vzero_mask = vceqq_f16(vi, vreinterpretq_f16_u16(vmovq_n_u16(0)));
uint16x8_t vspecial_value = vmovq_n_u16(UINT16_C(0x7E00));
vsqrtx = vfmaq_f16(vsqrtx, vresidual, vsqrtx);
vspecial_mask = vorrq_u16(vspecial_mask, vzero_mask);
const uint16x8_t vinfinity_mask = vceqq_u16(vreinterpretq_u16_f16(vi), vpositive_infinity);
const float16x8_t vadjustment = vfmsq_f16(vx, vsqrtx, vsqrtx);
const uint16x8_t vinput_mask = vorrq_u16(vinfinity_mask, vzero_mask);
vsqrtx = vfmaq_f16(vsqrtx, vhalfrsqrtx, vadjustment);
vspecial_value = vbslq_u16(vinput_mask, vreinterpretq_u16_f16(vi), vspecial_value);
float16x8_t vy = vreinterpretq_f16_s16(vaddq_s16(vreinterpretq_s16_f16(vsqrtx), vpostscale));
vy = vbslq_f16(vspecial_mask, vreinterpretq_f16_u16(vspecial_value), vy);
float16x4_t vy_lo = vget_low_f16(vy);
if (batch & (4 * sizeof(uint16_t))) {
vst1_u16(o, vreinterpret_u16_f16(vy_lo)); o += 4;
vy_lo = vget_high_f16(vy);
}
if (batch & (2 * sizeof(uint16_t))) {
vst1_lane_u32((void*) o, vreinterpret_u32_f16(vy_lo), 0); o += 2;
vy_lo = vext_f16(vy_lo, vy_lo, 2);
}
if (batch & (1 * sizeof(uint16_t))) {
vst1_lane_u16(o, vreinterpret_u16_f16(vy_lo), 0);
}
}
}
|