Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

X32-packw AVX GIO remove maskload for remainder handling #7428

Merged
merged 1 commit into from
Nov 6, 2024
+63 −162
Merged

X32-packw AVX GIO remove maskload for remainder handling #7428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 9 additions & 21 deletions src/x32-packw/gen/x32-packw-x16-gemm-gio-avx-prfm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.


#include <assert.h>
#include <stddef.h>
#include <stdint.h>
Expand Down Expand Up @@ -45,13 +44,6 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx_prfm(
assert(packed_weights != NULL);

const __m256 vzero = _mm256_setzero_ps();
static const int32_t mask_table[32] = {
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
};

const float* b = (const float*) bias;
float* packed_w = (float*) packed_weights;
do {
Expand All @@ -72,9 +64,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx_prfm(
}
packed_w += 16;

// KC main loop
// todo: KBLOCK rows at a time
for (size_t k = kc; k > 0; --k) {
size_t k = kc;
// KC remainder loop
for (; k > 0; --k) {
const __m256 v0 = _mm256_loadu_ps(w + 0);
const __m256 v8 = _mm256_loadu_ps(w + 8);
xnn_prefetch_to_l1((const int8_t*) w + 960);
Expand All @@ -90,14 +82,11 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx_prfm(
if XNN_UNLIKELY(n != 0) {
assert(n >= 1);
assert(n <= 15);
const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[16 - n]);
const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[16 - n]);

if XNN_LIKELY(b != NULL) {
const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
_mm256_store_ps(packed_w + 0, vb0);
_mm256_store_ps(packed_w + 8, vb8);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = b[i];
}
b += n;
} else {
_mm256_store_ps(packed_w + 0, vzero);
Expand All @@ -107,10 +96,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx_prfm(

// KC main loop
for (size_t k = kc; k > 0; --k) {
const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
_mm256_store_ps(packed_w + 0, v0);
_mm256_store_ps(packed_w + 8, v8);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = w[i];
}
w += k_stride;
packed_w += 16;
}
Expand Down
30 changes: 9 additions & 21 deletions src/x32-packw/gen/x32-packw-x16-gemm-gio-avx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.


#include <assert.h>
#include <stddef.h>
#include <stdint.h>
Expand Down Expand Up @@ -44,13 +43,6 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx(
assert(packed_weights != NULL);

const __m256 vzero = _mm256_setzero_ps();
static const int32_t mask_table[32] = {
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
};

const float* b = (const float*) bias;
float* packed_w = (float*) packed_weights;
do {
Expand All @@ -71,9 +63,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx(
}
packed_w += 16;

// KC main loop
// todo: KBLOCK rows at a time
for (size_t k = kc; k > 0; --k) {
size_t k = kc;
// KC remainder loop
for (; k > 0; --k) {
const __m256 v0 = _mm256_loadu_ps(w + 0);
const __m256 v8 = _mm256_loadu_ps(w + 8);
_mm256_store_ps(packed_w + 0, v0);
Expand All @@ -88,14 +80,11 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx(
if XNN_UNLIKELY(n != 0) {
assert(n >= 1);
assert(n <= 15);
const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[16 - n]);
const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[16 - n]);

if XNN_LIKELY(b != NULL) {
const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
_mm256_store_ps(packed_w + 0, vb0);
_mm256_store_ps(packed_w + 8, vb8);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = b[i];
}
b += n;
} else {
_mm256_store_ps(packed_w + 0, vzero);
Expand All @@ -105,10 +94,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x16__avx(

// KC main loop
for (size_t k = kc; k > 0; --k) {
const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
_mm256_store_ps(packed_w + 0, v0);
_mm256_store_ps(packed_w + 8, v8);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = w[i];
}
w += k_stride;
packed_w += 16;
}
Expand Down
44 changes: 9 additions & 35 deletions src/x32-packw/gen/x32-packw-x32-gemm-gio-avx-prfm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.


#include <assert.h>
#include <stddef.h>
#include <stdint.h>
Expand Down Expand Up @@ -45,17 +44,6 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx_prfm(
assert(packed_weights != NULL);

const __m256 vzero = _mm256_setzero_ps();
static const int32_t mask_table[64] = {
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
};

const float* b = (const float*) bias;
float* packed_w = (float*) packed_weights;
do {
Expand All @@ -82,9 +70,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx_prfm(
}
packed_w += 32;

// KC main loop
// todo: KBLOCK rows at a time
for (size_t k = kc; k > 0; --k) {
size_t k = kc;
// KC remainder loop
for (; k > 0; --k) {
const __m256 v0 = _mm256_loadu_ps(w + 0);
const __m256 v8 = _mm256_loadu_ps(w + 8);
const __m256 v16 = _mm256_loadu_ps(w + 16);
Expand All @@ -105,20 +93,11 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx_prfm(
if XNN_UNLIKELY(n != 0) {
assert(n >= 1);
assert(n <= 31);
const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
const __m256i vmask16 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
const __m256i vmask24 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);

if XNN_LIKELY(b != NULL) {
const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
const __m256 vb16 = _mm256_maskload_ps(b + 16, vmask16);
const __m256 vb24 = _mm256_maskload_ps(b + 24, vmask24);
_mm256_store_ps(packed_w + 0, vb0);
_mm256_store_ps(packed_w + 8, vb8);
_mm256_store_ps(packed_w + 16, vb16);
_mm256_store_ps(packed_w + 24, vb24);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = b[i];
}
b += n;
} else {
_mm256_store_ps(packed_w + 0, vzero);
Expand All @@ -130,14 +109,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx_prfm(

// KC main loop
for (size_t k = kc; k > 0; --k) {
const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
const __m256 v16 = _mm256_maskload_ps(w + 16, vmask16);
const __m256 v24 = _mm256_maskload_ps(w + 24, vmask24);
_mm256_store_ps(packed_w + 0, v0);
_mm256_store_ps(packed_w + 8, v8);
_mm256_store_ps(packed_w + 16, v16);
_mm256_store_ps(packed_w + 24, v24);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = w[i];
}
w += k_stride;
packed_w += 32;
}
Expand Down
44 changes: 9 additions & 35 deletions src/x32-packw/gen/x32-packw-x32-gemm-gio-avx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.


#include <assert.h>
#include <stddef.h>
#include <stdint.h>
Expand Down Expand Up @@ -44,17 +43,6 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx(
assert(packed_weights != NULL);

const __m256 vzero = _mm256_setzero_ps();
static const int32_t mask_table[64] = {
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
};

const float* b = (const float*) bias;
float* packed_w = (float*) packed_weights;
do {
Expand All @@ -81,9 +69,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx(
}
packed_w += 32;

// KC main loop
// todo: KBLOCK rows at a time
for (size_t k = kc; k > 0; --k) {
size_t k = kc;
// KC remainder loop
for (; k > 0; --k) {
const __m256 v0 = _mm256_loadu_ps(w + 0);
const __m256 v8 = _mm256_loadu_ps(w + 8);
const __m256 v16 = _mm256_loadu_ps(w + 16);
Expand All @@ -102,20 +90,11 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx(
if XNN_UNLIKELY(n != 0) {
assert(n >= 1);
assert(n <= 31);
const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
const __m256i vmask16 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
const __m256i vmask24 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);

if XNN_LIKELY(b != NULL) {
const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
const __m256 vb16 = _mm256_maskload_ps(b + 16, vmask16);
const __m256 vb24 = _mm256_maskload_ps(b + 24, vmask24);
_mm256_store_ps(packed_w + 0, vb0);
_mm256_store_ps(packed_w + 8, vb8);
_mm256_store_ps(packed_w + 16, vb16);
_mm256_store_ps(packed_w + 24, vb24);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = b[i];
}
b += n;
} else {
_mm256_store_ps(packed_w + 0, vzero);
Expand All @@ -127,14 +106,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x32__avx(

// KC main loop
for (size_t k = kc; k > 0; --k) {
const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
const __m256 v16 = _mm256_maskload_ps(w + 16, vmask16);
const __m256 v24 = _mm256_maskload_ps(w + 24, vmask24);
_mm256_store_ps(packed_w + 0, v0);
_mm256_store_ps(packed_w + 8, v8);
_mm256_store_ps(packed_w + 16, v16);
_mm256_store_ps(packed_w + 24, v24);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = w[i];
}
w += k_stride;
packed_w += 32;
}
Expand Down
23 changes: 9 additions & 14 deletions src/x32-packw/gen/x32-packw-x8-gemm-gio-avx-prfm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.


#include <assert.h>
#include <stddef.h>
#include <stdint.h>
Expand Down Expand Up @@ -45,11 +44,6 @@ void xnn_x32_packw_gemm_gio_ukernel_x8__avx_prfm(
assert(packed_weights != NULL);

const __m256 vzero = _mm256_setzero_ps();
static const int32_t mask_table[16] = {
-1, -1, -1, -1, -1, -1, -1, -1,
0, 0, 0, 0, 0, 0, 0, 0,
};

const float* b = (const float*) bias;
float* packed_w = (float*) packed_weights;
do {
Expand All @@ -67,9 +61,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x8__avx_prfm(
}
packed_w += 8;

// KC main loop
// todo: KBLOCK rows at a time
for (size_t k = kc; k > 0; --k) {
size_t k = kc;
// KC remainder loop
for (; k > 0; --k) {
const __m256 v0 = _mm256_loadu_ps(w + 0);
xnn_prefetch_to_l1((const int8_t*) w + 960);
_mm256_store_ps(packed_w + 0, v0);
Expand All @@ -83,11 +77,11 @@ void xnn_x32_packw_gemm_gio_ukernel_x8__avx_prfm(
if XNN_UNLIKELY(n != 0) {
assert(n >= 1);
assert(n <= 7);
const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[8 - n]);

if XNN_LIKELY(b != NULL) {
const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
_mm256_store_ps(packed_w + 0, vb0);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = b[i];
}
b += n;
} else {
_mm256_store_ps(packed_w + 0, vzero);
Expand All @@ -96,8 +90,9 @@ void xnn_x32_packw_gemm_gio_ukernel_x8__avx_prfm(

// KC main loop
for (size_t k = kc; k > 0; --k) {
const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
_mm256_store_ps(packed_w + 0, v0);
for (size_t i = 0; i < n; ++i) {
packed_w[i] = w[i];
}
w += k_stride;
packed_w += 8;
}
Expand Down
Loading