24.02.1
|
Go to the source code of this file.
Macros | |||||||||||||||||||||||||||||||||||||
#define | SCALAR_ACCESS_STR(offset, n0, x) scalar_access_##offset##_##n0(x) | ||||||||||||||||||||||||||||||||||||
Utility macro to access a vector with the scalar positions. More... | |||||||||||||||||||||||||||||||||||||
#define | SCALAR_ACCESS(offset, n0, x) SCALAR_ACCESS_STR(offset, n0, x) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_0_1(x) ((x).s0) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_0_2(x) ((x).s01) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_0_3(x) ((x).s012) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_0_4(x) ((x).s0123) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_0_8(x) ((x).s01234567) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_0_16(x) ((x).s0123456789ABCDEF) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_1_1(x) ((x).s1) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_1_2(x) ((x).s12) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_1_3(x) ((x).s123) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_1_4(x) ((x).s1234) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_1_8(x) ((x).s12345678) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_2_1(x) ((x).s2) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_2_2(x) ((x).s23) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_2_3(x) ((x).s234) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_2_4(x) ((x).s2345) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_2_8(x) ((x).s23456789) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_3_1(x) ((x).s3) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_3_2(x) ((x).s34) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_3_3(x) ((x).s345) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_3_4(x) ((x).s3456) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_3_8(x) ((x).s3456789A) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_4_1(x) ((x).s4) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_4_2(x) ((x).s45) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_4_3(x) ((x).s456) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_4_4(x) ((x).s4567) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_4_8(x) ((x).s456789AB) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_8_1(x) ((x).s8) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_8_2(x) ((x).s89) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_8_3(x) ((x).s89A) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_8_4(x) ((x).s89AB) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_8_8(x) ((x).s89ABCDEF) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_12_1(x) ((x).sC) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_12_2(x) ((x).sCD) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_12_3(x) ((x).sCDE) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_12_4(x) ((x).sCDEF) | ||||||||||||||||||||||||||||||||||||
#define | scalar_access_16_1(x) ((x).sF) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR(K0, IDX_COL, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
Create column vectors to contain the values at the given index for a set of given vectors. More... | |||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR(K0, IDX_COL, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
Create column vectors to contain the values at the given index. More... | |||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0XN0(K0, N0, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
Create transposed vectors form the given source vectors. More... | |||||||||||||||||||||||||||||||||||||
LOAD_TENSOR_ROW_n | |||||||||||||||||||||||||||||||||||||
Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) without allocating variables.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) ({}) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##0) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0)); | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_ROW_16(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_TENSOR | |||||||||||||||||||||||||||||||||||||
Load tensor (consecutive rows and columns) with Z offset. Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16 The data to load is expected to have consecutive names for each row. E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) LOAD_TENSOR_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_TENSOR_M0Xn | |||||||||||||||||||||||||||||||||||||
Load 2D tensor (consecutive rows and columns) with Z offset.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X0(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) ({}) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X1(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X2(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X3(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X4(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X5(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X6(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X7(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X8(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X9(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X10(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X11(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X12(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X13(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X14(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X15(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0X16(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); | ||||||||||||||||||||||||||||||||||||
LOAD_TENSOR_M0XN0 | |||||||||||||||||||||||||||||||||||||
Load 2D tensor (consecutive rows and columns) with Z offset.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) LOAD_TENSOR_M0X##N0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TENSOR_M0XN0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_ROW_n | |||||||||||||||||||||||||||||||||||||
Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_BLOCK | |||||||||||||||||||||||||||||||||||||
Load Blocks (consecutive rows and columns) with Z offset. Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16 The data to load is expected to have consecutive names for each row. E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_ROW_PARTIAL_n | |||||||||||||||||||||||||||||||||||||
Partially load the 0 to (n-1)th rows of the given variables Within each row, load the lower
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_PARTIAL_16(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_BLOCK_PARTIAL | |||||||||||||||||||||||||||||||||||||
Partially load a block of the given size LOAD_M0xLOAD_N0
The data to load is expected to have consecutive names for each row. E.g., for LOAD_M0=3 and basename=c, the expected names are c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for LOAD_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_PARTIAL_##LOAD_M0(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_PARTIAL(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) | ||||||||||||||||||||||||||||||||||||
Load a block that can be partial in both x and y dimensions. More... | |||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) | ||||||||||||||||||||||||||||||||||||
Load a block that can only be partial in x but not y. More... | |||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) | ||||||||||||||||||||||||||||||||||||
Load a block that can only be partial in y but not x. More... | |||||||||||||||||||||||||||||||||||||
LOAD_BLOCK_BOUNDARY_AWARE | |||||||||||||||||||||||||||||||||||||
Boundary-aware GeMM block load This macro assumes the following schemes to achieve boundary-awareness:
In the y dimension, we place the partial blocks at the beginning while in the x dimension, we place the partial blocks at the end. Say, the src tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/ "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters: *–x--> x == 0 x == 1 | |<---------------------------—N-----------------------—>| y |<-----------—N0----------—>|<-—PARTIAL_STORE_N0--—>| | ----------—#############################################################
Then
It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension, and select corresponding load methods such that the boundary detection logic is only added when needed. The data to load is expected to have consecutive names for each row. E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. The macro will result in a declaration of
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) | ||||||||||||||||||||||||||||||||||||
LOAD_TEXTURE2D_ROW_n | |||||||||||||||||||||||||||||||||||||
Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) BASENAME##0 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 0 * X_STEP_ROW), (Y_COORD + 0 * Y_STEP_ROW)) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_ROW_16(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
LOAD_TEXTURE2D | |||||||||||||||||||||||||||||||||||||
Load a 2D texture in unit of pixel. A pixel is made of 4 floating point values Supported cases are M0=1,2,3,...,16 and N0=1 The data to load is expected to have consecutive names for each row. E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) LOAD_TEXTURE2D_ROW_##M0(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_TEXTURE2D(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) | ||||||||||||||||||||||||||||||||||||
LOAD_ROW_INDIRECT_n | |||||||||||||||||||||||||||||||||||||
Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) passing the Y index for each row to be loaded.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ROW_INDIRECT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
LOAD_BLOCK_INDIRECT | |||||||||||||||||||||||||||||||||||||
Load blocks (consecutive rows and columns) with Y offset. Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16 The data to load is expected to have consecutive names for each row. E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) LOAD_ROW_INDIRECT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_BLOCK_INDIRECT(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) | ||||||||||||||||||||||||||||||||||||
LOAD_ELEMENT_n | |||||||||||||||||||||||||||||||||||||
Loads the elements from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_ELEMENT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
LOAD_SCALAR_AS_VECTOR | |||||||||||||||||||||||||||||||||||||
Load Scalar as Vector (consecutive elements). Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16 The data to load is expected to have consecutive names for each row. E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
| |||||||||||||||||||||||||||||||||||||
#define | LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) LOAD_ELEMENT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | LOAD_SCALAR_AS_VECTOR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
CALCULATE_Z_OFFSET_n | |||||||||||||||||||||||||||||||||||||
Basic macros to calculate Z offset values from Z0 to Zn-1
| |||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_8(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
CALCULATE_Z_OFFSET | |||||||||||||||||||||||||||||||||||||
Calculate Z offset values from Z0 to Zn-1 The Z offsets are expected to have consecutive names. E.g., for M0=3 and Z=zin, the expected names of Z offsets are zin1, zin2, zin3. Note that, CROSS_PLANE_PAD (cross plain padding) is required to take into account the possible cross plane paddings in case of the plance changes across the z-dimension.
| |||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
#define | CALCULATE_Z_OFFSET(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) | ||||||||||||||||||||||||||||||||||||
SCALE_ROW_n | |||||||||||||||||||||||||||||||||||||
Scale the rows in the given variables (BASENAME0 to BASENAMEn-1)
| |||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) BASENAME##0 *= (DATA_TYPE)SCALE; | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_ROW_16(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
SCALE_BLOCK | |||||||||||||||||||||||||||||||||||||
Scale elements stored in a block (BASENAME) Supported cases are N=1,2,3,...,16
| |||||||||||||||||||||||||||||||||||||
#define | SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) SCALE_ROW_##N(DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
#define | SCALE_BLOCK(N, DATA_TYPE, BASENAME, SCALE) SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) | ||||||||||||||||||||||||||||||||||||
COLUMN_VECTORn | |||||||||||||||||||||||||||||||||||||
Create a new vector containing the values at the given index for a set of given vectors
| |||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR1(IDX_COL, BASENAME, X, TYPE) TYPE BASENAME##IDX_COL = (TYPE)((X##0).s##IDX_COL); | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR2(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR3(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR4(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR8(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR16(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
COLUMN_VECTOR_SCALARn | |||||||||||||||||||||||||||||||||||||
Create a new vector containing the values at the given index. Utility macros for transposing a colum-vector
| |||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR1(IDX_COL, BASENAME, X, TYPE) TYPE BASENAME##IDX_COL = (TYPE)((X##0)); | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR2(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR3(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR4(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR8(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | COLUMN_VECTOR_SCALAR16(IDX_COL, BASENAME, X, TYPE) | ||||||||||||||||||||||||||||||||||||
TRANSPOSE_K0Xn | |||||||||||||||||||||||||||||||||||||
Create transposed vectors of the given vectors
| |||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0X1(K0, BASENAME, BS, TYPE) COLUMN_VECTOR_SCALAR(K0, 0, BASENAME, BS, TYPE); | ||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0X2(K0, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0X3(K0, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0X4(K0, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0X8(K0, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
#define | TRANSPOSE_K0X16(K0, BASENAME, BS, TYPE) | ||||||||||||||||||||||||||||||||||||
ADD_ROW_n | |||||||||||||||||||||||||||||||||||||
Add the variables (BIAS0 to BIASn-1) to the others (BASENAME0 to BASENAMEn-1)
| |||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_1(BASENAME, BIAS) BASENAME##0 += BIAS##0; | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_2(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_3(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_4(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_5(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_6(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_7(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_8(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_9(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_10(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_11(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_12(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_13(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_14(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_15(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_16(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
ADD_BLOCK | |||||||||||||||||||||||||||||||||||||
Add the block (BIAS) to another block (BASENAME) Supported cases are N=1,2,3,...,16
| |||||||||||||||||||||||||||||||||||||
#define | ADD_BLOCK_STR(N, BASENAME, BIAS) ADD_ROW_##N(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_BLOCK(N, BASENAME, BIAS) ADD_BLOCK_STR(N, BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
ADD_ROW_BROADCAST_n | |||||||||||||||||||||||||||||||||||||
Broadcast (add single value) to the each element of the destination variables
| |||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_1(BASENAME, BIAS) BASENAME##0 += BIAS; | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_2(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_3(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_4(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_5(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_6(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_7(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_8(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_9(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_10(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_11(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_12(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_13(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_14(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_15(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_ROW_BROADCAST_16(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
ADD_BLOCK_BROADCAST | |||||||||||||||||||||||||||||||||||||
Broadcast (add a value) to the each element of the destination block (BASENAME) Supported cases are N=1,2,3,...,16.
| |||||||||||||||||||||||||||||||||||||
#define | ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) ADD_ROW_BROADCAST_##N(BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
#define | ADD_BLOCK_BROADCAST(N, BASENAME, BIAS) ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) | ||||||||||||||||||||||||||||||||||||
ACTIVATION_ROW_n | |||||||||||||||||||||||||||||||||||||
Apply activation to the given variables
| |||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) BASENAME##0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##0, A_VAL, B_VAL); | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_ROW_16(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
ACTIVATION_BLOCK | |||||||||||||||||||||||||||||||||||||
Apply activation to a block (BASENAME) Supported cases are N=1,2,3,...,16.
| |||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) ACTIVATION_ROW_##N(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
#define | ACTIVATION_BLOCK(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) | ||||||||||||||||||||||||||||||||||||
CONVERT_ROW_n | |||||||||||||||||||||||||||||||||||||
Apply convert_<data_type> to the given variables
| |||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_ROW_16(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
CONVERT_BLOCK | |||||||||||||||||||||||||||||||||||||
Apply convert_<data_type> to a block (BASENAME_SRC) and save to another block (BASENAME_DST) Supported cases N=1,2,3,...,16.
| |||||||||||||||||||||||||||||||||||||
#define | CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define | CONVERT_BLOCK(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) | ||||||||||||||||||||||||||||||||||||
#define ACTIVATION_BLOCK | ( | N, | |
ACTIVATION_TYPE, | |||
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) | ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) |
Definition at line 1697 of file gemm_helpers.h.
#define ACTIVATION_BLOCK_STR | ( | N, | |
ACTIVATION_TYPE, | |||
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) | ACTIVATION_ROW_##N(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) |
Definition at line 1695 of file gemm_helpers.h.
#define ACTIVATION_ROW_1 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) | BASENAME##0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##0, A_VAL, B_VAL); |
Definition at line 1618 of file gemm_helpers.h.
#define ACTIVATION_ROW_10 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1653 of file gemm_helpers.h.
#define ACTIVATION_ROW_11 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1657 of file gemm_helpers.h.
#define ACTIVATION_ROW_12 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1661 of file gemm_helpers.h.
#define ACTIVATION_ROW_13 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1665 of file gemm_helpers.h.
#define ACTIVATION_ROW_14 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1669 of file gemm_helpers.h.
#define ACTIVATION_ROW_15 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1673 of file gemm_helpers.h.
#define ACTIVATION_ROW_16 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1677 of file gemm_helpers.h.
#define ACTIVATION_ROW_2 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1621 of file gemm_helpers.h.
#define ACTIVATION_ROW_3 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1625 of file gemm_helpers.h.
#define ACTIVATION_ROW_4 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1629 of file gemm_helpers.h.
#define ACTIVATION_ROW_5 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1633 of file gemm_helpers.h.
#define ACTIVATION_ROW_6 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1637 of file gemm_helpers.h.
#define ACTIVATION_ROW_7 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1641 of file gemm_helpers.h.
#define ACTIVATION_ROW_8 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1645 of file gemm_helpers.h.
#define ACTIVATION_ROW_9 | ( | ACTIVATION_TYPE, | |
DATA_TYPE, | |||
VEC_SIZE, | |||
BASENAME, | |||
A_VAL, | |||
B_VAL | |||
) |
Definition at line 1649 of file gemm_helpers.h.
#define ADD_BLOCK | ( | N, | |
BASENAME, | |||
BIAS | |||
) | ADD_BLOCK_STR(N, BASENAME, BIAS) |
Definition at line 1521 of file gemm_helpers.h.
#define ADD_BLOCK_BROADCAST | ( | N, | |
BASENAME, | |||
BIAS | |||
) | ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) |
Definition at line 1605 of file gemm_helpers.h.
Definition at line 1604 of file gemm_helpers.h.
Definition at line 1520 of file gemm_helpers.h.
#define ADD_ROW_1 | ( | BASENAME, | |
BIAS | |||
) | BASENAME##0 += BIAS##0; |
Definition at line 1446 of file gemm_helpers.h.
#define ADD_ROW_10 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1480 of file gemm_helpers.h.
#define ADD_ROW_11 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1484 of file gemm_helpers.h.
#define ADD_ROW_12 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1488 of file gemm_helpers.h.
#define ADD_ROW_13 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1492 of file gemm_helpers.h.
#define ADD_ROW_14 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1496 of file gemm_helpers.h.
#define ADD_ROW_15 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1500 of file gemm_helpers.h.
#define ADD_ROW_16 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1504 of file gemm_helpers.h.
#define ADD_ROW_2 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1448 of file gemm_helpers.h.
#define ADD_ROW_3 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1452 of file gemm_helpers.h.
#define ADD_ROW_4 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1456 of file gemm_helpers.h.
#define ADD_ROW_5 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1460 of file gemm_helpers.h.
#define ADD_ROW_6 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1464 of file gemm_helpers.h.
#define ADD_ROW_7 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1468 of file gemm_helpers.h.
#define ADD_ROW_8 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1472 of file gemm_helpers.h.
#define ADD_ROW_9 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1476 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_1 | ( | BASENAME, | |
BIAS | |||
) | BASENAME##0 += BIAS; |
Definition at line 1531 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_10 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1565 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_11 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1569 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_12 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1573 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_13 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1577 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_14 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1581 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_15 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1585 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_16 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1589 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_2 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1533 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_3 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1537 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_4 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1541 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_5 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1545 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_6 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1549 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_7 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1553 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_8 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1557 of file gemm_helpers.h.
#define ADD_ROW_BROADCAST_9 | ( | BASENAME, | |
BIAS | |||
) |
Definition at line 1561 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) | CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) |
Definition at line 1212 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_1 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1129 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_2 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1134 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_3 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1140 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_4 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1146 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_5 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1152 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_6 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1158 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_7 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1164 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_8 | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) |
Definition at line 1170 of file gemm_helpers.h.
#define CALCULATE_Z_OFFSET_STR | ( | M0, | |
DATA_TYPE, | |||
Z, | |||
Y, | |||
HEIGHT_GEMM3D, | |||
DEPTH_GEMM3D, | |||
CROSS_PLANE_PAD, | |||
STRIDE_Y | |||
) | CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) |
Definition at line 1210 of file gemm_helpers.h.
#define COLUMN_VECTOR | ( | K0, | |
IDX_COL, | |||
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Create column vectors to contain the values at the given index for a set of given vectors.
[in] | K0 | The number of source vectors |
[in] | IDX_COL | The index value |
[in] | BASENAME | The basename of the destination vectors |
[in] | BS | The basename of the source vectors |
[in] | TYPE | The data type of the destination vectors |
Definition at line 1410 of file gemm_helpers.h.
#define COLUMN_VECTOR1 | ( | IDX_COL, | |
BASENAME, | |||
X, | |||
TYPE | |||
) | TYPE BASENAME##IDX_COL = (TYPE)((X##0).s##IDX_COL); |
Definition at line 1311 of file gemm_helpers.h.
Definition at line 1327 of file gemm_helpers.h.
Definition at line 1312 of file gemm_helpers.h.
Definition at line 1315 of file gemm_helpers.h.
Definition at line 1318 of file gemm_helpers.h.
Definition at line 1322 of file gemm_helpers.h.
#define COLUMN_VECTOR_SCALAR | ( | K0, | |
IDX_COL, | |||
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Create column vectors to contain the values at the given index.
Utility macro for transposing a column-vector
[in] | K0 | The number of source vectors |
[in] | IDX_COL | The index value |
[in] | BASENAME | The basename of the destination vectors |
[in] | BS | The basename of the source vectors |
[in] | TYPE | The data type of the destination vectors |
Definition at line 1422 of file gemm_helpers.h.
#define COLUMN_VECTOR_SCALAR1 | ( | IDX_COL, | |
BASENAME, | |||
X, | |||
TYPE | |||
) | TYPE BASENAME##IDX_COL = (TYPE)((X##0)); |
Definition at line 1345 of file gemm_helpers.h.
Definition at line 1346 of file gemm_helpers.h.
Definition at line 1349 of file gemm_helpers.h.
Definition at line 1352 of file gemm_helpers.h.
Definition at line 1355 of file gemm_helpers.h.
#define CONVERT_BLOCK | ( | M, | |
N, | |||
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) | CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) |
Definition at line 1804 of file gemm_helpers.h.
#define CONVERT_BLOCK_STR | ( | M, | |
N, | |||
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) | CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) |
Definition at line 1802 of file gemm_helpers.h.
#define CONVERT_ROW_1 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1710 of file gemm_helpers.h.
#define CONVERT_ROW_10 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1754 of file gemm_helpers.h.
#define CONVERT_ROW_11 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1759 of file gemm_helpers.h.
#define CONVERT_ROW_12 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1764 of file gemm_helpers.h.
#define CONVERT_ROW_13 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1769 of file gemm_helpers.h.
#define CONVERT_ROW_14 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1774 of file gemm_helpers.h.
#define CONVERT_ROW_15 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1779 of file gemm_helpers.h.
#define CONVERT_ROW_16 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1784 of file gemm_helpers.h.
#define CONVERT_ROW_2 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1714 of file gemm_helpers.h.
#define CONVERT_ROW_3 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1719 of file gemm_helpers.h.
#define CONVERT_ROW_4 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1724 of file gemm_helpers.h.
#define CONVERT_ROW_5 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1729 of file gemm_helpers.h.
#define CONVERT_ROW_6 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1734 of file gemm_helpers.h.
#define CONVERT_ROW_7 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1739 of file gemm_helpers.h.
#define CONVERT_ROW_8 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1744 of file gemm_helpers.h.
#define CONVERT_ROW_9 | ( | N, | |
DATA_TYPE, | |||
BASENAME_SRC, | |||
BASENAME_DST | |||
) |
Definition at line 1749 of file gemm_helpers.h.
#define LOAD_BLOCK | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) |
Definition at line 402 of file gemm_helpers.h.
#define LOAD_BLOCK_BOUNDARY_AWARE | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z, | |||
PARTIAL_STORE_M0, | |||
PARTIAL_STORE_N0, | |||
PARTIAL_COND_Y, | |||
PARTIAL_COND_X | |||
) | LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) |
Definition at line 688 of file gemm_helpers.h.
#define LOAD_BLOCK_INDIRECT | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) | LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) |
Definition at line 997 of file gemm_helpers.h.
#define LOAD_BLOCK_INDIRECT_STR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) | LOAD_ROW_INDIRECT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) |
Definition at line 995 of file gemm_helpers.h.
#define LOAD_BLOCK_PARTIAL | ( | LOAD_M0, | |
LOAD_N0, | |||
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) |
Definition at line 526 of file gemm_helpers.h.
#define LOAD_BLOCK_PARTIAL_IN_X | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z, | |||
PARTIAL_STORE_N0, | |||
PARTIAL_COND_X | |||
) |
Load a block that can only be partial in x but not y.
N0
or PARTIAL_STORE_N0
!= 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.The data to load is expected to have consecutive names for each row. E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
[in] | M0 | The number of rows to load, for non-partial blocks. Supported: 1-16 |
[in] | N0 | The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 |
[in] | DATA_TYPE | The data type of the vectors |
[in] | BASENAME | The basename of the variables |
[in] | PTR | The base pointer |
[in] | OFFSET | The offset within a row |
[in] | STRIDE_Y | The stride value in y-axis direction |
[in] | Z | The offset in z-axis direction |
[in] | PARTIAL_STORE_N0 | The partial size in x, for partial blocks. Supported range: [1, N0 ) |
[in] | PARTIAL_COND_X | Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. |
Definition at line 588 of file gemm_helpers.h.
#define LOAD_BLOCK_PARTIAL_IN_X_AND_Y | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z, | |||
PARTIAL_STORE_M0, | |||
PARTIAL_STORE_N0, | |||
PARTIAL_COND_Y, | |||
PARTIAL_COND_X | |||
) |
Load a block that can be partial in both x and y dimensions.
PARTIAL_STORE_N0
!= 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.The data to load is expected to have consecutive names for each row. E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
[in] | M0 | The number of rows to load, for non-partial blocks. Supported: 1-16 |
[in] | N0 | The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 |
[in] | DATA_TYPE | The data type of the vectors |
[in] | BASENAME | The basename of the variables |
[in] | PTR | The base pointer |
[in] | OFFSET | The offset within a row |
[in] | STRIDE_Y | The stride value in y-axis direction |
[in] | Z | The offset in z-axis direction |
[in] | PARTIAL_STORE_M0 | The partial size in y, for partial blocks. Supported range: [1, M0 ) |
[in] | PARTIAL_STORE_N0 | The partial size in x, for partial blocks. Supported range: [1, N0 ) |
[in] | PARTIAL_COND_Y | Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0. |
[in] | PARTIAL_COND_X | Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. |
Definition at line 550 of file gemm_helpers.h.
#define LOAD_BLOCK_PARTIAL_IN_Y | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z, | |||
PARTIAL_STORE_M0, | |||
PARTIAL_COND_Y | |||
) |
Load a block that can only be partial in y but not x.
N0
or PARTIAL_STORE_N0
!= 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.The data to store is expected to have consecutive names for each row. E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. The Z offset is expected to have consecutive names. E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
[in] | M0 | The number of rows to store, for non-partial blocks. Supported: 1-16 |
[in] | N0 | The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 |
[in] | DATA_TYPE | The data type of the vectors |
[in] | BASENAME | The basename of the variables |
[in] | PTR | The base pointer |
[in] | OFFSET | The offset within a row |
[in] | STRIDE_Y | The stride value in y-axis direction |
[in] | Z | The offset in z-axis direction |
[in] | PARTIAL_STORE_M0 | The partial size in y, for partial blocks. Supported range: [1, M0 ) |
[in] | PARTIAL_COND_Y | Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. |
Definition at line 618 of file gemm_helpers.h.
#define LOAD_BLOCK_PARTIAL_STR | ( | LOAD_M0, | |
LOAD_N0, | |||
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_ROW_PARTIAL_##LOAD_M0(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) |
Definition at line 524 of file gemm_helpers.h.
#define LOAD_BLOCK_STR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) |
Definition at line 400 of file gemm_helpers.h.
#define LOAD_ELEMENT_1 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1012 of file gemm_helpers.h.
#define LOAD_ELEMENT_10 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1056 of file gemm_helpers.h.
#define LOAD_ELEMENT_11 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1061 of file gemm_helpers.h.
#define LOAD_ELEMENT_12 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1066 of file gemm_helpers.h.
#define LOAD_ELEMENT_13 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1071 of file gemm_helpers.h.
#define LOAD_ELEMENT_14 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1076 of file gemm_helpers.h.
#define LOAD_ELEMENT_15 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1081 of file gemm_helpers.h.
#define LOAD_ELEMENT_16 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1086 of file gemm_helpers.h.
#define LOAD_ELEMENT_2 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1016 of file gemm_helpers.h.
#define LOAD_ELEMENT_3 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1021 of file gemm_helpers.h.
#define LOAD_ELEMENT_4 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1026 of file gemm_helpers.h.
#define LOAD_ELEMENT_5 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1031 of file gemm_helpers.h.
#define LOAD_ELEMENT_6 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1036 of file gemm_helpers.h.
#define LOAD_ELEMENT_7 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1041 of file gemm_helpers.h.
#define LOAD_ELEMENT_8 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1046 of file gemm_helpers.h.
#define LOAD_ELEMENT_9 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) |
Definition at line 1051 of file gemm_helpers.h.
#define LOAD_ROW_1 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 300 of file gemm_helpers.h.
#define LOAD_ROW_10 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 344 of file gemm_helpers.h.
#define LOAD_ROW_11 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 349 of file gemm_helpers.h.
#define LOAD_ROW_12 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 354 of file gemm_helpers.h.
#define LOAD_ROW_13 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 359 of file gemm_helpers.h.
#define LOAD_ROW_14 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 364 of file gemm_helpers.h.
#define LOAD_ROW_15 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 369 of file gemm_helpers.h.
#define LOAD_ROW_16 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 374 of file gemm_helpers.h.
#define LOAD_ROW_2 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 304 of file gemm_helpers.h.
#define LOAD_ROW_3 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 309 of file gemm_helpers.h.
#define LOAD_ROW_4 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 314 of file gemm_helpers.h.
#define LOAD_ROW_5 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 319 of file gemm_helpers.h.
#define LOAD_ROW_6 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 324 of file gemm_helpers.h.
#define LOAD_ROW_7 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 329 of file gemm_helpers.h.
#define LOAD_ROW_8 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 334 of file gemm_helpers.h.
#define LOAD_ROW_9 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 339 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_1 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 831 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_10 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 911 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_11 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 920 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_12 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 929 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_13 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 938 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_14 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 947 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_15 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 956 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_16 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 965 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_2 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 839 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_3 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 848 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_4 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 857 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_5 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 866 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_6 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 875 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_7 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 884 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_8 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 893 of file gemm_helpers.h.
#define LOAD_ROW_INDIRECT_9 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Y, | |||
Y_MASK | |||
) |
Definition at line 902 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_1 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 422 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_10 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 466 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_11 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 471 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_12 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 476 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_13 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 481 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_14 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 486 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_15 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 491 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_16 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 496 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_2 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 426 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_3 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 431 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_4 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 436 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_5 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 441 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_6 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 446 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_7 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 451 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_8 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 456 of file gemm_helpers.h.
#define LOAD_ROW_PARTIAL_9 | ( | N0, | |
LOAD_N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 461 of file gemm_helpers.h.
#define LOAD_SCALAR_AS_VECTOR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) | LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) |
Definition at line 1111 of file gemm_helpers.h.
#define LOAD_SCALAR_AS_VECTOR_STR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
OFFSET, | |||
STRIDE_Y | |||
) | LOAD_ELEMENT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) |
Definition at line 1109 of file gemm_helpers.h.
#define LOAD_TENSOR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) |
Definition at line 190 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X0 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | ({}) |
Definition at line 206 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X1 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); |
Definition at line 208 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X10 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 239 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X11 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 243 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X12 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 247 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X13 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 251 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X14 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 256 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X15 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 261 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X16 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); |
Definition at line 266 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X2 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); |
Definition at line 211 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X3 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); |
Definition at line 214 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X4 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); |
Definition at line 217 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X5 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 220 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X6 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 224 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X7 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 228 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X8 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) | LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); |
Definition at line 232 of file gemm_helpers.h.
#define LOAD_TENSOR_M0X9 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
a, | |||
input_ptr, | |||
src_stride_y, | |||
zin | |||
) |
Definition at line 235 of file gemm_helpers.h.
#define LOAD_TENSOR_M0XN0 | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) |
Definition at line 284 of file gemm_helpers.h.
#define LOAD_TENSOR_M0XN0_STR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_TENSOR_M0X##N0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) |
Definition at line 282 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_0 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | ({}) |
Definition at line 103 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_1 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##0) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0)); |
Definition at line 105 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_10 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 140 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_11 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 144 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_12 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 148 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_13 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 152 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_14 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 156 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_15 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 160 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_16 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 164 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_2 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 108 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_3 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 112 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_4 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 116 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_5 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 120 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_6 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 124 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_7 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 128 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_8 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 132 of file gemm_helpers.h.
#define LOAD_TENSOR_ROW_9 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) |
Definition at line 136 of file gemm_helpers.h.
#define LOAD_TENSOR_STR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
PTR, | |||
COL_OFFSET, | |||
STRIDE_Y, | |||
Z | |||
) | LOAD_TENSOR_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) |
Definition at line 188 of file gemm_helpers.h.
#define LOAD_TEXTURE2D | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) | LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) |
Definition at line 814 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_1 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) | BASENAME##0 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 0 * X_STEP_ROW), (Y_COORD + 0 * Y_STEP_ROW)) |
Definition at line 730 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_10 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 765 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_11 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 769 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_12 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 773 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_13 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 777 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_14 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 781 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_15 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 785 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_16 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 789 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_2 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 733 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_3 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 737 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_4 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 741 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_5 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 745 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_6 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 749 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_7 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 753 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_8 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 757 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_ROW_9 | ( | N0, | |
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) |
Definition at line 761 of file gemm_helpers.h.
#define LOAD_TEXTURE2D_STR | ( | M0, | |
N0, | |||
DATA_TYPE, | |||
BASENAME, | |||
IMG, | |||
X_COORD, | |||
Y_COORD, | |||
X_STEP_ROW, | |||
Y_STEP_ROW | |||
) | LOAD_TEXTURE2D_ROW_##M0(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) |
Definition at line 812 of file gemm_helpers.h.
#define SCALAR_ACCESS | ( | offset, | |
n0, | |||
x | |||
) | SCALAR_ACCESS_STR(offset, n0, x) |
Definition at line 37 of file gemm_helpers.h.
#define scalar_access_0_1 | ( | x | ) | ((x).s0) |
Definition at line 40 of file gemm_helpers.h.
#define scalar_access_0_16 | ( | x | ) | ((x).s0123456789ABCDEF) |
Definition at line 45 of file gemm_helpers.h.
#define scalar_access_0_2 | ( | x | ) | ((x).s01) |
Definition at line 41 of file gemm_helpers.h.
#define scalar_access_0_3 | ( | x | ) | ((x).s012) |
Definition at line 42 of file gemm_helpers.h.
#define scalar_access_0_4 | ( | x | ) | ((x).s0123) |
Definition at line 43 of file gemm_helpers.h.
#define scalar_access_0_8 | ( | x | ) | ((x).s01234567) |
Definition at line 44 of file gemm_helpers.h.
#define scalar_access_12_1 | ( | x | ) | ((x).sC) |
Definition at line 83 of file gemm_helpers.h.
#define scalar_access_12_2 | ( | x | ) | ((x).sCD) |
Definition at line 84 of file gemm_helpers.h.
#define scalar_access_12_3 | ( | x | ) | ((x).sCDE) |
Definition at line 85 of file gemm_helpers.h.
#define scalar_access_12_4 | ( | x | ) | ((x).sCDEF) |
Definition at line 86 of file gemm_helpers.h.
#define scalar_access_16_1 | ( | x | ) | ((x).sF) |
Definition at line 89 of file gemm_helpers.h.
#define scalar_access_1_1 | ( | x | ) | ((x).s1) |
Definition at line 48 of file gemm_helpers.h.
#define scalar_access_1_2 | ( | x | ) | ((x).s12) |
Definition at line 49 of file gemm_helpers.h.
#define scalar_access_1_3 | ( | x | ) | ((x).s123) |
Definition at line 50 of file gemm_helpers.h.
#define scalar_access_1_4 | ( | x | ) | ((x).s1234) |
Definition at line 51 of file gemm_helpers.h.
#define scalar_access_1_8 | ( | x | ) | ((x).s12345678) |
Definition at line 52 of file gemm_helpers.h.
#define scalar_access_2_1 | ( | x | ) | ((x).s2) |
Definition at line 55 of file gemm_helpers.h.
#define scalar_access_2_2 | ( | x | ) | ((x).s23) |
Definition at line 56 of file gemm_helpers.h.
#define scalar_access_2_3 | ( | x | ) | ((x).s234) |
Definition at line 57 of file gemm_helpers.h.
#define scalar_access_2_4 | ( | x | ) | ((x).s2345) |
Definition at line 58 of file gemm_helpers.h.
#define scalar_access_2_8 | ( | x | ) | ((x).s23456789) |
Definition at line 59 of file gemm_helpers.h.
#define scalar_access_3_1 | ( | x | ) | ((x).s3) |
Definition at line 62 of file gemm_helpers.h.
#define scalar_access_3_2 | ( | x | ) | ((x).s34) |
Definition at line 63 of file gemm_helpers.h.
#define scalar_access_3_3 | ( | x | ) | ((x).s345) |
Definition at line 64 of file gemm_helpers.h.
#define scalar_access_3_4 | ( | x | ) | ((x).s3456) |
Definition at line 65 of file gemm_helpers.h.
#define scalar_access_3_8 | ( | x | ) | ((x).s3456789A) |
Definition at line 66 of file gemm_helpers.h.
#define scalar_access_4_1 | ( | x | ) | ((x).s4) |
Definition at line 69 of file gemm_helpers.h.
#define scalar_access_4_2 | ( | x | ) | ((x).s45) |
Definition at line 70 of file gemm_helpers.h.
#define scalar_access_4_3 | ( | x | ) | ((x).s456) |
Definition at line 71 of file gemm_helpers.h.
#define scalar_access_4_4 | ( | x | ) | ((x).s4567) |
Definition at line 72 of file gemm_helpers.h.
#define scalar_access_4_8 | ( | x | ) | ((x).s456789AB) |
Definition at line 73 of file gemm_helpers.h.
#define scalar_access_8_1 | ( | x | ) | ((x).s8) |
Definition at line 76 of file gemm_helpers.h.
#define scalar_access_8_2 | ( | x | ) | ((x).s89) |
Definition at line 77 of file gemm_helpers.h.
#define scalar_access_8_3 | ( | x | ) | ((x).s89A) |
Definition at line 78 of file gemm_helpers.h.
#define scalar_access_8_4 | ( | x | ) | ((x).s89AB) |
Definition at line 79 of file gemm_helpers.h.
#define scalar_access_8_8 | ( | x | ) | ((x).s89ABCDEF) |
Definition at line 80 of file gemm_helpers.h.
Utility macro to access a vector with the scalar positions.
Supported cases are: Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
[in] | offset | The offset within the vector. Offset can only be of the same size of the OpenCL vector (2,3,4,8,16) |
[in] | n0 | The number of consecutive columns to access. n0 + offset must be <= 16 |
[in] | x | Vector to access |
Definition at line 36 of file gemm_helpers.h.
#define SCALE_BLOCK | ( | N, | |
DATA_TYPE, | |||
BASENAME, | |||
SCALE | |||
) | SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) |
Definition at line 1299 of file gemm_helpers.h.
Definition at line 1298 of file gemm_helpers.h.
#define SCALE_ROW_1 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) | BASENAME##0 *= (DATA_TYPE)SCALE; |
Definition at line 1224 of file gemm_helpers.h.
#define SCALE_ROW_10 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1258 of file gemm_helpers.h.
#define SCALE_ROW_11 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1262 of file gemm_helpers.h.
#define SCALE_ROW_12 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1266 of file gemm_helpers.h.
#define SCALE_ROW_13 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1270 of file gemm_helpers.h.
#define SCALE_ROW_14 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1274 of file gemm_helpers.h.
#define SCALE_ROW_15 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1278 of file gemm_helpers.h.
#define SCALE_ROW_16 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1282 of file gemm_helpers.h.
#define SCALE_ROW_2 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1226 of file gemm_helpers.h.
#define SCALE_ROW_3 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1230 of file gemm_helpers.h.
#define SCALE_ROW_4 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1234 of file gemm_helpers.h.
#define SCALE_ROW_5 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1238 of file gemm_helpers.h.
#define SCALE_ROW_6 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1242 of file gemm_helpers.h.
#define SCALE_ROW_7 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1246 of file gemm_helpers.h.
#define SCALE_ROW_8 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1250 of file gemm_helpers.h.
#define SCALE_ROW_9 | ( | DATA_TYPE, | |
BASENAME, | |||
SCALE | |||
) |
Definition at line 1254 of file gemm_helpers.h.
#define TRANSPOSE_K0X1 | ( | K0, | |
BASENAME, | |||
BS, | |||
TYPE | |||
) | COLUMN_VECTOR_SCALAR(K0, 0, BASENAME, BS, TYPE); |
Definition at line 1373 of file gemm_helpers.h.
#define TRANSPOSE_K0X16 | ( | K0, | |
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Definition at line 1389 of file gemm_helpers.h.
#define TRANSPOSE_K0X2 | ( | K0, | |
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Definition at line 1374 of file gemm_helpers.h.
#define TRANSPOSE_K0X3 | ( | K0, | |
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Definition at line 1377 of file gemm_helpers.h.
#define TRANSPOSE_K0X4 | ( | K0, | |
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Definition at line 1380 of file gemm_helpers.h.
#define TRANSPOSE_K0X8 | ( | K0, | |
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Definition at line 1383 of file gemm_helpers.h.
#define TRANSPOSE_K0XN0 | ( | K0, | |
N0, | |||
BASENAME, | |||
BS, | |||
TYPE | |||
) |
Create transposed vectors form the given source vectors.
[in] | K0 | The size of source vectors |
[in] | N0 | The number of source vectors |
[in] | BASENAME | The basename of transposed vectors |
[in] | BS | The basename of source vectors for transposition |
[in] | TYPE | The data type of the transposed vectors |
Definition at line 1435 of file gemm_helpers.h.