CORTEXA57: Add assembly kernels for copy routines

amodra · Jan 10, 2017 · 6085386 · 6085386
1 parent 002b41f
commit 6085386
Show file tree

Hide file tree

Showing 5 changed files with 1,983 additions and 0 deletions.
diff --git a/kernel/arm64/KERNEL.CORTEXA57 b/kernel/arm64/KERNEL.CORTEXA57
@@ -75,14 +75,29 @@ SGEMMOTCOPYOBJ =  sgemm_otcopy.o
 
 DGEMMKERNEL    =  dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
 DTRMMKERNEL    =  dtrmm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
+
 ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N))
+
+ifeq ($(DGEMM_UNROLL_M), 8)
+DGEMMINCOPY    =  dgemm_ncopy_$(DGEMM_UNROLL_M).S
+DGEMMITCOPY    =  dgemm_tcopy_$(DGEMM_UNROLL_M).S
+else
 DGEMMINCOPY    =  ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c
 DGEMMITCOPY    =  ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c
+endif
+
 DGEMMINCOPYOBJ =  dgemm_incopy.o
 DGEMMITCOPYOBJ =  dgemm_itcopy.o
 endif
+
+ifeq ($(DGEMM_UNROLL_N), 4)
+DGEMMONCOPY    =  dgemm_ncopy_$(DGEMM_UNROLL_N).S
+DGEMMOTCOPY    =  dgemm_tcopy_$(DGEMM_UNROLL_N).S
+else
 DGEMMONCOPY    =  ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c
 DGEMMOTCOPY    =  ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c
+endif
+
 DGEMMONCOPYOBJ =  dgemm_oncopy.o
 DGEMMOTCOPYOBJ =  dgemm_otcopy.o
 

diff --git a/kernel/arm64/dgemm_ncopy_4.S b/kernel/arm64/dgemm_ncopy_4.S
@@ -0,0 +1,340 @@
+/***************************************************************************
+Copyright (c) 2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+#define	M	x0
+#define	N	x1
+#define	A00	x2
+#define	LDA	x3
+#define	B00	x4
+
+#define	A01	x5
+#define	A02	x6
+#define	A03	x7
+#define	A04	x8
+
+#define I	x9
+#define	J	x10
+
+#define	TEMP1	x11
+#define	TEMP2	x12
+
+#define A_PREFETCH	2560
+
+/**************************************************************************************
+* Macro definitions
+**************************************************************************************/
+
+.macro SAVE_REGS
+	add	sp, sp, #-(11 * 16)
+	stp	d8, d9, [sp, #(0 * 16)]
+	stp	d10, d11, [sp, #(1 * 16)]
+	stp	d12, d13, [sp, #(2 * 16)]
+	stp	d14, d15, [sp, #(3 * 16)]
+	stp	d16, d17, [sp, #(4 * 16)]
+	stp	x18, x19, [sp, #(5 * 16)]
+	stp	x20, x21, [sp, #(6 * 16)]
+	stp	x22, x23, [sp, #(7 * 16)]
+	stp	x24, x25, [sp, #(8 * 16)]
+	stp	x26, x27, [sp, #(9 * 16)]
+	str	x28, [sp, #(10 * 16)]
+.endm
+
+.macro RESTORE_REGS
+	ldp	d8, d9, [sp, #(0 * 16)]
+	ldp	d10, d11, [sp, #(1 * 16)]
+	ldp	d12, d13, [sp, #(2 * 16)]
+	ldp	d14, d15, [sp, #(3 * 16)]
+	ldp	d16, d17, [sp, #(4 * 16)]
+	ldp	x18, x19, [sp, #(5 * 16)]
+	ldp	x20, x21, [sp, #(6 * 16)]
+	ldp	x22, x23, [sp, #(7 * 16)]
+	ldp	x24, x25, [sp, #(8 * 16)]
+	ldp	x26, x27, [sp, #(9 * 16)]
+	ldr	x28, [sp, #(10 * 16)]
+	add	sp, sp, #(11*16)
+.endm
+
+.macro COPY4x4
+	//prfm	PLDL1KEEP, [A01, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A02, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A03, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A04, #A_PREFETCH]
+
+	ldp	q0, q1, [A01], #32
+	ins	v8.d[0], v0.d[0]
+	ins	v10.d[0], v0.d[1]
+	ins	v12.d[0], v1.d[0]
+	ins	v14.d[0], v1.d[1]
+
+	ldp	q2, q3, [A02], #32
+	ins	v8.d[1], v2.d[0]
+	ins	v10.d[1], v2.d[1]
+	ins	v12.d[1], v3.d[0]
+	ins	v14.d[1], v3.d[1]
+
+	ldp	q4, q5, [A03], #32
+	ins	v9.d[0], v4.d[0]
+	ins	v11.d[0], v4.d[1]
+	ins	v13.d[0], v5.d[0]
+	ins	v15.d[0], v5.d[1]
+
+	ldp	q6, q7, [A04], #32
+	ins	v9.d[1], v6.d[0]
+	ins	v11.d[1], v6.d[1]
+	ins	v13.d[1], v7.d[0]
+	ins	v15.d[1], v7.d[1]
+
+	st1	{v8.2d, v9.2d, v10.2d, v11.2d}, [B00]
+	add	B00, B00, #64
+
+	st1	{v12.2d, v13.2d, v14.2d, v15.2d}, [B00]
+	add	B00, B00, #64
+.endm
+
+.macro COPY1x4
+	//prfm	PLDL1KEEP, [A01, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A02, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A03, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A04, #A_PREFETCH]
+
+	ldr	d0, [A01], #8
+	ldr	d1, [A02], #8
+	ldr	d2, [A03], #8
+	ldr	d3, [A04], #8
+
+	st1	{v0.1d, v1.1d, v2.1d, v3.1d}, [B00]
+	add	B00, B00, #32
+.endm
+
+.macro COPY4x2
+	//prfm	PLDL1KEEP, [A01, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A02, #A_PREFETCH]
+
+	ldp	q0, q1, [A01], #32
+	ins	v8.d[0], v0.d[0]
+	ins	v9.d[0], v0.d[1]
+	ins	v10.d[0], v1.d[0]
+	ins	v11.d[0], v1.d[1]
+
+	ldp	q2, q3, [A02], #32
+	ins	v8.d[1], v2.d[0]
+	ins	v9.d[1], v2.d[1]
+	ins	v10.d[1], v3.d[0]
+	ins	v11.d[1], v3.d[1]
+
+	st1	{v8.2d, v9.2d, v10.2d, v11.2d}, [B00]
+	add	B00, B00, #64
+.endm
+
+
+.macro COPY1x2
+	//prfm	PLDL1KEEP, [A01, #A_PREFETCH]
+	//prfm	PLDL1KEEP, [A02, #A_PREFETCH]
+
+	ldr	d0, [A01], #8
+	ldr	d1, [A02], #8
+
+	stp	d0, d1, [B00]
+	add	B00, B00, #16
+.endm
+
+.macro COPY4x1
+	//prfm	PLDL1KEEP, [A01, #A_PREFETCH]
+
+	ldp	q0, q1, [A01], #32
+	stp	q0, q1, [B00], #32
+.endm
+
+
+.macro COPY1x1
+	//prfm	PLDL1KEEP, [A01, #A_PREFETCH]
+
+	ldr	d0, [A01], #8
+	str	d0, [B00], #8
+.endm
+
+/**************************************************************************************
+* End of macro definitions
+**************************************************************************************/
+
+	PROLOGUE
+
+	.align 5
+
+	SAVE_REGS
+
+	lsl	LDA, LDA, #3					// LDA = LDA * SIZE
+
+dgemm_ncopy_L4_BEGIN:
+
+	asr	J, N, #2					// J = N / 4
+	cmp 	J, #0
+	ble	dgemm_ncopy_L2_BEGIN
+
+	.align	5
+dgemm_ncopy_L4_M4_BEGIN:
+
+	mov	A01, A00
+	add	A02, A01, LDA
+	add	A03, A02, LDA
+	add	A04, A03, LDA
+	add	A00, A04, LDA
+
+	asr	I, M, #2					// I = M / 4
+	cmp	I, #0
+	ble	dgemm_ncopy_L4_M4_40
+
+	.align	5
+dgemm_ncopy_L4_M4_20:
+
+	COPY4x4
+
+	subs	I , I , #1
+	bne	dgemm_ncopy_L4_M4_20
+
+
+dgemm_ncopy_L4_M4_40:
+
+	and	I, M , #3
+	cmp	I, #0
+	ble	dgemm_ncopy_L4_M4_END
+
+	.align	5
+dgemm_ncopy_L4_M4_60:
+
+	COPY1x4
+
+	subs	I , I , #1
+	bne	dgemm_ncopy_L4_M4_60
+
+
+dgemm_ncopy_L4_M4_END:
+
+	subs	J , J, #1						// j--
+	bne	dgemm_ncopy_L4_M4_BEGIN
+
+
+
+/*********************************************************************************************/
+
+dgemm_ncopy_L2_BEGIN:
+
+	tst	N, #3
+	ble	dgemm_ncopy_L999
+
+	tst	N, #2
+	ble	dgemm_ncopy_L1_BEGIN
+
+dgemm_ncopy_L2_M4_BEGIN:
+	mov	A01, A00
+	add	A02, A01, LDA
+	add	A00, A02, LDA
+
+	asr	I, M, #2					// I = M / 4
+	cmp 	I, #0
+	ble	dgemm_ncopy_L2_M4_40
+
+	.align	5
+dgemm_ncopy_L2_M4_20:
+
+	COPY4x2
+
+	subs	I , I , #1
+	bne	dgemm_ncopy_L2_M4_20
+
+
+dgemm_ncopy_L2_M4_40:
+
+	and	I, M , #3
+	cmp	I, #0
+	ble	dgemm_ncopy_L2_M4_END
+
+	.align	5
+dgemm_ncopy_L2_M4_60:
+
+	COPY1x2
+
+	subs	I , I , #1
+	bne	dgemm_ncopy_L2_M4_60
+
+
+dgemm_ncopy_L2_M4_END:
+
+
+/*********************************************************************************************/
+
+dgemm_ncopy_L1_BEGIN:
+
+	tst	N, #1
+	ble	dgemm_ncopy_L999
+
+
+dgemm_ncopy_L1_M4_BEGIN:
+
+	mov	A01, A00
+
+	asr	I, M, #2					// I = M / 4
+	cmp	I, #0
+	ble	dgemm_ncopy_L1_M4_40
+
+	.align	5
+dgemm_ncopy_L1_M4_20:
+
+	COPY4x1
+
+	subs	I , I , #1
+	bne	dgemm_ncopy_L1_M4_20
+
+
+dgemm_ncopy_L1_M4_40:
+
+	and	I, M , #3
+	cmp	I, #0
+	ble	dgemm_ncopy_L1_M4_END
+
+	.align	5
+dgemm_ncopy_L1_M4_60:
+
+	COPY1x1
+
+	subs	I , I , #1
+	bne	dgemm_ncopy_L1_M4_60
+
+
+dgemm_ncopy_L1_M4_END:
+
+dgemm_ncopy_L999:
+
+	mov	x0, #0
+	RESTORE_REGS
+	ret
+
+	EPILOGUE
+