rpms
/
openblas
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
openblas/2023.patch

875 lines
35 KiB
Diff
Raw Blame History

From 9d8be1578983d9fec6a1a7ae81d4ef9c1ac4c08c Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 16 Feb 2019 18:24:11 +0100
Subject: [PATCH 1/4] Fix inline assembly constraints
rework indices to allow marking argument lda4 as input and output. For #2009
---
kernel/x86_64/sgemv_n_microk_nehalem-4.c | 54 ++++++++++++------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/kernel/x86_64/sgemv_n_microk_nehalem-4.c b/kernel/x86_64/sgemv_n_microk_nehalem-4.c
index 11a3e943b..d21232bfa 100644
--- a/kernel/x86_64/sgemv_n_microk_nehalem-4.c
+++ b/kernel/x86_64/sgemv_n_microk_nehalem-4.c
@@ -37,19 +37,19 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
__asm__ __volatile__
(
- "movss (%2), %%xmm12 \n\t" // x0
- "movss 4(%2), %%xmm13 \n\t" // x1
- "movss 8(%2), %%xmm14 \n\t" // x2
- "movss 12(%2), %%xmm15 \n\t" // x3
+ "movss (%3), %%xmm12 \n\t" // x0
+ "movss 4(%3), %%xmm13 \n\t" // x1
+ "movss 8(%3), %%xmm14 \n\t" // x2
+ "movss 12(%3), %%xmm15 \n\t" // x3
"shufps $0, %%xmm12, %%xmm12\n\t"
"shufps $0, %%xmm13, %%xmm13\n\t"
"shufps $0, %%xmm14, %%xmm14\n\t"
"shufps $0, %%xmm15, %%xmm15\n\t"
- "movss 16(%2), %%xmm0 \n\t" // x4
- "movss 20(%2), %%xmm1 \n\t" // x5
- "movss 24(%2), %%xmm2 \n\t" // x6
- "movss 28(%2), %%xmm3 \n\t" // x7
+ "movss 16(%3), %%xmm0 \n\t" // x4
+ "movss 20(%3), %%xmm1 \n\t" // x5
+ "movss 24(%3), %%xmm2 \n\t" // x6
+ "movss 28(%3), %%xmm3 \n\t" // x7
"shufps $0, %%xmm0 , %%xmm0 \n\t"
"shufps $0, %%xmm1 , %%xmm1 \n\t"
"shufps $0, %%xmm2 , %%xmm2 \n\t"
@@ -63,13 +63,13 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"1: \n\t"
"xorps %%xmm4 , %%xmm4 \n\t"
"xorps %%xmm5 , %%xmm5 \n\t"
- "movups (%3,%0,4), %%xmm7 \n\t" // 4 * y
+ "movups (%4,%0,4), %%xmm7 \n\t" // 4 * y
".p2align 1 \n\t"
- "movups (%4,%0,4), %%xmm8 \n\t"
- "movups (%5,%0,4), %%xmm9 \n\t"
- "movups (%6,%0,4), %%xmm10 \n\t"
- "movups (%7,%0,4), %%xmm11 \n\t"
+ "movups (%5,%0,4), %%xmm8 \n\t"
+ "movups (%6,%0,4), %%xmm9 \n\t"
+ "movups (%7,%0,4), %%xmm10 \n\t"
+ "movups (%8,%0,4), %%xmm11 \n\t"
".p2align 1 \n\t"
"mulps %%xmm12, %%xmm8 \n\t"
"mulps %%xmm13, %%xmm9 \n\t"
@@ -80,10 +80,10 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"addps %%xmm10, %%xmm4 \n\t"
"addps %%xmm11, %%xmm5 \n\t"
- "movups (%4,%8,4), %%xmm8 \n\t"
- "movups (%5,%8,4), %%xmm9 \n\t"
- "movups (%6,%8,4), %%xmm10 \n\t"
- "movups (%7,%8,4), %%xmm11 \n\t"
+ "movups (%5,%2,4), %%xmm8 \n\t"
+ "movups (%6,%2,4), %%xmm9 \n\t"
+ "movups (%7,%2,4), %%xmm10 \n\t"
+ "movups (%8,%2,4), %%xmm11 \n\t"
".p2align 1 \n\t"
"mulps %%xmm0 , %%xmm8 \n\t"
"mulps %%xmm1 , %%xmm9 \n\t"
@@ -94,28 +94,28 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"addps %%xmm10, %%xmm4 \n\t"
"addps %%xmm11, %%xmm5 \n\t"
- "addq $4 , %8 \n\t"
+ "addq $4 , %2 \n\t"
"addps %%xmm5 , %%xmm4 \n\t"
"addq $4 , %0 \n\t"
"mulps %%xmm6 , %%xmm4 \n\t"
"subq $4 , %1 \n\t"
"addps %%xmm4 , %%xmm7 \n\t"
- "movups %%xmm7 , -16(%3,%0,4) \n\t" // 4 * y
+ "movups %%xmm7 , -16(%4,%0,4) \n\t" // 4 * y
"jnz 1b \n\t"
:
"+r" (i), // 0
- "+r" (n) // 1
+ "+r" (n), // 1
+ "+r" (lda4) // 2
:
- "r" (x), // 2
- "r" (y), // 3
- "r" (ap[0]), // 4
- "r" (ap[1]), // 5
- "r" (ap[2]), // 6
- "r" (ap[3]), // 7
- "r" (lda4), // 8
+ "r" (x), // 3
+ "r" (y), // 4
+ "r" (ap[0]), // 5
+ "r" (ap[1]), // 6
+ "r" (ap[2]), // 7
+ "r" (ap[3]), // 8
"r" (alpha) // 9
: "cc",
"%xmm0", "%xmm1",
From e976557d2965efb687aaaf88e7829bdd9438a7a6 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 16 Feb 2019 18:36:39 +0100
Subject: [PATCH 2/4] Fix inline assembly constraints
rework indices to allow marking argument lda as input and output.
---
kernel/x86_64/sgemv_n_microk_sandy-4.c | 130 ++++++++++++-------------
1 file changed, 65 insertions(+), 65 deletions(-)
diff --git a/kernel/x86_64/sgemv_n_microk_sandy-4.c b/kernel/x86_64/sgemv_n_microk_sandy-4.c
index b35daa35b..3fc46542b 100644
--- a/kernel/x86_64/sgemv_n_microk_sandy-4.c
+++ b/kernel/x86_64/sgemv_n_microk_sandy-4.c
@@ -39,14 +39,14 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
__asm__ __volatile__
(
"vzeroupper \n\t"
- "vbroadcastss (%2), %%ymm12 \n\t" // x0
- "vbroadcastss 4(%2), %%ymm13 \n\t" // x1
- "vbroadcastss 8(%2), %%ymm14 \n\t" // x2
- "vbroadcastss 12(%2), %%ymm15 \n\t" // x3
- "vbroadcastss 16(%2), %%ymm0 \n\t" // x4
- "vbroadcastss 20(%2), %%ymm1 \n\t" // x5
- "vbroadcastss 24(%2), %%ymm2 \n\t" // x6
- "vbroadcastss 28(%2), %%ymm3 \n\t" // x7
+ "vbroadcastss (%3), %%ymm12 \n\t" // x0
+ "vbroadcastss 4(%3), %%ymm13 \n\t" // x1
+ "vbroadcastss 8(%3), %%ymm14 \n\t" // x2
+ "vbroadcastss 12(%3), %%ymm15 \n\t" // x3
+ "vbroadcastss 16(%3), %%ymm0 \n\t" // x4
+ "vbroadcastss 20(%3), %%ymm1 \n\t" // x5
+ "vbroadcastss 24(%3), %%ymm2 \n\t" // x6
+ "vbroadcastss 28(%3), %%ymm3 \n\t" // x7
"vbroadcastss (%9), %%ymm6 \n\t" // alpha
@@ -55,21 +55,21 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vxorps %%xmm4 , %%xmm4 , %%xmm4 \n\t"
"vxorps %%xmm5 , %%xmm5 , %%xmm5 \n\t"
- "vmovups (%3,%0,4), %%xmm7 \n\t" // 4 * y
+ "vmovups (%4,%0,4), %%xmm7 \n\t" // 4 * y
- "vmulps (%4,%0,4), %%xmm12, %%xmm8 \n\t"
- "vmulps (%5,%0,4), %%xmm13, %%xmm10 \n\t"
- "vmulps (%6,%0,4), %%xmm14, %%xmm9 \n\t"
- "vmulps (%7,%0,4), %%xmm15, %%xmm11 \n\t"
+ "vmulps (%5,%0,4), %%xmm12, %%xmm8 \n\t"
+ "vmulps (%6,%0,4), %%xmm13, %%xmm10 \n\t"
+ "vmulps (%7,%0,4), %%xmm14, %%xmm9 \n\t"
+ "vmulps (%8,%0,4), %%xmm15, %%xmm11 \n\t"
"vaddps %%xmm4, %%xmm8 , %%xmm4 \n\t"
"vaddps %%xmm5, %%xmm10, %%xmm5 \n\t"
"vaddps %%xmm4, %%xmm9 , %%xmm4 \n\t"
"vaddps %%xmm5, %%xmm11, %%xmm5 \n\t"
- "vmulps (%4,%8,4), %%xmm0 , %%xmm8 \n\t"
- "vmulps (%5,%8,4), %%xmm1 , %%xmm10 \n\t"
- "vmulps (%6,%8,4), %%xmm2 , %%xmm9 \n\t"
- "vmulps (%7,%8,4), %%xmm3 , %%xmm11 \n\t"
+ "vmulps (%5,%2,4), %%xmm0 , %%xmm8 \n\t"
+ "vmulps (%6,%2,4), %%xmm1 , %%xmm10 \n\t"
+ "vmulps (%7,%2,4), %%xmm2 , %%xmm9 \n\t"
+ "vmulps (%8,%2,4), %%xmm3 , %%xmm11 \n\t"
"vaddps %%xmm4, %%xmm8 , %%xmm4 \n\t"
"vaddps %%xmm5, %%xmm10, %%xmm5 \n\t"
"vaddps %%xmm4, %%xmm9 , %%xmm4 \n\t"
@@ -79,9 +79,9 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vmulps %%xmm6, %%xmm4 , %%xmm5 \n\t"
"vaddps %%xmm5, %%xmm7 , %%xmm5 \n\t"
- "vmovups %%xmm5, (%3,%0,4) \n\t" // 4 * y
+ "vmovups %%xmm5, (%4,%0,4) \n\t" // 4 * y
- "addq $4, %8 \n\t"
+ "addq $4, %2 \n\t"
"addq $4, %0 \n\t"
"subq $4, %1 \n\t"
@@ -92,21 +92,21 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t"
"vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t"
- "vmovups (%3,%0,4), %%ymm7 \n\t" // 8 * y
+ "vmovups (%4,%0,4), %%ymm7 \n\t" // 8 * y
- "vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t"
- "vmulps (%5,%0,4), %%ymm13, %%ymm10 \n\t"
- "vmulps (%6,%0,4), %%ymm14, %%ymm9 \n\t"
- "vmulps (%7,%0,4), %%ymm15, %%ymm11 \n\t"
+ "vmulps (%5,%0,4), %%ymm12, %%ymm8 \n\t"
+ "vmulps (%6,%0,4), %%ymm13, %%ymm10 \n\t"
+ "vmulps (%7,%0,4), %%ymm14, %%ymm9 \n\t"
+ "vmulps (%8,%0,4), %%ymm15, %%ymm11 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm10, %%ymm5 \n\t"
"vaddps %%ymm4, %%ymm9 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
- "vmulps (%4,%8,4), %%ymm0 , %%ymm8 \n\t"
- "vmulps (%5,%8,4), %%ymm1 , %%ymm10 \n\t"
- "vmulps (%6,%8,4), %%ymm2 , %%ymm9 \n\t"
- "vmulps (%7,%8,4), %%ymm3 , %%ymm11 \n\t"
+ "vmulps (%5,%2,4), %%ymm0 , %%ymm8 \n\t"
+ "vmulps (%6,%2,4), %%ymm1 , %%ymm10 \n\t"
+ "vmulps (%7,%2,4), %%ymm2 , %%ymm9 \n\t"
+ "vmulps (%8,%2,4), %%ymm3 , %%ymm11 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm10, %%ymm5 \n\t"
"vaddps %%ymm4, %%ymm9 , %%ymm4 \n\t"
@@ -116,9 +116,9 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vmulps %%ymm6, %%ymm4 , %%ymm5 \n\t"
"vaddps %%ymm5, %%ymm7 , %%ymm5 \n\t"
- "vmovups %%ymm5, (%3,%0,4) \n\t" // 8 * y
+ "vmovups %%ymm5, (%4,%0,4) \n\t" // 8 * y
- "addq $8, %8 \n\t"
+ "addq $8, %2 \n\t"
"addq $8, %0 \n\t"
"subq $8, %1 \n\t"
@@ -134,45 +134,45 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t"
"vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t"
- "prefetcht0 192(%4,%0,4) \n\t"
- "vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t"
- "vmulps 32(%4,%0,4), %%ymm12, %%ymm9 \n\t"
"prefetcht0 192(%5,%0,4) \n\t"
- "vmulps (%5,%0,4), %%ymm13, %%ymm10 \n\t"
- "vmulps 32(%5,%0,4), %%ymm13, %%ymm11 \n\t"
+ "vmulps (%5,%0,4), %%ymm12, %%ymm8 \n\t"
+ "vmulps 32(%5,%0,4), %%ymm12, %%ymm9 \n\t"
+ "prefetcht0 192(%6,%0,4) \n\t"
+ "vmulps (%6,%0,4), %%ymm13, %%ymm10 \n\t"
+ "vmulps 32(%6,%0,4), %%ymm13, %%ymm11 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm9 , %%ymm5 \n\t"
"vaddps %%ymm4, %%ymm10, %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
- "prefetcht0 192(%6,%0,4) \n\t"
- "vmulps (%6,%0,4), %%ymm14, %%ymm8 \n\t"
- "vmulps 32(%6,%0,4), %%ymm14, %%ymm9 \n\t"
"prefetcht0 192(%7,%0,4) \n\t"
- "vmulps (%7,%0,4), %%ymm15, %%ymm10 \n\t"
- "vmulps 32(%7,%0,4), %%ymm15, %%ymm11 \n\t"
+ "vmulps (%7,%0,4), %%ymm14, %%ymm8 \n\t"
+ "vmulps 32(%7,%0,4), %%ymm14, %%ymm9 \n\t"
+ "prefetcht0 192(%8,%0,4) \n\t"
+ "vmulps (%8,%0,4), %%ymm15, %%ymm10 \n\t"
+ "vmulps 32(%8,%0,4), %%ymm15, %%ymm11 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm9 , %%ymm5 \n\t"
"vaddps %%ymm4, %%ymm10, %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
- "prefetcht0 192(%4,%8,4) \n\t"
- "vmulps (%4,%8,4), %%ymm0 , %%ymm8 \n\t"
- "vmulps 32(%4,%8,4), %%ymm0 , %%ymm9 \n\t"
- "prefetcht0 192(%5,%8,4) \n\t"
- "vmulps (%5,%8,4), %%ymm1 , %%ymm10 \n\t"
- "vmulps 32(%5,%8,4), %%ymm1 , %%ymm11 \n\t"
+ "prefetcht0 192(%5,%2,4) \n\t"
+ "vmulps (%5,%2,4), %%ymm0 , %%ymm8 \n\t"
+ "vmulps 32(%5,%2,4), %%ymm0 , %%ymm9 \n\t"
+ "prefetcht0 192(%6,%2,4) \n\t"
+ "vmulps (%6,%2,4), %%ymm1 , %%ymm10 \n\t"
+ "vmulps 32(%6,%2,4), %%ymm1 , %%ymm11 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm9 , %%ymm5 \n\t"
"vaddps %%ymm4, %%ymm10, %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
- "prefetcht0 192(%6,%8,4) \n\t"
- "vmulps (%6,%8,4), %%ymm2 , %%ymm8 \n\t"
- "vmulps 32(%6,%8,4), %%ymm2 , %%ymm9 \n\t"
- "prefetcht0 192(%7,%8,4) \n\t"
- "vmulps (%7,%8,4), %%ymm3 , %%ymm10 \n\t"
- "vmulps 32(%7,%8,4), %%ymm3 , %%ymm11 \n\t"
+ "prefetcht0 192(%7,%2,4) \n\t"
+ "vmulps (%7,%2,4), %%ymm2 , %%ymm8 \n\t"
+ "vmulps 32(%7,%2,4), %%ymm2 , %%ymm9 \n\t"
+ "prefetcht0 192(%8,%2,4) \n\t"
+ "vmulps (%8,%2,4), %%ymm3 , %%ymm10 \n\t"
+ "vmulps 32(%8,%2,4), %%ymm3 , %%ymm11 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm9 , %%ymm5 \n\t"
"vaddps %%ymm4, %%ymm10, %%ymm4 \n\t"
@@ -181,13 +181,13 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vmulps %%ymm6, %%ymm4 , %%ymm4 \n\t"
"vmulps %%ymm6, %%ymm5 , %%ymm5 \n\t"
- "vaddps (%3,%0,4), %%ymm4 , %%ymm4 \n\t" // 8 * y
- "vaddps 32(%3,%0,4), %%ymm5 , %%ymm5 \n\t" // 8 * y
+ "vaddps (%4,%0,4), %%ymm4 , %%ymm4 \n\t" // 8 * y
+ "vaddps 32(%4,%0,4), %%ymm5 , %%ymm5 \n\t" // 8 * y
- "vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y
- "vmovups %%ymm5, 32(%3,%0,4) \n\t" // 8 * y
+ "vmovups %%ymm4, (%4,%0,4) \n\t" // 8 * y
+ "vmovups %%ymm5, 32(%4,%0,4) \n\t" // 8 * y
- "addq $16, %8 \n\t"
+ "addq $16, %2 \n\t"
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz 1b \n\t"
@@ -197,15 +197,15 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
:
"+r" (i), // 0
- "+r" (n) // 1
+ "+r" (n), // 1
+ "+r" (lda4) // 2
:
- "r" (x), // 2
- "r" (y), // 3
- "r" (ap[0]), // 4
- "r" (ap[1]), // 5
- "r" (ap[2]), // 6
- "r" (ap[3]), // 7
- "r" (lda4), // 8
+ "r" (x), // 3
+ "r" (y), // 4
+ "r" (ap[0]), // 5
+ "r" (ap[1]), // 6
+ "r" (ap[2]), // 7
+ "r" (ap[3]), // 8
"r" (alpha) // 9
: "cc",
"%xmm0", "%xmm1",
From efb9038f7273cddc1ef30fce6ed4df7967a2fb03 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 16 Feb 2019 18:46:17 +0100
Subject: [PATCH 3/4] Fix inline assembly constraints
---
kernel/x86_64/sgemv_n_microk_bulldozer-4.c | 194 ++++++++++-----------
1 file changed, 97 insertions(+), 97 deletions(-)
diff --git a/kernel/x86_64/sgemv_n_microk_bulldozer-4.c b/kernel/x86_64/sgemv_n_microk_bulldozer-4.c
index 31001c7f3..bbf06c84b 100644
--- a/kernel/x86_64/sgemv_n_microk_bulldozer-4.c
+++ b/kernel/x86_64/sgemv_n_microk_bulldozer-4.c
@@ -37,14 +37,14 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
__asm__ __volatile__
(
- "vbroadcastss (%2), %%xmm12 \n\t" // x0
- "vbroadcastss 4(%2), %%xmm13 \n\t" // x1
- "vbroadcastss 8(%2), %%xmm14 \n\t" // x2
- "vbroadcastss 12(%2), %%xmm15 \n\t" // x3
- "vbroadcastss 16(%2), %%xmm0 \n\t" // x4
- "vbroadcastss 20(%2), %%xmm1 \n\t" // x5
- "vbroadcastss 24(%2), %%xmm2 \n\t" // x6
- "vbroadcastss 28(%2), %%xmm3 \n\t" // x7
+ "vbroadcastss (%3), %%xmm12 \n\t" // x0
+ "vbroadcastss 4(%3), %%xmm13 \n\t" // x1
+ "vbroadcastss 8(%3), %%xmm14 \n\t" // x2
+ "vbroadcastss 12(%3), %%xmm15 \n\t" // x3
+ "vbroadcastss 16(%3), %%xmm0 \n\t" // x4
+ "vbroadcastss 20(%3), %%xmm1 \n\t" // x5
+ "vbroadcastss 24(%3), %%xmm2 \n\t" // x6
+ "vbroadcastss 28(%3), %%xmm3 \n\t" // x7
"vbroadcastss (%9), %%xmm8 \n\t" // alpha
@@ -54,22 +54,22 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vxorps %%xmm4, %%xmm4 , %%xmm4 \n\t"
"vxorps %%xmm5, %%xmm5 , %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%4,%0,4), %%xmm12, %%xmm4 \n\t"
- "vfmaddps %%xmm5, (%5,%0,4), %%xmm13, %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%6,%0,4), %%xmm14, %%xmm4 \n\t"
- "vfmaddps %%xmm5, (%7,%0,4), %%xmm15, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%5,%0,4), %%xmm12, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, (%6,%0,4), %%xmm13, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%7,%0,4), %%xmm14, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, (%8,%0,4), %%xmm15, %%xmm5 \n\t"
"addq $4 , %0 \n\t"
- "vfmaddps %%xmm4, (%4,%8,4), %%xmm0 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, (%5,%8,4), %%xmm1 , %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%6,%8,4), %%xmm2 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, (%7,%8,4), %%xmm3 , %%xmm5 \n\t"
- "addq $4 , %8 \n\t"
+ "vfmaddps %%xmm4, (%5,%2,4), %%xmm0 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, (%6,%2,4), %%xmm1 , %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%7,%2,4), %%xmm2 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, (%8,%2,4), %%xmm3 , %%xmm5 \n\t"
+ "addq $4 , %2 \n\t"
"vaddps %%xmm5 , %%xmm4, %%xmm4 \n\t"
- "vfmaddps -16(%3,%0,4) , %%xmm4, %%xmm8,%%xmm6 \n\t"
+ "vfmaddps -16(%4,%0,4) , %%xmm4, %%xmm8,%%xmm6 \n\t"
"subq $4 , %1 \n\t"
- "vmovups %%xmm6, -16(%3,%0,4) \n\t" // 4 * y
+ "vmovups %%xmm6, -16(%4,%0,4) \n\t" // 4 * y
"2: \n\t"
@@ -79,31 +79,31 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vxorps %%xmm4, %%xmm4 , %%xmm4 \n\t"
"vxorps %%xmm5, %%xmm5 , %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%4,%0,4), %%xmm12, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%4,%0,4), %%xmm12, %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%5,%0,4), %%xmm13, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%5,%0,4), %%xmm13, %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%6,%0,4), %%xmm14, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%6,%0,4), %%xmm14, %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%7,%0,4), %%xmm15, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%7,%0,4), %%xmm15, %%xmm5 \n\t"
-
- "vfmaddps %%xmm4, (%4,%8,4), %%xmm0 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%4,%8,4), %%xmm0 , %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%5,%8,4), %%xmm1 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%5,%8,4), %%xmm1 , %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%6,%8,4), %%xmm2 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%6,%8,4), %%xmm2 , %%xmm5 \n\t"
- "vfmaddps %%xmm4, (%7,%8,4), %%xmm3 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%7,%8,4), %%xmm3 , %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%5,%0,4), %%xmm12, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%5,%0,4), %%xmm12, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%6,%0,4), %%xmm13, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%6,%0,4), %%xmm13, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%7,%0,4), %%xmm14, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%7,%0,4), %%xmm14, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%8,%0,4), %%xmm15, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%8,%0,4), %%xmm15, %%xmm5 \n\t"
+
+ "vfmaddps %%xmm4, (%5,%2,4), %%xmm0 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%5,%2,4), %%xmm0 , %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%6,%2,4), %%xmm1 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%6,%2,4), %%xmm1 , %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%7,%2,4), %%xmm2 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%7,%2,4), %%xmm2 , %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%8,%2,4), %%xmm3 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%8,%2,4), %%xmm3 , %%xmm5 \n\t"
- "vfmaddps (%3,%0,4) , %%xmm4,%%xmm8,%%xmm4 \n\t"
- "vfmaddps 16(%3,%0,4) , %%xmm5,%%xmm8,%%xmm5 \n\t"
- "vmovups %%xmm4, (%3,%0,4) \n\t" // 4 * y
- "vmovups %%xmm5, 16(%3,%0,4) \n\t" // 4 * y
+ "vfmaddps (%4,%0,4) , %%xmm4,%%xmm8,%%xmm4 \n\t"
+ "vfmaddps 16(%4,%0,4) , %%xmm5,%%xmm8,%%xmm5 \n\t"
+ "vmovups %%xmm4, (%4,%0,4) \n\t" // 4 * y
+ "vmovups %%xmm5, 16(%4,%0,4) \n\t" // 4 * y
"addq $8 , %0 \n\t"
- "addq $8 , %8 \n\t"
+ "addq $8 , %2 \n\t"
"subq $8 , %1 \n\t"
@@ -120,62 +120,62 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
"vxorps %%xmm6, %%xmm6 , %%xmm6 \n\t"
"vxorps %%xmm7, %%xmm7 , %%xmm7 \n\t"
- "prefetcht0 192(%4,%0,4) \n\t"
- "vfmaddps %%xmm4, (%4,%0,4), %%xmm12, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%4,%0,4), %%xmm12, %%xmm5 \n\t"
"prefetcht0 192(%5,%0,4) \n\t"
- "vfmaddps %%xmm4, (%5,%0,4), %%xmm13, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%5,%0,4), %%xmm13, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%5,%0,4), %%xmm12, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%5,%0,4), %%xmm12, %%xmm5 \n\t"
"prefetcht0 192(%6,%0,4) \n\t"
- "vfmaddps %%xmm4, (%6,%0,4), %%xmm14, %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%6,%0,4), %%xmm14, %%xmm5 \n\t"
+ "vfmaddps %%xmm4, (%6,%0,4), %%xmm13, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%6,%0,4), %%xmm13, %%xmm5 \n\t"
"prefetcht0 192(%7,%0,4) \n\t"
- "vfmaddps %%xmm4, (%7,%0,4), %%xmm15, %%xmm4 \n\t"
+ "vfmaddps %%xmm4, (%7,%0,4), %%xmm14, %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%7,%0,4), %%xmm14, %%xmm5 \n\t"
+ "prefetcht0 192(%8,%0,4) \n\t"
+ "vfmaddps %%xmm4, (%8,%0,4), %%xmm15, %%xmm4 \n\t"
".align 2 \n\t"
- "vfmaddps %%xmm5, 16(%7,%0,4), %%xmm15, %%xmm5 \n\t"
-
- "vfmaddps %%xmm6, 32(%4,%0,4), %%xmm12, %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%4,%0,4), %%xmm12, %%xmm7 \n\t"
- "vfmaddps %%xmm6, 32(%5,%0,4), %%xmm13, %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%5,%0,4), %%xmm13, %%xmm7 \n\t"
- "vfmaddps %%xmm6, 32(%6,%0,4), %%xmm14, %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%6,%0,4), %%xmm14, %%xmm7 \n\t"
- "vfmaddps %%xmm6, 32(%7,%0,4), %%xmm15, %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%7,%0,4), %%xmm15, %%xmm7 \n\t"
-
- "prefetcht0 192(%4,%8,4) \n\t"
- "vfmaddps %%xmm4, (%4,%8,4), %%xmm0 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%4,%8,4), %%xmm0 , %%xmm5 \n\t"
- "prefetcht0 192(%5,%8,4) \n\t"
- "vfmaddps %%xmm4, (%5,%8,4), %%xmm1 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%5,%8,4), %%xmm1 , %%xmm5 \n\t"
- "prefetcht0 192(%6,%8,4) \n\t"
- "vfmaddps %%xmm4, (%6,%8,4), %%xmm2 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%6,%8,4), %%xmm2 , %%xmm5 \n\t"
- "prefetcht0 192(%7,%8,4) \n\t"
- "vfmaddps %%xmm4, (%7,%8,4), %%xmm3 , %%xmm4 \n\t"
- "vfmaddps %%xmm5, 16(%7,%8,4), %%xmm3 , %%xmm5 \n\t"
+ "vfmaddps %%xmm5, 16(%8,%0,4), %%xmm15, %%xmm5 \n\t"
+
+ "vfmaddps %%xmm6, 32(%5,%0,4), %%xmm12, %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%5,%0,4), %%xmm12, %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%6,%0,4), %%xmm13, %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%6,%0,4), %%xmm13, %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%7,%0,4), %%xmm14, %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%7,%0,4), %%xmm14, %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%8,%0,4), %%xmm15, %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%8,%0,4), %%xmm15, %%xmm7 \n\t"
+
+ "prefetcht0 192(%5,%2,4) \n\t"
+ "vfmaddps %%xmm4, (%5,%2,4), %%xmm0 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%5,%2,4), %%xmm0 , %%xmm5 \n\t"
+ "prefetcht0 192(%6,%2,4) \n\t"
+ "vfmaddps %%xmm4, (%6,%2,4), %%xmm1 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%6,%2,4), %%xmm1 , %%xmm5 \n\t"
+ "prefetcht0 192(%7,%2,4) \n\t"
+ "vfmaddps %%xmm4, (%7,%2,4), %%xmm2 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%7,%2,4), %%xmm2 , %%xmm5 \n\t"
+ "prefetcht0 192(%8,%2,4) \n\t"
+ "vfmaddps %%xmm4, (%8,%2,4), %%xmm3 , %%xmm4 \n\t"
+ "vfmaddps %%xmm5, 16(%8,%2,4), %%xmm3 , %%xmm5 \n\t"
- "vfmaddps %%xmm6, 32(%4,%8,4), %%xmm0 , %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%4,%8,4), %%xmm0 , %%xmm7 \n\t"
- "vfmaddps %%xmm6, 32(%5,%8,4), %%xmm1 , %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%5,%8,4), %%xmm1 , %%xmm7 \n\t"
- "vfmaddps %%xmm6, 32(%6,%8,4), %%xmm2 , %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%6,%8,4), %%xmm2 , %%xmm7 \n\t"
- "vfmaddps %%xmm6, 32(%7,%8,4), %%xmm3 , %%xmm6 \n\t"
- "vfmaddps %%xmm7, 48(%7,%8,4), %%xmm3 , %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%5,%2,4), %%xmm0 , %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%5,%2,4), %%xmm0 , %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%6,%2,4), %%xmm1 , %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%6,%2,4), %%xmm1 , %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%7,%2,4), %%xmm2 , %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%7,%2,4), %%xmm2 , %%xmm7 \n\t"
+ "vfmaddps %%xmm6, 32(%8,%2,4), %%xmm3 , %%xmm6 \n\t"
+ "vfmaddps %%xmm7, 48(%8,%2,4), %%xmm3 , %%xmm7 \n\t"
- "vfmaddps (%3,%0,4) , %%xmm4,%%xmm8,%%xmm4 \n\t"
- "vfmaddps 16(%3,%0,4) , %%xmm5,%%xmm8,%%xmm5 \n\t"
- "vfmaddps 32(%3,%0,4) , %%xmm6,%%xmm8,%%xmm6 \n\t"
- "vfmaddps 48(%3,%0,4) , %%xmm7,%%xmm8,%%xmm7 \n\t"
+ "vfmaddps (%4,%0,4) , %%xmm4,%%xmm8,%%xmm4 \n\t"
+ "vfmaddps 16(%4,%0,4) , %%xmm5,%%xmm8,%%xmm5 \n\t"
+ "vfmaddps 32(%4,%0,4) , %%xmm6,%%xmm8,%%xmm6 \n\t"
+ "vfmaddps 48(%4,%0,4) , %%xmm7,%%xmm8,%%xmm7 \n\t"
"addq $16, %0 \n\t"
- "vmovups %%xmm4,-64(%3,%0,4) \n\t" // 4 * y
- "vmovups %%xmm5,-48(%3,%0,4) \n\t" // 4 * y
- "addq $16, %8 \n\t"
- "vmovups %%xmm6,-32(%3,%0,4) \n\t" // 4 * y
- "vmovups %%xmm7,-16(%3,%0,4) \n\t" // 4 * y
+ "vmovups %%xmm4,-64(%4,%0,4) \n\t" // 4 * y
+ "vmovups %%xmm5,-48(%4,%0,4) \n\t" // 4 * y
+ "addq $16, %2 \n\t"
+ "vmovups %%xmm6,-32(%4,%0,4) \n\t" // 4 * y
+ "vmovups %%xmm7,-16(%4,%0,4) \n\t" // 4 * y
"subq $16, %1 \n\t"
"jnz 1b \n\t"
@@ -184,15 +184,15 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO
:
"+r" (i), // 0
- "+r" (n) // 1
+ "+r" (n), // 1
+ "+r" (lda4) // 2
:
- "r" (x), // 2
- "r" (y), // 3
- "r" (ap[0]), // 4
- "r" (ap[1]), // 5
- "r" (ap[2]), // 6
- "r" (ap[3]), // 7
- "r" (lda4), // 8
+ "r" (x), // 3
+ "r" (y), // 4
+ "r" (ap[0]), // 5
+ "r" (ap[1]), // 6
+ "r" (ap[2]), // 7
+ "r" (ap[3]), // 8
"r" (alpha) // 9
: "cc",
"%xmm0", "%xmm1",
From 8242b1fe3f6c3a49b342d99157cd04632267c009 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 16 Feb 2019 18:51:09 +0100
Subject: [PATCH 4/4] Fix inline assembly constraints
---
dgemv_n_microk_piledriver-4.c | 247 ++++++++++++++++++++++++++++++++++
1 file changed, 247 insertions(+)
create mode 100644 dgemv_n_microk_piledriver-4.c
diff --git a/dgemv_n_microk_piledriver-4.c b/dgemv_n_microk_piledriver-4.c
new file mode 100644
index 000000000..466931b82
--- /dev/null
+++ b/dgemv_n_microk_piledriver-4.c
@@ -0,0 +1,247 @@
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+
+#define HAVE_KERNEL_4x8 1
+static void dgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4, FLOAT *alpha) __attribute__ ((noinline));
+
+static void dgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4, FLOAT *alpha)
+{
+
+ BLASLONG register i = 0;
+
+ __asm__ __volatile__
+ (
+ "vzeroupper \n\t"
+ "vbroadcastsd (%3), %%ymm12 \n\t" // x0
+ "vbroadcastsd 8(%3), %%ymm13 \n\t" // x1
+ "vbroadcastsd 16(%3), %%ymm14 \n\t" // x2
+ "vbroadcastsd 24(%3), %%ymm15 \n\t" // x3
+ "vbroadcastsd 32(%3), %%ymm0 \n\t" // x4
+ "vbroadcastsd 40(%3), %%ymm1 \n\t" // x5
+ "vbroadcastsd 48(%3), %%ymm2 \n\t" // x6
+ "vbroadcastsd 56(%3), %%ymm3 \n\t" // x7
+
+ "vbroadcastsd (%9), %%ymm6 \n\t" // alpha
+
+ "testq $0x04, %1 \n\t"
+ "jz 2f \n\t"
+
+ "vmovupd (%4,%0,8), %%ymm7 \n\t" // 4 * y
+ "vxorpd %%ymm4 , %%ymm4, %%ymm4 \n\t"
+ "vxorpd %%ymm5 , %%ymm5, %%ymm5 \n\t"
+
+ "vfmadd231pd (%5,%0,8), %%ymm12, %%ymm4 \n\t"
+ "vfmadd231pd (%6,%0,8), %%ymm13, %%ymm5 \n\t"
+ "vfmadd231pd (%7,%0,8), %%ymm14, %%ymm4 \n\t"
+ "vfmadd231pd (%8,%0,8), %%ymm15, %%ymm5 \n\t"
+
+ "vfmadd231pd (%5,%2,8), %%ymm0 , %%ymm4 \n\t"
+ "vfmadd231pd (%6,%2,8), %%ymm1 , %%ymm5 \n\t"
+ "vfmadd231pd (%7,%2,8), %%ymm2 , %%ymm4 \n\t"
+ "vfmadd231pd (%8,%2,8), %%ymm3 , %%ymm5 \n\t"
+
+ "vaddpd %%ymm4 , %%ymm5 , %%ymm5 \n\t"
+ "vmulpd %%ymm6 , %%ymm5 , %%ymm5 \n\t"
+ "vaddpd %%ymm7 , %%ymm5 , %%ymm5 \n\t"
+
+
+ "vmovupd %%ymm5, (%4,%0,8) \n\t" // 4 * y
+
+ "addq $4 , %2 \n\t"
+ "addq $4 , %0 \n\t"
+ "subq $4 , %1 \n\t"
+
+ "2: \n\t"
+
+ "cmpq $0, %1 \n\t"
+ "je 3f \n\t"
+
+
+ ".align 16 \n\t"
+ "1: \n\t"
+
+ "vxorpd %%ymm4 , %%ymm4, %%ymm4 \n\t"
+ "vxorpd %%ymm5 , %%ymm5, %%ymm5 \n\t"
+ "vmovupd (%4,%0,8), %%ymm8 \n\t" // 4 * y
+ "vmovupd 32(%4,%0,8), %%ymm9 \n\t" // 4 * y
+
+ "vfmadd231pd (%5,%0,8), %%ymm12, %%ymm4 \n\t"
+ "vfmadd231pd 32(%5,%0,8), %%ymm12, %%ymm5 \n\t"
+ "vfmadd231pd (%6,%0,8), %%ymm13, %%ymm4 \n\t"
+ "vfmadd231pd 32(%6,%0,8), %%ymm13, %%ymm5 \n\t"
+ "vfmadd231pd (%7,%0,8), %%ymm14, %%ymm4 \n\t"
+ "vfmadd231pd 32(%7,%0,8), %%ymm14, %%ymm5 \n\t"
+ "vfmadd231pd (%8,%0,8), %%ymm15, %%ymm4 \n\t"
+ "vfmadd231pd 32(%8,%0,8), %%ymm15, %%ymm5 \n\t"
+
+ "vfmadd231pd (%5,%2,8), %%ymm0 , %%ymm4 \n\t"
+ "addq $8 , %0 \n\t"
+ "vfmadd231pd 32(%5,%2,8), %%ymm0 , %%ymm5 \n\t"
+ "vfmadd231pd (%6,%2,8), %%ymm1 , %%ymm4 \n\t"
+ "vfmadd231pd 32(%6,%2,8), %%ymm1 , %%ymm5 \n\t"
+ "vfmadd231pd (%7,%2,8), %%ymm2 , %%ymm4 \n\t"
+ "vfmadd231pd 32(%7,%2,8), %%ymm2 , %%ymm5 \n\t"
+ "vfmadd231pd (%8,%2,8), %%ymm3 , %%ymm4 \n\t"
+ "vfmadd231pd 32(%8,%2,8), %%ymm3 , %%ymm5 \n\t"
+
+ "vfmadd231pd %%ymm6 , %%ymm4 , %%ymm8 \n\t"
+ "vfmadd231pd %%ymm6 , %%ymm5 , %%ymm9 \n\t"
+
+ "addq $8 , %2 \n\t"
+ "vmovupd %%ymm8,-64(%3,%0,8) \n\t" // 4 * y
+ "subq $8 , %1 \n\t"
+ "vmovupd %%ymm9,-32(%4,%0,8) \n\t" // 4 * y
+
+ "jnz 1b \n\t"
+
+ "3: \n\t"
+ "vzeroupper \n\t"
+
+ :
+ "+r" (i), // 0
+ "+r" (n), // 1
+ "+r" (lda4) // 2
+ :
+ "r" (x), // 3
+ "r" (y), // 4
+ "r" (ap[0]), // 5
+ "r" (ap[1]), // 6
+ "r" (ap[2]), // 7
+ "r" (ap[3]), // 8
+ "r" (alpha) // 9
+ : "cc",
+ "%xmm0", "%xmm1",
+ "%xmm2", "%xmm3",
+ "%xmm4", "%xmm5",
+ "%xmm6", "%xmm7",
+ "%xmm8", "%xmm9",
+ "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+ "memory"
+ );
+
+}
+
+
+
+#define HAVE_KERNEL_4x4 1
+static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));
+
+static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)
+{
+
+ BLASLONG register i = 0;
+
+ __asm__ __volatile__
+ (
+ "vzeroupper \n\t"
+ "vbroadcastsd (%2), %%ymm12 \n\t" // x0
+ "vbroadcastsd 8(%2), %%ymm13 \n\t" // x1
+ "vbroadcastsd 16(%2), %%ymm14 \n\t" // x2
+ "vbroadcastsd 24(%2), %%ymm15 \n\t" // x3
+
+ "vbroadcastsd (%8), %%ymm6 \n\t" // alpha
+
+ "testq $0x04, %1 \n\t"
+ "jz 2f \n\t"
+
+ "vxorpd %%ymm4 , %%ymm4, %%ymm4 \n\t"
+ "vxorpd %%ymm5 , %%ymm5, %%ymm5 \n\t"
+ "vmovupd (%3,%0,8), %%ymm7 \n\t" // 4 * y
+
+ "vfmadd231pd (%4,%0,8), %%ymm12, %%ymm4 \n\t"
+ "vfmadd231pd (%5,%0,8), %%ymm13, %%ymm5 \n\t"
+ "vfmadd231pd (%6,%0,8), %%ymm14, %%ymm4 \n\t"
+ "vfmadd231pd (%7,%0,8), %%ymm15, %%ymm5 \n\t"
+
+ "vaddpd %%ymm4 , %%ymm5 , %%ymm5 \n\t"
+ "vmulpd %%ymm6 , %%ymm5 , %%ymm5 \n\t"
+ "vaddpd %%ymm7 , %%ymm5 , %%ymm5 \n\t"
+
+ "vmovupd %%ymm5, (%3,%0,8) \n\t" // 4 * y
+
+ "addq $4 , %0 \n\t"
+ "subq $4 , %1 \n\t"
+
+ "2: \n\t"
+
+ "cmpq $0, %1 \n\t"
+ "je 3f \n\t"
+
+
+ ".align 16 \n\t"
+ "1: \n\t"
+ "vxorpd %%ymm4 , %%ymm4, %%ymm4 \n\t"
+ "vxorpd %%ymm5 , %%ymm5, %%ymm5 \n\t"
+ "vmovupd (%3,%0,8), %%ymm8 \n\t" // 4 * y
+ "vmovupd 32(%3,%0,8), %%ymm9 \n\t" // 4 * y
+
+ "vfmadd231pd (%4,%0,8), %%ymm12, %%ymm4 \n\t"
+ "vfmadd231pd 32(%4,%0,8), %%ymm12, %%ymm5 \n\t"
+ "vfmadd231pd (%5,%0,8), %%ymm13, %%ymm4 \n\t"
+ "vfmadd231pd 32(%5,%0,8), %%ymm13, %%ymm5 \n\t"
+ "vfmadd231pd (%6,%0,8), %%ymm14, %%ymm4 \n\t"
+ "vfmadd231pd 32(%6,%0,8), %%ymm14, %%ymm5 \n\t"
+ "vfmadd231pd (%7,%0,8), %%ymm15, %%ymm4 \n\t"
+ "vfmadd231pd 32(%7,%0,8), %%ymm15, %%ymm5 \n\t"
+
+ "vfmadd231pd %%ymm6 , %%ymm4 , %%ymm8 \n\t"
+ "vfmadd231pd %%ymm6 , %%ymm5 , %%ymm9 \n\t"
+
+ "vmovupd %%ymm8, (%3,%0,8) \n\t" // 4 * y
+ "vmovupd %%ymm9, 32(%3,%0,8) \n\t" // 4 * y
+
+ "addq $8 , %0 \n\t"
+ "subq $8 , %1 \n\t"
+ "jnz 1b \n\t"
+
+ "3: \n\t"
+ "vzeroupper \n\t"
+
+ :
+ "+r" (i), // 0
+ "+r" (n) // 1
+ :
+ "r" (x), // 2
+ "r" (y), // 3
+ "r" (ap[0]), // 4
+ "r" (ap[1]), // 5
+ "r" (ap[2]), // 6
+ "r" (ap[3]), // 7
+ "r" (alpha) // 8
+ : "cc",
+ "%xmm4", "%xmm5",
+ "%xmm6", "%xmm7",
+ "%xmm8", "%xmm9",
+ "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+ "memory"
+ );
+
+}
+
+
Reference in New Issue View Git Blame Copy Permalink