From b5f2a50fe97c34ef2804c9fbc217ed88327e5a71 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Tue, 10 Feb 2026 17:30:05 +0000
Subject: [PATCH 01/12] Added ability to accumulate in FP16 for GEMM.  Widens
 once at the end of loops.

---
 kernel/riscv64/shgemm_kernel_16x8_zvl256b.c | 472 +++++++++++++++++++-
 1 file changed, 462 insertions(+), 10 deletions(-)

diff --git a/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
index 746e07d55f..217b6dbfca 100644
--- a/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
@@ -1,6 +1,8 @@
-
 #include "common.h"
 #include <riscv_vector.h>
+
+#define FP16_NARROW
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -28,7 +30,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 16;
-            
+
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+            vfloat16m1_t result1 = __riscv_vfmul_vf_f16m1( A0, B1, gvl);
+            vfloat16m1_t result2 = __riscv_vfmul_vf_f16m1( A0, B2, gvl);
+            vfloat16m1_t result3 = __riscv_vfmul_vf_f16m1( A0, B3, gvl);
+            vfloat16m1_t result4 = __riscv_vfmul_vf_f16m1( A0, B4, gvl);
+            vfloat16m1_t result5 = __riscv_vfmul_vf_f16m1( A0, B5, gvl);
+            vfloat16m1_t result6 = __riscv_vfmul_vf_f16m1( A0, B6, gvl);
+            vfloat16m1_t result7 = __riscv_vfmul_vf_f16m1( A0, B7, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
             vfloat32m2_t result1 = __riscv_vfwmul_vf_f32m2( A0, B1, gvl);
             vfloat32m2_t result2 = __riscv_vfwmul_vf_f32m2( A0, B2, gvl);
@@ -37,6 +49,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t result5 = __riscv_vfwmul_vf_f32m2( A0, B5, gvl);
             vfloat32m2_t result6 = __riscv_vfwmul_vf_f32m2( A0, B6, gvl);
             vfloat32m2_t result7 = __riscv_vfwmul_vf_f32m2( A0, B7, gvl);
+#endif
             
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -50,7 +63,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 bi += 8;
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 16;
-                
+
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16m1(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f16m1(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f16m1(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f16m1(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f16m1(result7, B7, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m2(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m2(result2, B2, A0, gvl);
@@ -59,6 +82,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmacc_vf_f32m2(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmacc_vf_f32m2(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmacc_vf_f32m2(result7, B7, A0, gvl);
+#endif
             }
             
             BLASLONG ci=n_top*ldc+m_top;
@@ -71,7 +95,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c5 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c6 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c7 = __riscv_vle32_v_f32m2( &C[ci], gvl);
-            
+
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m2(c4, alpha, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m2(c5, alpha, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m2(c6, alpha, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m2(c7, alpha, result7, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m2(c2, alpha, result2, gvl);
@@ -80,6 +114,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             c5 = __riscv_vfmacc_vf_f32m2(c5, alpha, result5, gvl);
             c6 = __riscv_vfmacc_vf_f32m2(c6, alpha, result6, gvl);
             c7 = __riscv_vfmacc_vf_f32m2(c7, alpha, result7, gvl);
+#endif
 
             ci=n_top*ldc+m_top;
 
@@ -116,6 +151,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+            vfloat16mf2_t result2 = __riscv_vfmul_vf_f16mf2( A0, B2, gvl);
+            vfloat16mf2_t result3 = __riscv_vfmul_vf_f16mf2( A0, B3, gvl);
+            vfloat16mf2_t result4 = __riscv_vfmul_vf_f16mf2( A0, B4, gvl);
+            vfloat16mf2_t result5 = __riscv_vfmul_vf_f16mf2( A0, B5, gvl);
+            vfloat16mf2_t result6 = __riscv_vfmul_vf_f16mf2( A0, B6, gvl);
+            vfloat16mf2_t result7 = __riscv_vfmul_vf_f16mf2( A0, B7, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
             vfloat32m1_t result2 = __riscv_vfwmul_vf_f32m1( A0, B2, gvl);
@@ -124,6 +169,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result5 = __riscv_vfwmul_vf_f32m1( A0, B5, gvl);
             vfloat32m1_t result6 = __riscv_vfwmul_vf_f32m1( A0, B6, gvl);
             vfloat32m1_t result7 = __riscv_vfwmul_vf_f32m1( A0, B7, gvl);
+#endif
 
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -138,8 +184,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 8;
-                
-		        result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
+
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16mf2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16mf2(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f16mf2(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f16mf2(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f16mf2(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f16mf2(result7, B7, A0, gvl);
+#else
+                result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmacc_vf_f32m1(result3, B3, A0, gvl);
@@ -147,6 +203,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmacc_vf_f32m1(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmacc_vf_f32m1(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmacc_vf_f32m1(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -159,7 +216,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c5 = __riscv_vle32_v_f32m1( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m1_t c6 = __riscv_vle32_v_f32m1( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m1_t c7 = __riscv_vle32_v_f32m1( &C[ci], gvl);
-            
+
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha, result7, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m1(c2, alpha, result2, gvl);
@@ -168,6 +235,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             c5 = __riscv_vfmacc_vf_f32m1(c5, alpha, result5, gvl);
             c6 = __riscv_vfmacc_vf_f32m1(c6, alpha, result6, gvl);
             c7 = __riscv_vfmacc_vf_f32m1(c7, alpha, result7, gvl);
+#endif
 
             ci=n_top*ldc+m_top;
 
@@ -201,6 +269,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+            vfloat16mf2_t result2 = __riscv_vfmul_vf_f16mf2( A0, B2, gvl);
+            vfloat16mf2_t result3 = __riscv_vfmul_vf_f16mf2( A0, B3, gvl);
+            vfloat16mf2_t result4 = __riscv_vfmul_vf_f16mf2( A0, B4, gvl);
+            vfloat16mf2_t result5 = __riscv_vfmul_vf_f16mf2( A0, B5, gvl);
+            vfloat16mf2_t result6 = __riscv_vfmul_vf_f16mf2( A0, B6, gvl);
+            vfloat16mf2_t result7 = __riscv_vfmul_vf_f16mf2( A0, B7, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
             vfloat32m1_t result2 = __riscv_vfwmul_vf_f32m1( A0, B2, gvl);
@@ -209,6 +287,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result5 = __riscv_vfwmul_vf_f32m1( A0, B5, gvl);
             vfloat32m1_t result6 = __riscv_vfwmul_vf_f32m1( A0, B6, gvl);
             vfloat32m1_t result7 = __riscv_vfwmul_vf_f32m1( A0, B7, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -224,6 +303,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16mf2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16mf2(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f16mf2(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f16mf2(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f16mf2(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f16mf2(result7, B7, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m1(result2, B2, A0, gvl);
@@ -232,6 +321,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmacc_vf_f32m1(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmacc_vf_f32m1(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmacc_vf_f32m1(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -251,6 +341,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c6 = __riscv_vle32_v_f32m1(&C[ci], gvl);
             ci += ldc - gvl * 0;
             vfloat32m1_t c7 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha, result7, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m1(c2, alpha, result2, gvl);
@@ -259,6 +360,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             c5 = __riscv_vfmacc_vf_f32m1(c5, alpha, result5, gvl);
             c6 = __riscv_vfmacc_vf_f32m1(c6, alpha, result6, gvl);
             c7 = __riscv_vfmacc_vf_f32m1(c7, alpha, result7, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -275,6 +377,24 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         }
 
         if( M & 2 ) {
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+            _Float16 result4 = 0;
+            _Float16 result5 = 0;
+            _Float16 result6 = 0;
+            _Float16 result7 = 0;
+            _Float16 result8 = 0;
+            _Float16 result9 = 0;
+            _Float16 result10 = 0;
+            _Float16 result11 = 0;
+            _Float16 result12 = 0;
+            _Float16 result13 = 0;
+            _Float16 result14 = 0;
+            _Float16 result15 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
@@ -291,10 +411,29 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result13 = 0;
             float result14 = 0;
             float result15 = 0;
+#endif
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
             
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+                result2+=A[ai+0]*B[bi+1];
+                result3+=A[ai+1]*B[bi+1];
+                result4+=A[ai+0]*B[bi+2];
+                result5+=A[ai+1]*B[bi+2];
+                result6+=A[ai+0]*B[bi+3];
+                result7+=A[ai+1]*B[bi+3];
+                result8+=A[ai+0]*B[bi+4];
+                result9+=A[ai+1]*B[bi+4];
+                result10+=A[ai+0]*B[bi+5];
+                result11+=A[ai+1]*B[bi+5];
+                result12+=A[ai+0]*B[bi+6];
+                result13+=A[ai+1]*B[bi+6];
+                result14+=A[ai+0]*B[bi+7];
+                result15+=A[ai+1]*B[bi+7];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+1]);
@@ -311,12 +450,31 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result13+=(float)(A[ai+1])*(float)(B[bi+6]);
                 result14+=(float)(A[ai+0])*(float)(B[bi+7]);
                 result15+=(float)(A[ai+1])*(float)(B[bi+7]);
+#endif
                 ai+=2;
                 bi+=8;
             }
 
             BLASLONG ci=n_top*ldc+m_top;
 
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
+            C[ci + 4 * ldc + 0] += alpha * (float)(result8);
+            C[ci + 4 * ldc + 1] += alpha * (float)(result9);
+            C[ci + 5 * ldc + 0] += alpha * (float)(result10);
+            C[ci + 5 * ldc + 1] += alpha * (float)(result11);
+            C[ci + 6 * ldc + 0] += alpha * (float)(result12);
+            C[ci + 6 * ldc + 1] += alpha * (float)(result13);
+            C[ci + 7 * ldc + 0] += alpha * (float)(result14);
+            C[ci + 7 * ldc + 1] += alpha * (float)(result15);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
             C[ci + 1 * ldc + 0] += alpha * result2;
@@ -333,6 +491,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             C[ci + 6 * ldc + 1] += alpha * result13;
             C[ci + 7 * ldc + 0] += alpha * result14;
             C[ci + 7 * ldc + 1] += alpha * result15;
+#endif
 
             m_top+=2;
         }
@@ -340,6 +499,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+            _Float16 result4 = 0;
+            _Float16 result5 = 0;
+            _Float16 result6 = 0;
+            _Float16 result7 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
@@ -348,11 +517,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result5 = 0;
             float result6 = 0;
             float result7 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+0]*B[bi+1];
+                result2+=A[ai+0]*B[bi+2];
+                result3+=A[ai+0]*B[bi+3];
+                result4+=A[ai+0]*B[bi+4];
+                result5+=A[ai+0]*B[bi+5];
+                result6+=A[ai+0]*B[bi+6];
+                result7+=A[ai+0]*B[bi+7];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+0])*(float)(B[bi+1]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+2]);
@@ -361,11 +541,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5+=(float)(A[ai+0])*(float)(B[bi+5]);
                 result6+=(float)(A[ai+0])*(float)(B[bi+6]);
                 result7+=(float)(A[ai+0])*(float)(B[bi+7]);
+#endif
                 ai+=1;
                 bi+=8;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
+            C[ci + 4 * ldc + 0] += alpha * (float)(result4);
+            C[ci + 5 * ldc + 0] += alpha * (float)(result5);
+            C[ci + 6 * ldc + 0] += alpha * (float)(result6);
+            C[ci + 7 * ldc + 0] += alpha * (float)(result7);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
             C[ci + 2 * ldc + 0] += alpha * result2;
@@ -374,6 +565,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             C[ci + 5 * ldc + 0] += alpha * result5;
             C[ci + 6 * ldc + 0] += alpha * result6;
             C[ci + 7 * ldc + 0] += alpha * result7;
+#endif
             m_top+=1;
         }
         n_top += 8;
@@ -396,11 +588,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 16;
-            
+
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+            vfloat16m1_t result1 = __riscv_vfmul_vf_f16m1( A0, B1, gvl);
+            vfloat16m1_t result2 = __riscv_vfmul_vf_f16m1( A0, B2, gvl);
+            vfloat16m1_t result3 = __riscv_vfmul_vf_f16m1( A0, B3, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
             vfloat32m2_t result1 = __riscv_vfwmul_vf_f32m2( A0, B1, gvl);
             vfloat32m2_t result2 = __riscv_vfwmul_vf_f32m2( A0, B2, gvl);
             vfloat32m2_t result3 = __riscv_vfwmul_vf_f32m2( A0, B3, gvl);
+#endif
             
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -412,10 +611,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 16;
                 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16m1(result3, B3, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m2(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m2(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmacc_vf_f32m2(result3, B3, A0, gvl);
+#endif
             }
             
             BLASLONG ci=n_top*ldc+m_top;
@@ -424,11 +630,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c1 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c2 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c3 = __riscv_vle32_v_f32m2( &C[ci], gvl);
-            
+
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m2(c2, alpha, result2, gvl);
             c3 = __riscv_vfmacc_vf_f32m2(c3, alpha, result3, gvl);
+#endif
 
             ci=n_top*ldc+m_top;
 
@@ -453,10 +666,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+            vfloat16mf2_t result2 = __riscv_vfmul_vf_f16mf2( A0, B2, gvl);
+            vfloat16mf2_t result3 = __riscv_vfmul_vf_f16mf2( A0, B3, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
             vfloat32m1_t result2 = __riscv_vfwmul_vf_f32m1( A0, B2, gvl);
             vfloat32m1_t result3 = __riscv_vfwmul_vf_f32m1( A0, B3, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -468,10 +688,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 8;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16mf2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16mf2(result3, B3, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmacc_vf_f32m1(result3, B3, A0, gvl);
+#endif
             }
             
 
@@ -482,10 +709,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c2 = __riscv_vle32_v_f32m1( &C[ci], gvl); ci += ldc - gvl * 0;
             vfloat32m1_t c3 = __riscv_vle32_v_f32m1( &C[ci], gvl);
             
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m1(c2, alpha, result2, gvl);
             c3 = __riscv_vfmacc_vf_f32m1(c3, alpha, result3, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -510,10 +744,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+            vfloat16mf2_t result2 = __riscv_vfmul_vf_f16mf2( A0, B2, gvl);
+            vfloat16mf2_t result3 = __riscv_vfmul_vf_f16mf2( A0, B3, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
             vfloat32m1_t result2 = __riscv_vfwmul_vf_f32m1( A0, B2, gvl);
             vfloat32m1_t result3 = __riscv_vfwmul_vf_f32m1( A0, B3, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -525,10 +766,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16mf2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16mf2(result3, B3, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmacc_vf_f32m1(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -540,10 +788,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c2 = __riscv_vle32_v_f32m1(&C[ci], gvl);
             ci += ldc - gvl * 0;
             vfloat32m1_t c3 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m1(c2, alpha, result2, gvl);
             c3 = __riscv_vfmacc_vf_f32m1(c3, alpha, result3, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -557,6 +812,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 
         if( M & 2 ) {
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+            _Float16 result4 = 0;
+            _Float16 result5 = 0;
+            _Float16 result6 = 0;
+            _Float16 result7 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
@@ -565,10 +830,21 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result5 = 0;
             float result6 = 0;
             float result7 = 0;
+#endif
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+                result2+=A[ai+0]*B[bi+1];
+                result3+=A[ai+1]*B[bi+1];
+                result4+=A[ai+0]*B[bi+2];
+                result5+=A[ai+1]*B[bi+2];
+                result6+=A[ai+0]*B[bi+3];
+                result7+=A[ai+1]*B[bi+3];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+1]);
@@ -577,12 +853,23 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5+=(float)(A[ai+1])*(float)(B[bi+2]);
                 result6+=(float)(A[ai+0])*(float)(B[bi+3]);
                 result7+=(float)(A[ai+1])*(float)(B[bi+3]);
+#endif
                 ai+=2;
                 bi+=4;
             }
             
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
             C[ci + 1 * ldc + 0] += alpha * result2;
@@ -591,6 +878,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             C[ci + 2 * ldc + 1] += alpha * result5;
             C[ci + 3 * ldc + 0] += alpha * result6;
             C[ci + 3 * ldc + 1] += alpha * result7;
+#endif
 
             m_top += 2;
         }
@@ -598,28 +886,49 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
             float result3 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+0]*B[bi+1];
+                result2+=A[ai+0]*B[bi+2];
+                result3+=A[ai+0]*B[bi+3];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+0])*(float)(B[bi+1]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+2]);
                 result3+=(float)(A[ai+0])*(float)(B[bi+3]);
+#endif
                 ai+=1;
                 bi+=4;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
             C[ci + 2 * ldc + 0] += alpha * result2;
             C[ci + 3 * ldc + 0] += alpha * result3;
+#endif
             m_top += 1;
         }
 
@@ -645,8 +954,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 16;
             
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+            vfloat16m1_t result1 = __riscv_vfmul_vf_f16m1( A0, B1, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
             vfloat32m2_t result1 = __riscv_vfwmul_vf_f32m2( A0, B1, gvl);
+#endif
             
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -656,16 +970,26 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 16;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m1(result1, B1, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m2(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
 
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c1 = __riscv_vle32_v_f32m2( &C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
+#endif
 
             ci=n_top*ldc+m_top;
 
@@ -686,8 +1010,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -697,8 +1026,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 8;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
+#endif
             }
 
 
@@ -706,9 +1040,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1( &C[ci], gvl); ci += ldc - gvl * 0;
             vfloat32m1_t c1 = __riscv_vle32_v_f32m1( &C[ci], gvl); 
-            
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -729,8 +1067,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -740,8 +1083,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -749,8 +1097,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1(&C[ci], gvl);
             ci += ldc - gvl * 0;
             vfloat32m1_t c1 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -762,27 +1115,48 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 
         if( M & 2 ) {
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
             float result3 = 0;
+#endif
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+                result2+=A[ai+0]*B[bi+1];
+                result3+=A[ai+1]*B[bi+1];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+1]);
                 result3+=(float)(A[ai+1])*(float)(B[bi+1]);
+#endif
                 ai+=2;
                 bi+=2;
             }
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
             C[ci + 1 * ldc + 0] += alpha * result2;
             C[ci + 1 * ldc + 1] += alpha * result3;
+#endif
 
             m_top += 2;
         }
@@ -790,22 +1164,37 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+0]*B[bi+1];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+0])*(float)(B[bi+1]);
+#endif
                 ai+=1;
                 bi+=2;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
+#endif
             m_top += 1;
         }
 
@@ -828,7 +1217,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 16;
 
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
+#endif
 
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -837,14 +1230,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 16;
                 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
+#endif
             }
             
             BLASLONG ci=n_top*ldc+m_top;
 
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl);
             
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
+#endif
 
             ci=n_top*ldc+m_top;
 
@@ -863,7 +1264,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -872,7 +1277,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 8;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
 
@@ -880,7 +1289,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1( &C[ci], gvl);
             
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -899,7 +1312,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -908,13 +1325,21 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2( &A[ai+0*gvl], gvl );
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
 
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -924,22 +1349,37 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 
         if( M & 2 ) {
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
+#endif
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
+#endif
                 ai+=2;
                 bi+=1;
             }
             
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
+#endif
 
             m_top += 2;
         }
@@ -947,19 +1387,31 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+#else
             float result0 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
+#endif
                 ai+=1;
                 bi+=1;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
+#endif
             m_top += 1;
         }
 

From aa1cebd45b89f48e0d76021f47d3095e8893ba13 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Tue, 10 Feb 2026 18:30:02 +0000
Subject: [PATCH 02/12] 128-bit versions.

---
 kernel/riscv64/shgemm_kernel_8x8_zvl128b.c | 372 ++++++++++++++++++++-
 1 file changed, 370 insertions(+), 2 deletions(-)

diff --git a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
index 3197f7c005..9e720fd844 100644
--- a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
@@ -1,4 +1,3 @@
-
 #include "common.h"
 #include <riscv_vector.h>
 
@@ -30,6 +29,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+            vfloat16m1_t result1 = __riscv_vfmul_vf_f16m1( A0, B1, gvl);
+            vfloat16m1_t result2 = __riscv_vfmul_vf_f16m1( A0, B2, gvl);
+            vfloat16m1_t result3 = __riscv_vfmul_vf_f16m1( A0, B3, gvl);
+            vfloat16m1_t result4 = __riscv_vfmul_vf_f16m1( A0, B4, gvl);
+            vfloat16m1_t result5 = __riscv_vfmul_vf_f16m1( A0, B5, gvl);
+            vfloat16m1_t result6 = __riscv_vfmul_vf_f16m1( A0, B6, gvl);
+            vfloat16m1_t result7 = __riscv_vfmul_vf_f16m1( A0, B7, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
             vfloat32m2_t result1 = __riscv_vfwmul_vf_f32m2( A0, B1, gvl);
             vfloat32m2_t result2 = __riscv_vfwmul_vf_f32m2( A0, B2, gvl);
@@ -38,6 +47,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t result5 = __riscv_vfwmul_vf_f32m2( A0, B5, gvl);
             vfloat32m2_t result6 = __riscv_vfwmul_vf_f32m2( A0, B6, gvl);
             vfloat32m2_t result7 = __riscv_vfwmul_vf_f32m2( A0, B7, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -54,6 +64,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 ai += 8;
 
                 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16m1(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f16m1(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f16m1(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f16m1(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f16m1(result7, B7, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m2(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m2(result2, B2, A0, gvl);
@@ -62,6 +82,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmacc_vf_f32m2(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmacc_vf_f32m2(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmacc_vf_f32m2(result7, B7, A0, gvl);
+#endif
             }
             
 
@@ -75,7 +96,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c5 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c6 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c7 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
-            
+
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m2(c4, alpha, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m2(c5, alpha, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m2(c6, alpha, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m2(c7, alpha, result7, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m2(c2, alpha, result2, gvl);
@@ -84,6 +115,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             c5 = __riscv_vfmacc_vf_f32m2(c5, alpha, result5, gvl);
             c6 = __riscv_vfmacc_vf_f32m2(c6, alpha, result6, gvl);
             c7 = __riscv_vfmacc_vf_f32m2(c7, alpha, result7, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -118,6 +150,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+            vfloat16mf2_t result2 = __riscv_vfmul_vf_f16mf2( A0, B2, gvl);
+            vfloat16mf2_t result3 = __riscv_vfmul_vf_f16mf2( A0, B3, gvl);
+            vfloat16mf2_t result4 = __riscv_vfmul_vf_f16mf2( A0, B4, gvl);
+            vfloat16mf2_t result5 = __riscv_vfmul_vf_f16mf2( A0, B5, gvl);
+            vfloat16mf2_t result6 = __riscv_vfmul_vf_f16mf2( A0, B6, gvl);
+            vfloat16mf2_t result7 = __riscv_vfmul_vf_f16mf2( A0, B7, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
             vfloat32m1_t result2 = __riscv_vfwmul_vf_f32m1( A0, B2, gvl);
@@ -126,6 +168,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result5 = __riscv_vfwmul_vf_f32m1( A0, B5, gvl);
             vfloat32m1_t result6 = __riscv_vfwmul_vf_f32m1( A0, B6, gvl);
             vfloat32m1_t result7 = __riscv_vfwmul_vf_f32m1( A0, B7, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -141,6 +184,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16mf2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16mf2(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f16mf2(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f16mf2(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f16mf2(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f16mf2(result7, B7, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m1(result2, B2, A0, gvl);
@@ -149,6 +202,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmacc_vf_f32m1(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmacc_vf_f32m1(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmacc_vf_f32m1(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -168,6 +222,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c6 = __riscv_vle32_v_f32m1(&C[ci], gvl);
             ci += ldc - gvl * 0;
             vfloat32m1_t c7 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha, result7, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m1(c2, alpha, result2, gvl);
@@ -176,6 +240,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             c5 = __riscv_vfmacc_vf_f32m1(c5, alpha, result5, gvl);
             c6 = __riscv_vfmacc_vf_f32m1(c6, alpha, result6, gvl);
             c7 = __riscv_vfmacc_vf_f32m1(c7, alpha, result7, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -197,6 +262,24 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+            _Float16 result4 = 0;
+            _Float16 result5 = 0;
+            _Float16 result6 = 0;
+            _Float16 result7 = 0;
+            _Float16 result8 = 0;
+            _Float16 result9 = 0;
+            _Float16 result10 = 0;
+            _Float16 result11 = 0;
+            _Float16 result12 = 0;
+            _Float16 result13 = 0;
+            _Float16 result14 = 0;
+            _Float16 result15 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
@@ -213,8 +296,27 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result13 = 0;
             float result14 = 0;
             float result15 = 0;
+#endif
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+                result2+=A[ai+0]*B[bi+1];
+                result3+=A[ai+1]*B[bi+1];
+                result4+=A[ai+0]*B[bi+2];
+                result5+=A[ai+1]*B[bi+2];
+                result6+=A[ai+0]*B[bi+3];
+                result7+=A[ai+1]*B[bi+3];
+                result8+=A[ai+0]*B[bi+4];
+                result9+=A[ai+1]*B[bi+4];
+                result10+=A[ai+0]*B[bi+5];
+                result11+=A[ai+1]*B[bi+5];
+                result12+=A[ai+0]*B[bi+6];
+                result13+=A[ai+1]*B[bi+6];
+                result14+=A[ai+0]*B[bi+7];
+                result15+=A[ai+1]*B[bi+7];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+1]);
@@ -231,12 +333,31 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result13+=(float)(A[ai+1])*(float)(B[bi+6]);
                 result14+=(float)(A[ai+0])*(float)(B[bi+7]);
                 result15+=(float)(A[ai+1])*(float)(B[bi+7]);
+#endif
                 ai+=2;
                 bi+=8;
             }
             
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
+            C[ci + 4 * ldc + 0] += alpha * (float)(result8);
+            C[ci + 4 * ldc + 1] += alpha * (float)(result9);
+            C[ci + 5 * ldc + 0] += alpha * (float)(result10);
+            C[ci + 5 * ldc + 1] += alpha * (float)(result11);
+            C[ci + 6 * ldc + 0] += alpha * (float)(result12);
+            C[ci + 6 * ldc + 1] += alpha * (float)(result13);
+            C[ci + 7 * ldc + 0] += alpha * (float)(result14);
+            C[ci + 7 * ldc + 1] += alpha * (float)(result15);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
             C[ci + 1 * ldc + 0] += alpha * result2;
@@ -253,6 +374,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             C[ci + 6 * ldc + 1] += alpha * result13;
             C[ci + 7 * ldc + 0] += alpha * result14;
             C[ci + 7 * ldc + 1] += alpha * result15;
+#endif
 
             m_top+=2;
         }
@@ -260,6 +382,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+            _Float16 result4 = 0;
+            _Float16 result5 = 0;
+            _Float16 result6 = 0;
+            _Float16 result7 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
@@ -268,11 +400,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result5 = 0;
             float result6 = 0;
             float result7 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+0]*B[bi+1];
+                result2+=A[ai+0]*B[bi+2];
+                result3+=A[ai+0]*B[bi+3];
+                result4+=A[ai+0]*B[bi+4];
+                result5+=A[ai+0]*B[bi+5];
+                result6+=A[ai+0]*B[bi+6];
+                result7+=A[ai+0]*B[bi+7];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+0])*(float)(B[bi+1]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+2]);
@@ -281,11 +424,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5+=(float)(A[ai+0])*(float)(B[bi+5]);
                 result6+=(float)(A[ai+0])*(float)(B[bi+6]);
                 result7+=(float)(A[ai+0])*(float)(B[bi+7]);
+#endif
                 ai+=1;
                 bi+=8;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
+            C[ci + 4 * ldc + 0] += alpha * (float)(result4);
+            C[ci + 5 * ldc + 0] += alpha * (float)(result5);
+            C[ci + 6 * ldc + 0] += alpha * (float)(result6);
+            C[ci + 7 * ldc + 0] += alpha * (float)(result7);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
             C[ci + 2 * ldc + 0] += alpha * result2;
@@ -294,6 +448,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             C[ci + 5 * ldc + 0] += alpha * result5;
             C[ci + 6 * ldc + 0] += alpha * result6;
             C[ci + 7 * ldc + 0] += alpha * result7;
+#endif
             m_top+=1;
         }
 
@@ -318,10 +473,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+            vfloat16m1_t result1 = __riscv_vfmul_vf_f16m1( A0, B1, gvl);
+            vfloat16m1_t result2 = __riscv_vfmul_vf_f16m1( A0, B2, gvl);
+            vfloat16m1_t result3 = __riscv_vfmul_vf_f16m1( A0, B3, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
             vfloat32m2_t result1 = __riscv_vfwmul_vf_f32m2( A0, B1, gvl);
             vfloat32m2_t result2 = __riscv_vfwmul_vf_f32m2( A0, B2, gvl);
             vfloat32m2_t result3 = __riscv_vfwmul_vf_f32m2( A0, B3, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -333,10 +495,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 8;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16m1(result3, B3, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m2(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m2(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmacc_vf_f32m2(result3, B3, A0, gvl);
+#endif
             }
             
 
@@ -347,10 +516,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c2 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc - gvl * 0;
             vfloat32m2_t c3 = __riscv_vle32_v_f32m2( &C[ci], gvl);
             
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m2(c2, alpha, result2, gvl);
             c3 = __riscv_vfmacc_vf_f32m2(c3, alpha, result3, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -375,10 +551,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+            vfloat16mf2_t result2 = __riscv_vfmul_vf_f16mf2( A0, B2, gvl);
+            vfloat16mf2_t result3 = __riscv_vfmul_vf_f16mf2( A0, B3, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
             vfloat32m1_t result2 = __riscv_vfwmul_vf_f32m1( A0, B2, gvl);
             vfloat32m1_t result3 = __riscv_vfwmul_vf_f32m1( A0, B3, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -390,10 +573,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16mf2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16mf2(result3, B3, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmacc_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmacc_vf_f32m1(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -405,10 +595,17 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c2 = __riscv_vle32_v_f32m1(&C[ci], gvl);
             ci += ldc - gvl * 0;
             vfloat32m1_t c3 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
             c2 = __riscv_vfmacc_vf_f32m1(c2, alpha, result2, gvl);
             c3 = __riscv_vfmacc_vf_f32m1(c3, alpha, result3, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -426,6 +623,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+            _Float16 result4 = 0;
+            _Float16 result5 = 0;
+            _Float16 result6 = 0;
+            _Float16 result7 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
@@ -434,8 +641,19 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result5 = 0;
             float result6 = 0;
             float result7 = 0;
+#endif
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+                result2+=A[ai+0]*B[bi+1];
+                result3+=A[ai+1]*B[bi+1];
+                result4+=A[ai+0]*B[bi+2];
+                result5+=A[ai+1]*B[bi+2];
+                result6+=A[ai+0]*B[bi+3];
+                result7+=A[ai+1]*B[bi+3];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+1]);
@@ -444,12 +662,23 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5+=(float)(A[ai+1])*(float)(B[bi+2]);
                 result6+=(float)(A[ai+0])*(float)(B[bi+3]);
                 result7+=(float)(A[ai+1])*(float)(B[bi+3]);
+#endif
                 ai+=2;
                 bi+=4;
             }
             
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
             C[ci + 1 * ldc + 0] += alpha * result2;
@@ -458,6 +687,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             C[ci + 2 * ldc + 1] += alpha * result5;
             C[ci + 3 * ldc + 0] += alpha * result6;
             C[ci + 3 * ldc + 1] += alpha * result7;
+#endif
 
             m_top += 2;
         }
@@ -465,28 +695,49 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
             float result3 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+0]*B[bi+1];
+                result2+=A[ai+0]*B[bi+2];
+                result3+=A[ai+0]*B[bi+3];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+0])*(float)(B[bi+1]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+2]);
                 result3+=(float)(A[ai+0])*(float)(B[bi+3]);
+#endif
                 ai+=1;
                 bi+=4;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
             C[ci + 2 * ldc + 0] += alpha * result2;
             C[ci + 3 * ldc + 0] += alpha * result3;
+#endif
             m_top += 1;
         }
 
@@ -511,8 +762,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+            vfloat16m1_t result1 = __riscv_vfmul_vf_f16m1( A0, B1, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
             vfloat32m2_t result1 = __riscv_vfwmul_vf_f32m2( A0, B1, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -522,8 +778,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 8;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m1(result1, B1, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m2(result1, B1, A0, gvl);
+#endif
             }
 
 
@@ -532,8 +793,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc - gvl * 0;
             vfloat32m2_t c1 = __riscv_vle32_v_f32m2( &C[ci], gvl); 
             
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -554,8 +820,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+            vfloat16mf2_t result1 = __riscv_vfmul_vf_f16mf2( A0, B1, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
             vfloat32m1_t result1 = __riscv_vfwmul_vf_f32m1( A0, B1, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -565,8 +836,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16mf2(result1, B1, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmacc_vf_f32m1(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -574,8 +850,13 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1(&C[ci], gvl);
             ci += ldc - gvl * 0;
             vfloat32m1_t c1 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -591,25 +872,46 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+            _Float16 result2 = 0;
+            _Float16 result3 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
             float result2 = 0;
             float result3 = 0;
+#endif
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+                result2+=A[ai+0]*B[bi+1];
+                result3+=A[ai+1]*B[bi+1];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
                 result2+=(float)(A[ai+0])*(float)(B[bi+1]);
                 result3+=(float)(A[ai+1])*(float)(B[bi+1]);
+#endif
                 ai+=2;
                 bi+=2;
             }
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
             C[ci + 1 * ldc + 0] += alpha * result2;
             C[ci + 1 * ldc + 1] += alpha * result3;
+#endif
 
             m_top += 2;
         }
@@ -617,22 +919,37 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+0]*B[bi+1];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+0])*(float)(B[bi+1]);
+#endif
                 ai+=1;
                 bi+=2;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
+#endif
             m_top += 1;
         }
 
@@ -656,7 +973,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16m1_t A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
             ai += 8;
 
+#ifdef FP16_NARROW
+            vfloat16m1_t result0 = __riscv_vfmul_vf_f16m1( A0, B0, gvl);
+#else
             vfloat32m2_t result0 = __riscv_vfwmul_vf_f32m2( A0, B0, gvl);
+#endif
 		
             for(BLASLONG k=1; k<K; k++) {
                 B0 = B[bi+0];
@@ -665,7 +986,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], gvl );
                 ai += 8;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16m1(result0, B0, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m2(result0, B0, A0, gvl);
+#endif
             }
 
 
@@ -673,7 +998,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl);
             
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
+#endif
 
             ci = n_top * ldc + m_top;
 
@@ -692,7 +1021,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat16mf2_t A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
             ai += 4;
 
+#ifdef FP16_NARROW
+            vfloat16mf2_t result0 = __riscv_vfmul_vf_f16mf2( A0, B0, gvl);
+#else
             vfloat32m1_t result0 = __riscv_vfwmul_vf_f32m1( A0, B0, gvl);
+#endif
 
             for(BLASLONG k=1; k < K; ++k) {
                 B0 = B[bi+0];
@@ -701,13 +1034,21 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 A0 = __riscv_vle16_v_f16mf2(&A[ai + 0 * gvl], gvl);
                 ai += 4;
 
+#ifdef FP16_NARROW
+                result0 = __riscv_vfmacc_vf_f16mf2(result0, B0, A0, gvl);
+#else
                 result0 = __riscv_vfwmacc_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
 
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1(&C[ci], gvl);
+#ifdef FP16_NARROW
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+#else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
+#endif
 
             ci= n_top * ldc + m_top;
 
@@ -721,20 +1062,35 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+            _Float16 result1 = 0;
+#else
             float result0 = 0;
             float result1 = 0;
+#endif
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+                result1+=A[ai+1]*B[bi+0];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
                 result1+=(float)(A[ai+1])*(float)(B[bi+0]);
+#endif
                 ai+=2;
                 bi+=1;
             }
             
             
             BLASLONG ci=n_top*ldc+m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
+#endif
 
             m_top += 2;
         }
@@ -742,19 +1098,31 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if( M & 1 ) {
             
+#ifdef FP16_NARROW
+            _Float16 result0 = 0;
+#else
             float result0 = 0;
+#endif
             
             BLASLONG ai = m_top * K;
             BLASLONG bi = n_top * K;
 
             for(BLASLONG k=0; k<K; k++) {
+#ifdef FP16_NARROW
+                result0+=A[ai+0]*B[bi+0];
+#else
                 result0+=(float)(A[ai+0])*(float)(B[bi+0]);
+#endif
                 ai+=1;
                 bi+=1;
             }
 
             BLASLONG ci = n_top * ldc + m_top;
+#ifdef FP16_NARROW
+            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+#else
             C[ci + 0 * ldc + 0] += alpha * result0;
+#endif
             m_top += 1;
         }
 

From 74d9fe28320f2eb79faa4c22e5f0c6511e635593 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Tue, 10 Feb 2026 19:00:26 +0000
Subject: [PATCH 03/12] Forget to add defintion.

---
 kernel/riscv64/shgemm_kernel_8x8_zvl128b.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
index 9e720fd844..1173a4a0fe 100644
--- a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
@@ -1,6 +1,8 @@
 #include "common.h"
 #include <riscv_vector.h>
 
+#define FP16_NARROW
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;

From e3cb067bf482a0cea305d09ffc388f9da969bc93 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Wed, 11 Feb 2026 00:27:27 +0000
Subject: [PATCH 04/12] Fixed MADD to use float16 values.  Use LMUL = 2 in main
 loop.  Now 1.85X faster on BananaPi.

---
 kernel/riscv64/shgemm_kernel_16x8_zvl256b.c | 448 ++++++++++++++++----
 kernel/riscv64/shgemm_kernel_8x8_zvl128b.c  | 153 +++----
 2 files changed, 433 insertions(+), 168 deletions(-)

diff --git a/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
index 217b6dbfca..4ec59f5e02 100644
--- a/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
@@ -8,13 +8,114 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     BLASLONG gvl = 0;
     BLASLONG m_top = 0;
     BLASLONG n_top = 0;
+#ifdef FP16_NARROW
+    IFLOAT alpha16 = (IFLOAT)(alpha);
+#endif
 
     // -- MAIN PASS
     for (BLASLONG j=0; j<N/8; j+=1) {
         m_top = 0;
+#ifdef FP16_NARROW
+        BLASLONG gvl = __riscv_vsetvl_e16m2(32);
+
+        for (BLASLONG i=0; i<M/32; i+=1) {
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+
+            _Float16 B0 = B[bi+0];
+            _Float16 B1 = B[bi+1];
+            _Float16 B2 = B[bi+2];
+            _Float16 B3 = B[bi+3];
+            _Float16 B4 = B[bi+4];
+            _Float16 B5 = B[bi+5];
+            _Float16 B6 = B[bi+6];
+            _Float16 B7 = B[bi+7];
+            bi += 8;
+
+            vfloat16m1_t A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+            vfloat16m1_t A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+            vfloat16m2_t A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+            ai += 16;
+
+            vfloat16m2_t result0 = __riscv_vfmul_vf_f16m2( A0, B0, gvl);
+            vfloat16m2_t result1 = __riscv_vfmul_vf_f16m2( A0, B1, gvl);
+            vfloat16m2_t result2 = __riscv_vfmul_vf_f16m2( A0, B2, gvl);
+            vfloat16m2_t result3 = __riscv_vfmul_vf_f16m2( A0, B3, gvl);
+            vfloat16m2_t result4 = __riscv_vfmul_vf_f16m2( A0, B4, gvl);
+            vfloat16m2_t result5 = __riscv_vfmul_vf_f16m2( A0, B5, gvl);
+            vfloat16m2_t result6 = __riscv_vfmul_vf_f16m2( A0, B6, gvl);
+            vfloat16m2_t result7 = __riscv_vfmul_vf_f16m2( A0, B7, gvl);
+
+            for(BLASLONG k=1; k<K; k++) {
+                B0 = B[bi+0];
+                B1 = B[bi+1];
+                B2 = B[bi+2];
+                B3 = B[bi+3];
+                B4 = B[bi+4];
+                B5 = B[bi+5];
+                B6 = B[bi+6];
+                B7 = B[bi+7];
+                bi += 8;
+                A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+                A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+                A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+                ai += 16;
+
+                result0 = __riscv_vfmacc_vf_f16m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16m2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16m2(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f16m2(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f16m2(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f16m2(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f16m2(result7, B7, A0, gvl);
+            }
+
+            BLASLONG ci=n_top*ldc+m_top;
+
+            vfloat32m4_t c0 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c1 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c2 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c3 = __riscv_vle32_v_f32m4( &C[ci], gvl);
+
+            ci-=ldc*3;
+
+            c0 = __riscv_vfwmacc_vf_f32m4(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m4(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m4(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m4(c3, alpha16, result3, gvl);
+
+            __riscv_vse32_v_f32m4( &C[ci], c0, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c1, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c2, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c3, gvl); ci += ldc-gvl*0;
+
+            vfloat32m4_t c4 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c5 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c6 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c7 = __riscv_vle32_v_f32m4( &C[ci], gvl);
+
+            ci-=ldc*3;
+
+            c4 = __riscv_vfwmacc_vf_f32m4(c4, alpha16, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m4(c5, alpha16, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m4(c6, alpha16, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m4(c7, alpha16, result7, gvl);
+
+            __riscv_vse32_v_f32m4( &C[ci], c4, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c5, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c6, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c7, gvl);
+            m_top += 32;
+        }
+
+        if (M & 16) {
+          gvl = __riscv_vsetvl_e16m1(16);
+#else
         BLASLONG gvl = __riscv_vsetvl_e16m1(16);
 
         for (BLASLONG i=0; i<M/16; i+=1) {
+#endif
             BLASLONG ai=m_top*K;	
             BLASLONG bi=n_top*K;	
 
@@ -97,14 +198,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c7 = __riscv_vle32_v_f32m2( &C[ci], gvl);
 
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
-            c4 = __riscv_vfwmacc_vf_f32m2(c4, alpha, result4, gvl);
-            c5 = __riscv_vfwmacc_vf_f32m2(c5, alpha, result5, gvl);
-            c6 = __riscv_vfwmacc_vf_f32m2(c6, alpha, result6, gvl);
-            c7 = __riscv_vfwmacc_vf_f32m2(c7, alpha, result7, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha16, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m2(c4, alpha16, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m2(c5, alpha16, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m2(c6, alpha16, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m2(c7, alpha16, result7, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
@@ -218,14 +319,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c7 = __riscv_vle32_v_f32m1( &C[ci], gvl);
 
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
-            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha, result4, gvl);
-            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha, result5, gvl);
-            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha, result6, gvl);
-            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha, result7, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha16, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha16, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha16, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha16, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha16, result7, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -343,14 +444,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c7 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
-            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha, result4, gvl);
-            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha, result5, gvl);
-            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha, result6, gvl);
-            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha, result7, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha16, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha16, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha16, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha16, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha16, result7, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -458,22 +559,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             BLASLONG ci=n_top*ldc+m_top;
 
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
-            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
-            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
-            C[ci + 4 * ldc + 0] += alpha * (float)(result8);
-            C[ci + 4 * ldc + 1] += alpha * (float)(result9);
-            C[ci + 5 * ldc + 0] += alpha * (float)(result10);
-            C[ci + 5 * ldc + 1] += alpha * (float)(result11);
-            C[ci + 6 * ldc + 0] += alpha * (float)(result12);
-            C[ci + 6 * ldc + 1] += alpha * (float)(result13);
-            C[ci + 7 * ldc + 0] += alpha * (float)(result14);
-            C[ci + 7 * ldc + 1] += alpha * (float)(result15);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha16 * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha16 * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha16 * (float)(result7);
+            C[ci + 4 * ldc + 0] += alpha16 * (float)(result8);
+            C[ci + 4 * ldc + 1] += alpha16 * (float)(result9);
+            C[ci + 5 * ldc + 0] += alpha16 * (float)(result10);
+            C[ci + 5 * ldc + 1] += alpha16 * (float)(result11);
+            C[ci + 6 * ldc + 0] += alpha16 * (float)(result12);
+            C[ci + 6 * ldc + 1] += alpha16 * (float)(result13);
+            C[ci + 7 * ldc + 0] += alpha16 * (float)(result14);
+            C[ci + 7 * ldc + 1] += alpha16 * (float)(result15);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -548,14 +649,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
-            C[ci + 4 * ldc + 0] += alpha * (float)(result4);
-            C[ci + 5 * ldc + 0] += alpha * (float)(result5);
-            C[ci + 6 * ldc + 0] += alpha * (float)(result6);
-            C[ci + 7 * ldc + 0] += alpha * (float)(result7);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result3);
+            C[ci + 4 * ldc + 0] += alpha16 * (float)(result4);
+            C[ci + 5 * ldc + 0] += alpha16 * (float)(result5);
+            C[ci + 6 * ldc + 0] += alpha16 * (float)(result6);
+            C[ci + 7 * ldc + 0] += alpha16 * (float)(result7);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
@@ -572,10 +673,76 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     }
 
     if( N & 4 ) {
-        gvl = __riscv_vsetvl_e16m1(16);
         m_top = 0;
+#ifdef FP16_NARROW
+        gvl = __riscv_vsetvl_e16m2(32);
+
+        for (BLASLONG i=0; i<M/32; i+=1) {
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+
+            _Float16 B0 = B[bi+0];
+            _Float16 B1 = B[bi+1];
+            _Float16 B2 = B[bi+2];
+            _Float16 B3 = B[bi+3];
+            bi += 4;
+
+            vfloat16m1_t A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+            vfloat16m1_t A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+            vfloat16m2_t A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+            ai += 16;
+
+            vfloat16m2_t result0 = __riscv_vfmul_vf_f16m2( A0, B0, gvl);
+            vfloat16m2_t result1 = __riscv_vfmul_vf_f16m2( A0, B1, gvl);
+            vfloat16m2_t result2 = __riscv_vfmul_vf_f16m2( A0, B2, gvl);
+            vfloat16m2_t result3 = __riscv_vfmul_vf_f16m2( A0, B3, gvl);
+
+            for(BLASLONG k=1; k<K; k++) {
+                B0 = B[bi+0];
+                B1 = B[bi+1];
+                B2 = B[bi+2];
+                B3 = B[bi+3];
+                bi += 4;
+
+                A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+                A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+                A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+                ai += 16;
+
+                result0 = __riscv_vfmacc_vf_f16m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f16m2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f16m2(result3, B3, A0, gvl);
+            }
+
+            BLASLONG ci=n_top*ldc+m_top;
+
+            vfloat32m4_t c0 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c1 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c2 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c3 = __riscv_vle32_v_f32m4( &C[ci], gvl);
+
+            c0 = __riscv_vfwmacc_vf_f32m4(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m4(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m4(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m4(c3, alpha16, result3, gvl);
+
+            ci=n_top*ldc+m_top;
+
+            __riscv_vse32_v_f32m4( &C[ci], c0, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c1, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c2, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c3, gvl);
+            m_top += 32;
+        }
+
+        if (M & 16) {
+          gvl = __riscv_vsetvl_e16m1(16);
+#else
+        gvl = __riscv_vsetvl_e16m1(16);
 
         for (BLASLONG i=0; i<M/16; i+=1) {
+#endif
             BLASLONG ai=m_top*K;	
             BLASLONG bi=n_top*K;	
             
@@ -632,10 +799,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c3 = __riscv_vle32_v_f32m2( &C[ci], gvl);
 
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha16, result3, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
@@ -710,10 +877,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c3 = __riscv_vle32_v_f32m1( &C[ci], gvl);
             
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha16, result3, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -789,10 +956,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             ci += ldc - gvl * 0;
             vfloat32m1_t c3 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha16, result3, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -861,14 +1028,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
-            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
-            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha16 * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha16 * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha16 * (float)(result7);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -919,10 +1086,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result3);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
@@ -939,10 +1106,61 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
     // -- tails for N=2
     if( N & 2 ) {
-        gvl = __riscv_vsetvl_e16m1(16);
         m_top = 0;
+#ifdef FP16_NARROW
+        gvl = __riscv_vsetvl_e16m2(32);
+
+        for (BLASLONG i=0; i<M/32; i+=1) {
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+
+            _Float16 B0 = B[bi+0];
+            _Float16 B1 = B[bi+1];
+            bi += 2;
+
+            vfloat16m1_t A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+            vfloat16m1_t A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+            vfloat16m2_t A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+            ai += 16;
+
+            vfloat16m2_t result0 = __riscv_vfmul_vf_f16m2( A0, B0, gvl);
+            vfloat16m2_t result1 = __riscv_vfmul_vf_f16m2( A0, B1, gvl);
+
+            for(BLASLONG k=1; k<K; k++) {
+                B0 = B[bi+0];
+                B1 = B[bi+1];
+                bi += 2;
+
+                A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+                A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+                A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+                ai += 16;
+
+                result0 = __riscv_vfmacc_vf_f16m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f16m2(result1, B1, A0, gvl);
+            }
+
+            BLASLONG ci=n_top*ldc+m_top;
+
+            vfloat32m4_t c0 = __riscv_vle32_v_f32m4( &C[ci], gvl); ci += ldc-gvl*0;
+            vfloat32m4_t c1 = __riscv_vle32_v_f32m4( &C[ci], gvl);
+            c0 = __riscv_vfwmacc_vf_f32m4(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m4(c1, alpha16, result1, gvl);
+
+            ci=n_top*ldc+m_top;
+
+            __riscv_vse32_v_f32m4( &C[ci], c0, gvl); ci += ldc-gvl*0;
+            __riscv_vse32_v_f32m4( &C[ci], c1, gvl);
+            m_top += 32;
+        }
+
+        if (M & 16) {
+          gvl = __riscv_vsetvl_e16m1(16);
+#else
+        gvl = __riscv_vsetvl_e16m1(16);
 
         for (BLASLONG i=0; i<M/16; i+=1) {
+#endif
             BLASLONG ai=m_top*K;	
             BLASLONG bi=n_top*K;	
             
@@ -984,8 +1202,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
             vfloat32m2_t c1 = __riscv_vle32_v_f32m2( &C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha16, result1, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
@@ -1041,8 +1259,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1( &C[ci], gvl); ci += ldc - gvl * 0;
             vfloat32m1_t c1 = __riscv_vle32_v_f32m1( &C[ci], gvl); 
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -1098,8 +1316,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             ci += ldc - gvl * 0;
             vfloat32m1_t c1 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -1147,10 +1365,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha16 * (float)(result3);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -1189,8 +1407,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result1);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
@@ -1205,10 +1423,54 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
     // -- tails for N=1
     if( N & 1 ) {
-        gvl = __riscv_vsetvl_e16m1(16);
         m_top = 0;
+#ifdef FP16_NARROW
+        gvl = __riscv_vsetvl_e16m2(32);
+
+        for (BLASLONG i=0; i<M/32; i+=1) {
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+            _Float16 B0 = B[bi+0];
+            bi += 1;
+
+            vfloat16m1_t A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+            vfloat16m1_t A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+            vfloat16m2_t A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+            ai += 16;
+
+            vfloat16m2_t result0 = __riscv_vfmul_vf_f16m2( A0, B0, gvl);
+
+            for(BLASLONG k=1; k<K; k++) {
+                B0 = B[bi+0];
+                bi += 1;
+
+                A00 = __riscv_vle16_v_f16m1( &A[ai+0*gvl], 16 );
+                A01 = __riscv_vle16_v_f16m1( &A[ai+0*gvl+16*K], 16 );
+                A0 = __riscv_vcreate_v_f16m1_f16m2(A00, A01);
+                ai += 16;
+
+                result0 = __riscv_vfmacc_vf_f16m2(result0, B0, A0, gvl);
+            }
+
+            BLASLONG ci=n_top*ldc+m_top;
+
+            vfloat32m4_t c0 = __riscv_vle32_v_f32m4( &C[ci], gvl);
+
+            c0 = __riscv_vfwmacc_vf_f32m4(c0, alpha16, result0, gvl);
+
+            ci=n_top*ldc+m_top;
+
+            __riscv_vse32_v_f32m4( &C[ci], c0, gvl);
+            m_top += 32;
+        }
+
+        if (M & 16) {
+          gvl = __riscv_vsetvl_e16m1(16);
+#else
+        gvl = __riscv_vsetvl_e16m1(16);
 
         for (BLASLONG i=0; i<M/16; i+=1) {
+#endif
             BLASLONG ai=m_top*K;	
             BLASLONG bi=n_top*K;	
             _Float16 B0 = B[bi+0];
@@ -1242,7 +1504,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl);
             
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
 #endif
@@ -1290,7 +1552,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1( &C[ci], gvl);
             
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
 #endif
@@ -1336,7 +1598,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
 #endif
@@ -1374,8 +1636,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -1408,7 +1670,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
 #endif
diff --git a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
index 1173a4a0fe..915e1032ad 100644
--- a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
@@ -8,6 +8,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     BLASLONG gvl = 0;
     BLASLONG m_top = 0;
     BLASLONG n_top = 0;
+#ifdef FP16_NARROW
+    IFLOAT alpha16 = (IFLOAT)(alpha);
+#endif
 
     // -- MAIN PASS
     for (BLASLONG j=0; j<N/8; j+=1) {
@@ -100,14 +103,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c7 = __riscv_vle32_v_f32m2( &C[ci], gvl); ci += ldc-gvl*0;
 
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
-            c4 = __riscv_vfwmacc_vf_f32m2(c4, alpha, result4, gvl);
-            c5 = __riscv_vfwmacc_vf_f32m2(c5, alpha, result5, gvl);
-            c6 = __riscv_vfwmacc_vf_f32m2(c6, alpha, result6, gvl);
-            c7 = __riscv_vfwmacc_vf_f32m2(c7, alpha, result7, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha16, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m2(c4, alpha16, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m2(c5, alpha16, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m2(c6, alpha16, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m2(c7, alpha16, result7, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
@@ -225,14 +228,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             ci += ldc - gvl * 0;
             vfloat32m1_t c7 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
-            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha, result4, gvl);
-            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha, result5, gvl);
-            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha, result6, gvl);
-            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha, result7, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha16, result3, gvl);
+            c4 = __riscv_vfwmacc_vf_f32m1(c4, alpha16, result4, gvl);
+            c5 = __riscv_vfwmacc_vf_f32m1(c5, alpha16, result5, gvl);
+            c6 = __riscv_vfwmacc_vf_f32m1(c6, alpha16, result6, gvl);
+            c7 = __riscv_vfwmacc_vf_f32m1(c7, alpha16, result7, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -343,22 +346,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
-            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
-            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
-            C[ci + 4 * ldc + 0] += alpha * (float)(result8);
-            C[ci + 4 * ldc + 1] += alpha * (float)(result9);
-            C[ci + 5 * ldc + 0] += alpha * (float)(result10);
-            C[ci + 5 * ldc + 1] += alpha * (float)(result11);
-            C[ci + 6 * ldc + 0] += alpha * (float)(result12);
-            C[ci + 6 * ldc + 1] += alpha * (float)(result13);
-            C[ci + 7 * ldc + 0] += alpha * (float)(result14);
-            C[ci + 7 * ldc + 1] += alpha * (float)(result15);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha16 * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha16 * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha16 * (float)(result7);
+            C[ci + 4 * ldc + 0] += alpha16 * (float)(result8);
+            C[ci + 4 * ldc + 1] += alpha16 * (float)(result9);
+            C[ci + 5 * ldc + 0] += alpha16 * (float)(result10);
+            C[ci + 5 * ldc + 1] += alpha16 * (float)(result11);
+            C[ci + 6 * ldc + 0] += alpha16 * (float)(result12);
+            C[ci + 6 * ldc + 1] += alpha16 * (float)(result13);
+            C[ci + 7 * ldc + 0] += alpha16 * (float)(result14);
+            C[ci + 7 * ldc + 1] += alpha16 * (float)(result15);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -433,14 +436,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
-            C[ci + 4 * ldc + 0] += alpha * (float)(result4);
-            C[ci + 5 * ldc + 0] += alpha * (float)(result5);
-            C[ci + 6 * ldc + 0] += alpha * (float)(result6);
-            C[ci + 7 * ldc + 0] += alpha * (float)(result7);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result3);
+            C[ci + 4 * ldc + 0] += alpha16 * (float)(result4);
+            C[ci + 5 * ldc + 0] += alpha16 * (float)(result5);
+            C[ci + 6 * ldc + 0] += alpha16 * (float)(result6);
+            C[ci + 7 * ldc + 0] += alpha16 * (float)(result7);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
@@ -519,10 +522,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c3 = __riscv_vle32_v_f32m2( &C[ci], gvl);
             
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha, result3, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m2(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m2(c3, alpha16, result3, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
@@ -598,10 +601,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             ci += ldc - gvl * 0;
             vfloat32m1_t c3 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
-            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha, result2, gvl);
-            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha, result3, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
+            c2 = __riscv_vfwmacc_vf_f32m1(c2, alpha16, result2, gvl);
+            c3 = __riscv_vfwmacc_vf_f32m1(c3, alpha16, result3, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -672,14 +675,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result4);
-            C[ci + 2 * ldc + 1] += alpha * (float)(result5);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result6);
-            C[ci + 3 * ldc + 1] += alpha * (float)(result7);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha16 * (float)(result3);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result4);
+            C[ci + 2 * ldc + 1] += alpha16 * (float)(result5);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result6);
+            C[ci + 3 * ldc + 1] += alpha16 * (float)(result7);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -730,10 +733,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
-            C[ci + 2 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 3 * ldc + 0] += alpha * (float)(result3);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result1);
+            C[ci + 2 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 3 * ldc + 0] += alpha16 * (float)(result3);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
@@ -796,8 +799,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c1 = __riscv_vle32_v_f32m2( &C[ci], gvl); 
             
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha, result1, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m2(c1, alpha16, result1, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m2(c1, alpha, result1, gvl);
@@ -853,8 +856,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             ci += ldc - gvl * 0;
             vfloat32m1_t c1 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
-            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha, result1, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
+            c1 = __riscv_vfwmacc_vf_f32m1(c1, alpha16, result1, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
             c1 = __riscv_vfmacc_vf_f32m1(c1, alpha, result1, gvl);
@@ -904,10 +907,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result2);
-            C[ci + 1 * ldc + 1] += alpha * (float)(result3);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result2);
+            C[ci + 1 * ldc + 1] += alpha16 * (float)(result3);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -946,8 +949,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 1 * ldc + 0] += alpha * (float)(result1);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 1 * ldc + 0] += alpha16 * (float)(result1);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 1 * ldc + 0] += alpha * result1;
@@ -1001,7 +1004,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t c0 = __riscv_vle32_v_f32m2( &C[ci], gvl);
             
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha, result0, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m2(c0, alpha16, result0, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m2(c0, alpha, result0, gvl);
 #endif
@@ -1047,7 +1050,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             vfloat32m1_t c0 = __riscv_vle32_v_f32m1(&C[ci], gvl);
 #ifdef FP16_NARROW
-            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha, result0, gvl);
+            c0 = __riscv_vfwmacc_vf_f32m1(c0, alpha16, result0, gvl);
 #else
             c0 = __riscv_vfmacc_vf_f32m1(c0, alpha, result0, gvl);
 #endif
@@ -1087,8 +1090,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             
             BLASLONG ci=n_top*ldc+m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
-            C[ci + 0 * ldc + 1] += alpha * (float)(result1);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
+            C[ci + 0 * ldc + 1] += alpha16 * (float)(result1);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
             C[ci + 0 * ldc + 1] += alpha * result1;
@@ -1121,7 +1124,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             BLASLONG ci = n_top * ldc + m_top;
 #ifdef FP16_NARROW
-            C[ci + 0 * ldc + 0] += alpha * (float)(result0);
+            C[ci + 0 * ldc + 0] += alpha16 * (float)(result0);
 #else
             C[ci + 0 * ldc + 0] += alpha * result0;
 #endif

From 33560437f563dc664adc17622664124cf6d13e03 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Wed, 11 Feb 2026 19:50:48 +0000
Subject: [PATCH 05/12] Convert inputs from BF16 to FP32 and use FP32 vector
 madds.  18% faster.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 128 ++++++++++++++++++++
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  |  86 +++++++++++++
 2 files changed, 214 insertions(+)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index 6e7b06884d..6a0844017c 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -1,6 +1,8 @@
 #include "common.h"
 #include <riscv_vector.h>
 
+#define BF16_WIDEN_ONE
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -28,6 +30,30 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t result7 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                float B4 = (float)(BB[bi+4]);
+                float B5 = (float)(BB[bi+5]);
+                float B6 = (float)(BB[bi+6]);
+                float B7 = (float)(BB[bi+7]);
+                bi += 8;
+
+                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
+                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
+                ai += 16;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f32m2(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f32m2(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f32m2(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f32m2(result7, B7, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -49,6 +75,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmaccbf16_vf_f32m2(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmaccbf16_vf_f32m2(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmaccbf16_vf_f32m2(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -102,6 +129,30 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result7 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                float B4 = (float)(BB[bi+4]);
+                float B5 = (float)(BB[bi+5]);
+                float B6 = (float)(BB[bi+6]);
+                float B7 = (float)(BB[bi+7]);
+                bi += 8;
+
+                vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
+                vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
+                ai += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m1(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f32m1(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f32m1(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f32m1(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f32m1(result7, B7, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -123,6 +174,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmaccbf16_vf_f32m1(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmaccbf16_vf_f32m1(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmaccbf16_vf_f32m1(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -174,6 +226,30 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result7 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                float B4 = (float)(BB[bi+4]);
+                float B5 = (float)(BB[bi+5]);
+                float B6 = (float)(BB[bi+6]);
+                float B7 = (float)(BB[bi+7]);
+                bi += 8;
+
+                vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
+                ai += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m1(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f32m1(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f32m1(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f32m1(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f32m1(result7, B7, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -195,6 +271,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmaccbf16_vf_f32m1(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmaccbf16_vf_f32m1(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmaccbf16_vf_f32m1(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -356,6 +433,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t result3 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                bi += 4;
+
+                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
+                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
+                ai += 16;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -369,6 +462,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result1 = __riscv_vfwmaccbf16_vf_f32m2(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmaccbf16_vf_f32m2(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmaccbf16_vf_f32m2(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -403,6 +497,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result3 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                bi += 4;
+
+                vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
+                vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
+                ai += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m1(result3, B3, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -416,6 +526,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result1 = __riscv_vfwmaccbf16_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmaccbf16_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmaccbf16_vf_f32m1(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -451,6 +562,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result3 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                bi += 4;
+
+                vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
+                ai += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m1(result3, B3, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -464,6 +591,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result1 = __riscv_vfwmaccbf16_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmaccbf16_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmaccbf16_vf_f32m1(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index 74679f8cbf..0996265290 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -1,6 +1,8 @@
 #include "common.h"
 #include <riscv_vector.h>
 
+#define BF16_WIDEN_ONE
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -28,6 +30,30 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t result7 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                float B4 = (float)(BB[bi+4]);
+                float B5 = (float)(BB[bi+5]);
+                float B6 = (float)(BB[bi+6]);
+                float B7 = (float)(BB[bi+7]);
+                bi += 8;
+
+                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
+                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
+                ai += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f32m2(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f32m2(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f32m2(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f32m2(result7, B7, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -49,6 +75,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmaccbf16_vf_f32m2(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmaccbf16_vf_f32m2(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmaccbf16_vf_f32m2(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -102,6 +129,30 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result7 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                float B4 = (float)(BB[bi+4]);
+                float B5 = (float)(BB[bi+5]);
+                float B6 = (float)(BB[bi+6]);
+                float B7 = (float)(BB[bi+7]);
+                bi += 8;
+
+                vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
+                vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
+                ai += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m1(result3, B3, A0, gvl);
+                result4 = __riscv_vfmacc_vf_f32m1(result4, B4, A0, gvl);
+                result5 = __riscv_vfmacc_vf_f32m1(result5, B5, A0, gvl);
+                result6 = __riscv_vfmacc_vf_f32m1(result6, B6, A0, gvl);
+                result7 = __riscv_vfmacc_vf_f32m1(result7, B7, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -123,6 +174,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result5 = __riscv_vfwmaccbf16_vf_f32m1(result5, B5, A0, gvl);
                 result6 = __riscv_vfwmaccbf16_vf_f32m1(result6, B6, A0, gvl);
                 result7 = __riscv_vfwmaccbf16_vf_f32m1(result7, B7, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -286,6 +338,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m2_t result3 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                bi += 4;
+
+                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
+                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
+                ai += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -299,6 +367,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result1 = __riscv_vfwmaccbf16_vf_f32m2(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmaccbf16_vf_f32m2(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmaccbf16_vf_f32m2(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -334,6 +403,22 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             vfloat32m1_t result3 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = (float)(BB[bi+0]);
+                float B1 = (float)(BB[bi+1]);
+                float B2 = (float)(BB[bi+2]);
+                float B3 = (float)(BB[bi+3]);
+                bi += 4;
+
+                vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
+                ai += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+                result2 = __riscv_vfmacc_vf_f32m1(result2, B2, A0, gvl);
+                result3 = __riscv_vfmacc_vf_f32m1(result3, B3, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 __bf16 B2 = BB[bi+2];
@@ -347,6 +432,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result1 = __riscv_vfwmaccbf16_vf_f32m1(result1, B1, A0, gvl);
                 result2 = __riscv_vfwmaccbf16_vf_f32m1(result2, B2, A0, gvl);
                 result3 = __riscv_vfwmaccbf16_vf_f32m1(result3, B3, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;

From 4121a22c0214e0225f335cc59c4216a26f87acbf Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Thu, 12 Feb 2026 18:45:39 +0000
Subject: [PATCH 06/12] Convert BF16 values once (and vectorized).

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 150 ++++++++++++++------
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 116 +++++++++++----
 2 files changed, 195 insertions(+), 71 deletions(-)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index 6a0844017c..68f8aefcb3 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -11,14 +11,41 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     __bf16 *BB = (__bf16 *)(B);
     __bf16 *AA = (__bf16 *)(A);
 
+#ifdef BF16_WIDEN_ONE
+    FLOAT *B_CONV = NULL;
+    if ((M >= 4) && (N >= 4) && (K > 0)) {
+        B_CONV = (FLOAT *)(malloc(K * 8 * sizeof(FLOAT)));
+        if (!B_CONV) return 1;
+    }
+#endif
+
     // -- MAIN PASS
     for (BLASLONG j=0; j<N/8; j+=1) {
         m_top = 0;
         BLASLONG gvl = __riscv_vsetvl_e16m1(16);
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2;
+        if (B_CONV) {
+            BLASLONG bi3 = 0;
+            BLASLONG gvl2;
+            bi2 = K * 8;
+            do {
+                gvl2 = __riscv_vsetvl_e16m4(bi2);
+                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
+                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
+                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
+                bi3 += gvl2;
+            } while (bi2 -= gvl2);
+        }
+#endif
 
         for (BLASLONG i=0; i<M/16; i+=1) {
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
             vfloat32m2_t result1 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
@@ -31,15 +58,15 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                float B4 = (float)(BB[bi+4]);
-                float B5 = (float)(BB[bi+5]);
-                float B6 = (float)(BB[bi+6]);
-                float B7 = (float)(BB[bi+7]);
-                bi += 8;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                float B4 = B_CONV[bi2+4];
+                float B5 = B_CONV[bi2+5];
+                float B6 = B_CONV[bi2+6];
+                float B7 = B_CONV[bi2+7];
+                bi2 += 8;
 
                 vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
                 vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
@@ -117,7 +144,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             gvl = __riscv_vsetvl_e16mf2(8);
 
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
@@ -130,15 +161,15 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                float B4 = (float)(BB[bi+4]);
-                float B5 = (float)(BB[bi+5]);
-                float B6 = (float)(BB[bi+6]);
-                float B7 = (float)(BB[bi+7]);
-                bi += 8;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                float B4 = B_CONV[bi2+4];
+                float B5 = B_CONV[bi2+5];
+                float B6 = B_CONV[bi2+6];
+                float B7 = B_CONV[bi2+7];
+                bi2 += 8;
 
                 vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
                 vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
@@ -214,7 +245,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             gvl = __riscv_vsetvl_e16mf2(4);
 
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
@@ -227,15 +262,15 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                float B4 = (float)(BB[bi+4]);
-                float B5 = (float)(BB[bi+5]);
-                float B6 = (float)(BB[bi+6]);
-                float B7 = (float)(BB[bi+7]);
-                bi += 8;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                float B4 = B_CONV[bi2+4];
+                float B5 = B_CONV[bi2+5];
+                float B6 = B_CONV[bi2+6];
+                float B7 = B_CONV[bi2+7];
+                bi2 += 8;
 
                 vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
                 vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
@@ -423,9 +458,29 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         gvl = __riscv_vsetvl_e16m1(16);
         m_top = 0;
 
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2;
+        if (B_CONV) {
+            BLASLONG bi3 = 0;
+            BLASLONG gvl2;
+            bi2 = K * 4;
+            do {
+                gvl2 = __riscv_vsetvl_e16m4(bi2);
+                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
+                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
+                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
+                bi3 += gvl2;
+            } while (bi2 -= gvl2);
+        }
+#endif
+
         for (BLASLONG i=0; i<M/16; i+=1) {
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
             vfloat32m2_t result1 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
@@ -434,11 +489,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                bi += 4;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                bi2 += 4;
 
                 vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
                 vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
@@ -489,7 +544,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 8 ) {
             gvl = __riscv_vsetvl_e16mf2(8);
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
@@ -498,11 +557,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                bi += 4;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                bi2 += 4;
 
                 vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
                 vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
@@ -554,7 +613,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             gvl = __riscv_vsetvl_e16mf2(4);
 
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
@@ -563,11 +626,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                bi += 4;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                bi2 += 4;
 
                 vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
                 vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
@@ -977,5 +1040,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         n_top += 1;
     }
+#ifdef BF16_WIDEN_ONE
+    if (B_CONV) free(B_CONV);
+#endif
     return 0;
 }
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index 0996265290..53099bc9ee 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -11,14 +11,42 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     __bf16 *BB = (__bf16 *)(B);
     __bf16 *AA = (__bf16 *)(A);
 
+#ifdef BF16_WIDEN_ONE
+    FLOAT *B_CONV = NULL;
+    if ((M >= 4) && (N >= 4) && (K > 0)) {
+        B_CONV = (FLOAT *)(malloc(K * 8 * sizeof(FLOAT)));
+        if (!B_CONV) return 1;
+    }
+#endif
+
     // -- MAIN PASS
     for (BLASLONG j=0; j<N/8; j+=1) {
         m_top = 0;
         BLASLONG gvl = __riscv_vsetvl_e16m1(8);
 
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2;
+        if (B_CONV) {
+            BLASLONG bi3 = 0;
+            BLASLONG gvl2;
+            bi2 = K * 8;
+            do {
+                gvl2 = __riscv_vsetvl_e16m4(bi2);
+                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
+                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
+                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
+                bi3 += gvl2;
+            } while (bi2 -= gvl2);
+        }
+#endif
+
         for (BLASLONG i=0; i<M/8; i+=1) {
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
             vfloat32m2_t result1 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
@@ -31,15 +59,15 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                float B4 = (float)(BB[bi+4]);
-                float B5 = (float)(BB[bi+5]);
-                float B6 = (float)(BB[bi+6]);
-                float B7 = (float)(BB[bi+7]);
-                bi += 8;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                float B4 = B_CONV[bi2+4];
+                float B5 = B_CONV[bi2+5];
+                float B6 = B_CONV[bi2+6];
+                float B7 = B_CONV[bi2+7];
+                bi2 += 8;
 
                 vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
                 vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
@@ -117,7 +145,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             gvl = __riscv_vsetvl_e16m1(4);
 
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
@@ -130,15 +162,15 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                float B4 = (float)(BB[bi+4]);
-                float B5 = (float)(BB[bi+5]);
-                float B6 = (float)(BB[bi+6]);
-                float B7 = (float)(BB[bi+7]);
-                bi += 8;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                float B4 = B_CONV[bi2+4];
+                float B5 = B_CONV[bi2+5];
+                float B6 = B_CONV[bi2+6];
+                float B7 = B_CONV[bi2+7];
+                bi2 += 8;
 
                 vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
                 vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
@@ -328,9 +360,29 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         gvl = __riscv_vsetvl_e16m1(8);
         m_top = 0;
 
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2;
+        if (B_CONV) {
+            BLASLONG bi3 = 0;
+            BLASLONG gvl2;
+            bi2 = K * 4;
+            do {
+                gvl2 = __riscv_vsetvl_e16m4(bi2);
+                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
+                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
+                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
+                bi3 += gvl2;
+            } while (bi2 -= gvl2);
+        }
+#endif
+
         for (BLASLONG i=0; i<M/8; i+=1) {
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
             vfloat32m2_t result1 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
@@ -339,11 +391,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                bi += 4;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                bi2 += 4;
 
                 vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
                 vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
@@ -395,7 +447,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             gvl = __riscv_vsetvl_e16m1(4);
 
             BLASLONG ai=m_top*K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
@@ -404,11 +460,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = (float)(BB[bi+0]);
-                float B1 = (float)(BB[bi+1]);
-                float B2 = (float)(BB[bi+2]);
-                float B3 = (float)(BB[bi+3]);
-                bi += 4;
+                float B0 = B_CONV[bi2+0];
+                float B1 = B_CONV[bi2+1];
+                float B2 = B_CONV[bi2+2];
+                float B3 = B_CONV[bi2+3];
+                bi2 += 4;
 
                 vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
                 vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
@@ -762,6 +818,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         n_top += 1;
     }
-
+#ifdef BF16_WIDEN_ONE
+    if (B_CONV) free(B_CONV);
+#endif
     return 0;
 }

From 9701a80a9f6d0fe03e0a166e5f4ce44c8d1cb6ef Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Thu, 12 Feb 2026 20:35:41 +0000
Subject: [PATCH 07/12] One small change.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 4 ++--
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index 68f8aefcb3..8aa114a10e 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -25,7 +25,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         BLASLONG gvl = __riscv_vsetvl_e16m1(16);
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2;
-        if (B_CONV) {
+        {
             BLASLONG bi3 = 0;
             BLASLONG gvl2;
             bi2 = K * 8;
@@ -460,7 +460,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2;
-        if (B_CONV) {
+        {
             BLASLONG bi3 = 0;
             BLASLONG gvl2;
             bi2 = K * 4;
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index 53099bc9ee..f251f90608 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -26,7 +26,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2;
-        if (B_CONV) {
+        {
             BLASLONG bi3 = 0;
             BLASLONG gvl2;
             bi2 = K * 8;
@@ -362,7 +362,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2;
-        if (B_CONV) {
+        {
             BLASLONG bi3 = 0;
             BLASLONG gvl2;
             bi2 = K * 4;

From 0acb60aab3c0134e879a68292904d8346dcd50ef Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Fri, 13 Feb 2026 17:55:15 +0000
Subject: [PATCH 08/12] Conversion from BF16 to FP32 only once.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 181 ++++++++++----------
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 143 ++++++++--------
 2 files changed, 164 insertions(+), 160 deletions(-)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index 8aa114a10e..1e2bf369e7 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -3,6 +3,32 @@
 
 #define BF16_WIDEN_ONE
 
+#ifdef BF16_WIDEN_ONE
+#define FORCEINLINE      inline __attribute__((always_inline))
+#define B_UNROLL         64
+
+// Convert from BF16 to FP32
+static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
+{
+    BLASLONG count2 = (count & (B_UNROLL - 1));
+    count &= -B_UNROLL;
+    while (count) {
+        vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, B_UNROLL);
+        vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, B_UNROLL);
+        __riscv_vse32_v_f32m8(CONV, B0, B_UNROLL);
+        BB += B_UNROLL;
+        CONV += B_UNROLL;
+        count -= B_UNROLL;
+    }
+    if (count2) {
+        BLASLONG gvl2 = __riscv_vsetvl_e16m4(count2);
+        vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, gvl2);
+        vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, gvl2);
+        __riscv_vse32_v_f32m8(CONV, B0, gvl2);
+    }
+}
+#endif
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -12,10 +38,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     __bf16 *AA = (__bf16 *)(A);
 
 #ifdef BF16_WIDEN_ONE
-    FLOAT *B_CONV = NULL;
+    FLOAT *CONV = NULL;
     if ((M >= 4) && (N >= 4) && (K > 0)) {
-        B_CONV = (FLOAT *)(malloc(K * 8 * sizeof(FLOAT)));
-        if (!B_CONV) return 1;
+        CONV = (FLOAT *)(malloc((K * (8 + (M & -4))) * sizeof(FLOAT)));
+        if (!CONV) return 1;
+        B_CONV(AA, CONV + (K * 8), (M & -4) * K);
     }
 #endif
 
@@ -24,26 +51,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         m_top = 0;
         BLASLONG gvl = __riscv_vsetvl_e16m1(16);
 #ifdef BF16_WIDEN_ONE
-        BLASLONG bi2;
-        {
-            BLASLONG bi3 = 0;
-            BLASLONG gvl2;
-            bi2 = K * 8;
-            do {
-                gvl2 = __riscv_vsetvl_e16m4(bi2);
-                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
-                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
-                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
-                bi3 += gvl2;
-            } while (bi2 -= gvl2);
-        }
+        BLASLONG bi2 = K * 8;
+        B_CONV(BB + (n_top*K), CONV, bi2);
+        BLASLONG ai2 = K * 8;
 #endif
 
         for (BLASLONG i=0; i<M/16; i+=1) {
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -58,19 +75,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
-                float B4 = B_CONV[bi2+4];
-                float B5 = B_CONV[bi2+5];
-                float B6 = B_CONV[bi2+6];
-                float B7 = B_CONV[bi2+7];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
+                float B4 = CONV[bi2+4];
+                float B5 = CONV[bi2+5];
+                float B6 = CONV[bi2+6];
+                float B7 = CONV[bi2+7];
                 bi2 += 8;
 
-                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
-                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
-                ai += 16;
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+		ai2 += 16;
 
                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
@@ -143,10 +159,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 8 ) {
             gvl = __riscv_vsetvl_e16mf2(8);
 
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -161,19 +177,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
-                float B4 = B_CONV[bi2+4];
-                float B5 = B_CONV[bi2+5];
-                float B6 = B_CONV[bi2+6];
-                float B7 = B_CONV[bi2+7];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
+                float B4 = CONV[bi2+4];
+                float B5 = CONV[bi2+5];
+                float B6 = CONV[bi2+6];
+                float B7 = CONV[bi2+7];
                 bi2 += 8;
 
-                vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
-                vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
-                ai += 8;
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 8;
 
                 result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
@@ -244,10 +259,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16mf2(4);
 
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -262,19 +277,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
-                float B4 = B_CONV[bi2+4];
-                float B5 = B_CONV[bi2+5];
-                float B6 = B_CONV[bi2+6];
-                float B7 = B_CONV[bi2+7];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
+                float B4 = CONV[bi2+4];
+                float B5 = CONV[bi2+5];
+                float B6 = CONV[bi2+6];
+                float B7 = CONV[bi2+7];
                 bi2 += 8;
 
-                vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
-                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
-                ai += 4;
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vle32_v_f32mf2(&CONV[ai2], gvl));
+                ai2 += 4;
 
                 result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
@@ -459,26 +473,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         m_top = 0;
 
 #ifdef BF16_WIDEN_ONE
-        BLASLONG bi2;
-        {
-            BLASLONG bi3 = 0;
-            BLASLONG gvl2;
-            bi2 = K * 4;
-            do {
-                gvl2 = __riscv_vsetvl_e16m4(bi2);
-                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
-                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
-                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
-                bi3 += gvl2;
-            } while (bi2 -= gvl2);
-        }
+        BLASLONG bi2 = K * 4;
+        B_CONV(BB + (n_top*K), CONV, bi2);
+        BLASLONG ai2 = K * 8;
 #endif
 
         for (BLASLONG i=0; i<M/16; i+=1) {
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -489,15 +493,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
                 bi2 += 4;
 
-                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
-                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
-                ai += 16;
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 16;
 
                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
@@ -543,10 +546,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 8 ) {
             gvl = __riscv_vsetvl_e16mf2(8);
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -557,15 +560,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
                 bi2 += 4;
 
-                vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
-                vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
-                ai += 8;
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 8;
 
                 result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
@@ -612,10 +614,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16mf2(4);
 
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -626,15 +628,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
                 bi2 += 4;
 
-                vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
-                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
-                ai += 4;
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vle32_v_f32mf2(&CONV[ai2], gvl));
+                ai2 += 4;
 
                 result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
@@ -1041,7 +1042,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         n_top += 1;
     }
 #ifdef BF16_WIDEN_ONE
-    if (B_CONV) free(B_CONV);
+    if (CONV) free(CONV);
 #endif
     return 0;
 }
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index f251f90608..4caa3241e3 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -3,6 +3,32 @@
 
 #define BF16_WIDEN_ONE
 
+#ifdef BF16_WIDEN_ONE
+#define FORCEINLINE      inline __attribute__((always_inline))
+#define B_UNROLL         32
+
+// Convert from BF16 to FP32
+static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
+{
+    BLASLONG count2 = (count & (B_UNROLL - 1));
+    count &= -B_UNROLL;
+    while (count) {
+        vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, B_UNROLL);
+        vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, B_UNROLL);
+        __riscv_vse32_v_f32m8(CONV, B0, B_UNROLL);
+        BB += B_UNROLL;
+        CONV += B_UNROLL;
+        count -= B_UNROLL;
+    }
+    if (count2) {
+        BLASLONG gvl2 = __riscv_vsetvl_e16m4(count2);
+        vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, gvl2);
+        vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, gvl2);
+        __riscv_vse32_v_f32m8(CONV, B0, gvl2);
+    }
+}
+#endif
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -12,10 +38,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     __bf16 *AA = (__bf16 *)(A);
 
 #ifdef BF16_WIDEN_ONE
-    FLOAT *B_CONV = NULL;
+    FLOAT *CONV = NULL;
     if ((M >= 4) && (N >= 4) && (K > 0)) {
-        B_CONV = (FLOAT *)(malloc(K * 8 * sizeof(FLOAT)));
-        if (!B_CONV) return 1;
+        CONV = (FLOAT *)(malloc((K * (8 + (M & -4))) * sizeof(FLOAT)));
+        if (!CONV) return 1;
+        B_CONV(AA, CONV + (K * 8), (M & -4) * K);
     }
 #endif
 
@@ -25,26 +52,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         BLASLONG gvl = __riscv_vsetvl_e16m1(8);
 
 #ifdef BF16_WIDEN_ONE
-        BLASLONG bi2;
-        {
-            BLASLONG bi3 = 0;
-            BLASLONG gvl2;
-            bi2 = K * 8;
-            do {
-                gvl2 = __riscv_vsetvl_e16m4(bi2);
-                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
-                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
-                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
-                bi3 += gvl2;
-            } while (bi2 -= gvl2);
-        }
+        BLASLONG bi2 = K * 8;
+        B_CONV(BB + (n_top*K), CONV, bi2);
+        BLASLONG ai2 = K * 8;
 #endif
 
         for (BLASLONG i=0; i<M/8; i+=1) {
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -59,19 +76,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
-                float B4 = B_CONV[bi2+4];
-                float B5 = B_CONV[bi2+5];
-                float B6 = B_CONV[bi2+6];
-                float B7 = B_CONV[bi2+7];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
+                float B4 = CONV[bi2+4];
+                float B5 = CONV[bi2+5];
+                float B6 = CONV[bi2+6];
+                float B7 = CONV[bi2+7];
                 bi2 += 8;
 
-                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
-                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
-                ai += 8;
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 8;
 
                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
@@ -144,10 +160,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16m1(4);
 
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -162,19 +178,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
-                float B4 = B_CONV[bi2+4];
-                float B5 = B_CONV[bi2+5];
-                float B6 = B_CONV[bi2+6];
-                float B7 = B_CONV[bi2+7];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
+                float B4 = CONV[bi2+4];
+                float B5 = CONV[bi2+5];
+                float B6 = CONV[bi2+6];
+                float B7 = CONV[bi2+7];
                 bi2 += 8;
 
-                vbfloat16mf2_t A00 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
-                vfloat32m1_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m1(A00, gvl);
-                ai += 4;
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 4;
 
                 result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
@@ -361,26 +376,16 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         m_top = 0;
 
 #ifdef BF16_WIDEN_ONE
-        BLASLONG bi2;
-        {
-            BLASLONG bi3 = 0;
-            BLASLONG gvl2;
-            bi2 = K * 4;
-            do {
-                gvl2 = __riscv_vsetvl_e16m4(bi2);
-                vbfloat16m4_t A00 = __riscv_vle16_v_bf16m4(&BB[bi3 + (n_top*K)], gvl2);
-                vfloat32m8_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m8(A00, gvl2);
-                __riscv_vse32_v_f32m8(&B_CONV[bi3], A0, gvl2);
-                bi3 += gvl2;
-            } while (bi2 -= gvl2);
-        }
+        BLASLONG bi2 = K * 4;
+        B_CONV(BB + (n_top*K), CONV, bi2);
+        BLASLONG ai2 = K * 8;
 #endif
 
         for (BLASLONG i=0; i<M/8; i+=1) {
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -391,15 +396,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k<K; k++) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
                 bi2 += 4;
 
-                vbfloat16m1_t A00 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
-                vfloat32m2_t A0 = __riscv_vfwcvtbf16_f_f_v_f32m2(A00, gvl);
-                ai += 8;
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 8;
 
                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
@@ -446,10 +450,10 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16m1(4);
 
-            BLASLONG ai=m_top*K;
 #ifdef BF16_WIDEN_ONE
             bi2 = 0;
 #else
+            BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
 #endif
 
@@ -460,15 +464,14 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             for (BLASLONG k=0; k < K; ++k) {
 #ifdef BF16_WIDEN_ONE
-                float B0 = B_CONV[bi2+0];
-                float B1 = B_CONV[bi2+1];
-                float B2 = B_CONV[bi2+2];
-                float B3 = B_CONV[bi2+3];
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                float B2 = CONV[bi2+2];
+                float B3 = CONV[bi2+3];
                 bi2 += 4;
 
-                vbfloat16mf4_t A00 = __riscv_vle16_v_bf16mf4( &AA[ai+0*gvl], gvl );
-                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vfwcvtbf16_f_f_v_f32mf2(A00, gvl));
-                ai += 4;
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 4;
 
                 result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
@@ -819,7 +822,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         n_top += 1;
     }
 #ifdef BF16_WIDEN_ONE
-    if (B_CONV) free(B_CONV);
+    if (CONV) free(CONV);
 #endif
     return 0;
 }

From 1cc377ef61d498b75c852aa4b9b042fe9422c347 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Fri, 13 Feb 2026 18:14:11 +0000
Subject: [PATCH 09/12] Only convert B if M is greater or equal to 4.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 8 ++++++--
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index 1e2bf369e7..fc1a170a82 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -52,7 +52,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         BLASLONG gvl = __riscv_vsetvl_e16m1(16);
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 8;
-        B_CONV(BB + (n_top*K), CONV, bi2);
+        if (M >= 4) {
+            B_CONV(BB + (n_top*K), CONV, bi2);
+        }
         BLASLONG ai2 = K * 8;
 #endif
 
@@ -474,7 +476,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 4;
-        B_CONV(BB + (n_top*K), CONV, bi2);
+        if (M >= 4) {
+            B_CONV(BB + (n_top*K), CONV, bi2);
+        }
         BLASLONG ai2 = K * 8;
 #endif
 
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index 4caa3241e3..29521a286d 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -53,7 +53,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 8;
-        B_CONV(BB + (n_top*K), CONV, bi2);
+        if (M >= 4) {
+          B_CONV(BB + (n_top*K), CONV, bi2);
+        }
         BLASLONG ai2 = K * 8;
 #endif
 
@@ -377,7 +379,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 4;
-        B_CONV(BB + (n_top*K), CONV, bi2);
+        if (M >= 4) {
+            B_CONV(BB + (n_top*K), CONV, bi2);
+        }
         BLASLONG ai2 = K * 8;
 #endif
 

From 7a1d23400fef84f04c4cb61c6cf0f4fd94a64dc4 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Fri, 13 Feb 2026 19:00:41 +0000
Subject: [PATCH 10/12] Add flag for not converting A & B - will be used in
 future to do conversion during packing.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 6 ++++++
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index fc1a170a82..9abdf689b7 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -42,7 +42,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     if ((M >= 4) && (N >= 4) && (K > 0)) {
         CONV = (FLOAT *)(malloc((K * (8 + (M & -4))) * sizeof(FLOAT)));
         if (!CONV) return 1;
+#ifndef BF16_DONT_CONV
         B_CONV(AA, CONV + (K * 8), (M & -4) * K);
+#endif
     }
 #endif
 
@@ -52,9 +54,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         BLASLONG gvl = __riscv_vsetvl_e16m1(16);
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 8;
+#ifndef BF16_DONT_CONV
         if (M >= 4) {
             B_CONV(BB + (n_top*K), CONV, bi2);
         }
+#endif
         BLASLONG ai2 = K * 8;
 #endif
 
@@ -476,9 +480,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 4;
+#ifndef BF16_DONT_CONV
         if (M >= 4) {
             B_CONV(BB + (n_top*K), CONV, bi2);
         }
+#endif
         BLASLONG ai2 = K * 8;
 #endif
 
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index 29521a286d..7a9f17e607 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -42,7 +42,9 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     if ((M >= 4) && (N >= 4) && (K > 0)) {
         CONV = (FLOAT *)(malloc((K * (8 + (M & -4))) * sizeof(FLOAT)));
         if (!CONV) return 1;
+#ifndef BF16_DONT_CONV
         B_CONV(AA, CONV + (K * 8), (M & -4) * K);
+#endif
     }
 #endif
 
@@ -53,9 +55,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 8;
+#ifndef BF16_DONT_CONV
         if (M >= 4) {
           B_CONV(BB + (n_top*K), CONV, bi2);
         }
+#endif
         BLASLONG ai2 = K * 8;
 #endif
 
@@ -379,9 +383,11 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 4;
+#ifndef BF16_DONT_CONV
         if (M >= 4) {
             B_CONV(BB + (n_top*K), CONV, bi2);
         }
+#endif
         BLASLONG ai2 = K * 8;
 #endif
 

From 1d6aa0dc3115485de5ca44e05d5bbaaeeee6d194 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Fri, 13 Feb 2026 20:03:35 +0000
Subject: [PATCH 11/12] Add dummy memsets - just in case.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 8 ++++++++
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index 9abdf689b7..dab6361911 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -29,6 +29,12 @@ static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
 }
 #endif
 
+#ifndef VECTORIZE_MEMSET
+#define memset_zero(ptr, size, dir)  memset(ptr, 0, size)
+#else
+void memset_zero(void *input, BLASLONG size, bool dir);
+#endif
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -44,6 +50,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if (!CONV) return 1;
 #ifndef BF16_DONT_CONV
         B_CONV(AA, CONV + (K * 8), (M & -4) * K);
+#else
+	memset_zero(CONV, (K * (8 + (M & -4))) * sizeof(FLOAT), false);
 #endif
     }
 #endif
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index 7a9f17e607..c062da045a 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -29,6 +29,12 @@ static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
 }
 #endif
 
+#ifndef VECTORIZE_MEMSET
+#define memset_zero(ptr, size, dir)  memset(ptr, 0, size)
+#else
+void memset_zero(void *input, BLASLONG size, bool dir);
+#endif
+
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
     BLASLONG gvl = 0;
@@ -44,6 +50,8 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if (!CONV) return 1;
 #ifndef BF16_DONT_CONV
         B_CONV(AA, CONV + (K * 8), (M & -4) * K);
+#else
+        memset_zero(CONV, (K * (8 + (M & -4))) * sizeof(FLOAT), false);
 #endif
     }
 #endif

From efe63e797011e009b364e0ea47dd3e116b474eb6 Mon Sep 17 00:00:00 2001
From: Chip Kerchner <ckerchner@tenstorrent.com>
Date: Sun, 15 Feb 2026 15:49:59 +0000
Subject: [PATCH 12/12] Add pre-RVA23 to BF16 GEMM.

---
 kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c | 290 +++++++++++++++++---
 kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c  | 263 +++++++++++++++---
 kernel/riscv64/shgemm_kernel_16x8_zvl256b.c |   2 +-
 kernel/riscv64/shgemm_kernel_8x8_zvl128b.c  |   2 +-
 4 files changed, 491 insertions(+), 66 deletions(-)

diff --git a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
index dab6361911..f83abb2339 100644
--- a/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/sbgemm_kernel_16x8_zvl256b.c
@@ -1,7 +1,8 @@
 #include "common.h"
 #include <riscv_vector.h>
 
-#define BF16_WIDEN_ONE
+#define BF16_WIDEN_ONE  // Convert pre-hand and do operations in FP32
+#define USE_BF16_CVT    // Comment out for pre-RVA23 systems like BananaPi
 
 #ifdef BF16_WIDEN_ONE
 #define FORCEINLINE      inline __attribute__((always_inline))
@@ -14,7 +15,12 @@ static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
     count &= -B_UNROLL;
     while (count) {
         vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, B_UNROLL);
+#ifdef USE_BF16_CVT
         vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, B_UNROLL);
+#else
+	vfloat32m8_t B0 = __riscv_vreinterpret_v_u32m8_f32m8(__riscv_vsll_vx_u32m8(
+            __riscv_vwcvtu_x_x_v_u32m8(__riscv_vreinterpret_v_bf16m4_u16m4(B00), B_UNROLL), 16, B_UNROLL));
+#endif
         __riscv_vse32_v_f32m8(CONV, B0, B_UNROLL);
         BB += B_UNROLL;
         CONV += B_UNROLL;
@@ -23,7 +29,12 @@ static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
     if (count2) {
         BLASLONG gvl2 = __riscv_vsetvl_e16m4(count2);
         vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, gvl2);
+#ifdef USE_BF16_CVT
         vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, gvl2);
+#else
+        vfloat32m8_t B0 = __riscv_vreinterpret_v_u32m8_f32m8(__riscv_vsll_vx_u32m8(
+            __riscv_vwcvtu_x_x_v_u32m8(__riscv_vreinterpret_v_bf16m4_u16m4(B00), gvl2), 16, gvl2));
+#endif
         __riscv_vse32_v_f32m8(CONV, B0, gvl2);
     }
 }
@@ -40,20 +51,19 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     BLASLONG gvl = 0;
     BLASLONG m_top = 0;
     BLASLONG n_top = 0;
+#if !defined(BF16_WIDEN_ONE) || !defined(BF16_DONT_CONV)
     __bf16 *BB = (__bf16 *)(B);
     __bf16 *AA = (__bf16 *)(A);
+#endif
 
 #ifdef BF16_WIDEN_ONE
-    FLOAT *CONV = NULL;
-    if ((M >= 4) && (N >= 4) && (K > 0)) {
-        CONV = (FLOAT *)(malloc((K * (8 + (M & -4))) * sizeof(FLOAT)));
-        if (!CONV) return 1;
+    FLOAT *CONV = (FLOAT *)(malloc((K * (8 + M)) * sizeof(FLOAT)));
+    if (!CONV) return 1;
 #ifndef BF16_DONT_CONV
-        B_CONV(AA, CONV + (K * 8), (M & -4) * K);
+    B_CONV(AA, CONV + (K * 8), K * M);
 #else
-	memset_zero(CONV, (K * (8 + (M & -4))) * sizeof(FLOAT), false);
+    memset_zero(CONV, (K * (8 + M)) * sizeof(FLOAT), false);
 #endif
-    }
 #endif
 
     // -- MAIN PASS
@@ -63,9 +73,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 8;
 #ifndef BF16_DONT_CONV
-        if (M >= 4) {
-            B_CONV(BB + (n_top*K), CONV, bi2);
-        }
+        B_CONV(BB + (n_top*K), CONV, bi2);
 #endif
         BLASLONG ai2 = K * 8;
 #endif
@@ -394,10 +402,34 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result13 = 0;
             float result14 = 0;
             float result15 = 0;
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                result2+=CONV[ai2+0]*CONV[bi2+1];
+                result3+=CONV[ai2+1]*CONV[bi2+1];
+                result4+=CONV[ai2+0]*CONV[bi2+2];
+                result5+=CONV[ai2+1]*CONV[bi2+2];
+                result6+=CONV[ai2+0]*CONV[bi2+3];
+                result7+=CONV[ai2+1]*CONV[bi2+3];
+                result8+=CONV[ai2+0]*CONV[bi2+4];
+                result9+=CONV[ai2+1]*CONV[bi2+4];
+                result10+=CONV[ai2+0]*CONV[bi2+5];
+                result11+=CONV[ai2+1]*CONV[bi2+5];
+                result12+=CONV[ai2+0]*CONV[bi2+6];
+                result13+=CONV[ai2+1]*CONV[bi2+6];
+                result14+=CONV[ai2+0]*CONV[bi2+7];
+                result15+=CONV[ai2+1]*CONV[bi2+7];
+                ai2+=2;
+                bi2+=8;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+1]);
@@ -416,6 +448,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result15+=(float)(AA[ai+1])*(float)(BB[bi+7]);
                 ai+=2;
                 bi+=8;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -452,10 +485,26 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result6 = 0;
             float result7 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+0]*CONV[bi2+1];
+                result2+=CONV[ai2+0]*CONV[bi2+2];
+                result3+=CONV[ai2+0]*CONV[bi2+3];
+                result4+=CONV[ai2+0]*CONV[bi2+4];
+                result5+=CONV[ai2+0]*CONV[bi2+5];
+                result6+=CONV[ai2+0]*CONV[bi2+6];
+                result7+=CONV[ai2+0]*CONV[bi2+7];
+                ai2+=1;
+                bi2+=8;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+2]);
@@ -466,6 +515,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result7+=(float)(AA[ai+0])*(float)(BB[bi+7]);
                 ai+=1;
                 bi+=8;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -489,9 +539,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 4;
 #ifndef BF16_DONT_CONV
-        if (M >= 4) {
-            B_CONV(BB + (n_top*K), CONV, bi2);
-        }
+        B_CONV(BB + (n_top*K), CONV, bi2);
 #endif
         BLASLONG ai2 = K * 8;
 #endif
@@ -709,10 +757,26 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result5 = 0;
             float result6 = 0;
             float result7 = 0;
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                result2+=CONV[ai2+0]*CONV[bi2+1];
+                result3+=CONV[ai2+1]*CONV[bi2+1];
+                result4+=CONV[ai2+0]*CONV[bi2+2];
+                result5+=CONV[ai2+1]*CONV[bi2+2];
+                result6+=CONV[ai2+0]*CONV[bi2+3];
+                result7+=CONV[ai2+1]*CONV[bi2+3];
+                ai2+=2;
+                bi2+=4;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+1]);
@@ -723,6 +787,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result7+=(float)(AA[ai+1])*(float)(BB[bi+3]);
                 ai+=2;
                 bi+=4;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -745,16 +810,29 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result2 = 0;
             float result3 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+0]*CONV[bi2+1];
+                result2+=CONV[ai2+0]*CONV[bi2+2];
+                result3+=CONV[ai2+0]*CONV[bi2+3];
+                ai2+=1;
+                bi2+=4;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+2]);
                 result3+=(float)(AA[ai+0])*(float)(BB[bi+3]);
                 ai+=1;
                 bi+=4;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -772,15 +850,37 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     if ( N & 2 ) {
         gvl = __riscv_vsetvl_e16m1(16);
         m_top = 0;
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2 = K * 2;
+#ifndef BF16_DONT_CONV
+        B_CONV(BB + (n_top*K), CONV, bi2);
+#endif
+        BLASLONG ai2 = K * 8;
+#endif
 
         for (BLASLONG i=0; i<M/16; i+=1) {
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
             vfloat32m2_t result1 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                bi2 += 2;
+
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 16;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 bi += 2;
@@ -790,6 +890,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmaccbf16_vf_f32m2(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -808,13 +909,28 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 8 ) {
             gvl = __riscv_vsetvl_e16mf2(8);
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                bi2 += 2;
+
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 bi += 2;
@@ -824,6 +940,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmaccbf16_vf_f32m1(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -844,13 +961,28 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16mf2(4);
 
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                bi2 += 2;
+
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vle32_v_f32mf2(&CONV[ai2], gvl));
+                ai2 += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 bi += 2;
@@ -860,6 +992,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmaccbf16_vf_f32m1(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -883,16 +1016,29 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result1 = 0;
             float result2 = 0;
             float result3 = 0;
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                result2+=CONV[ai2+0]*CONV[bi2+1];
+                result3+=CONV[ai2+1]*CONV[bi2+1];
+                ai2+=2;
+                bi2+=2;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 result3+=(float)(AA[ai+1])*(float)(BB[bi+1]);
                 ai+=2;
                 bi+=2;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -909,14 +1055,25 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result0 = 0;
             float result1 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+0]*CONV[bi2+1];
+                ai2+=1;
+                bi2+=2;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 ai+=1;
                 bi+=2;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -932,14 +1089,35 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     if ( N & 1 ) {
         gvl = __riscv_vsetvl_e16m1(16);
         m_top = 0;
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2 = K * 1;
+#ifndef BF16_DONT_CONV
+        B_CONV(BB + (n_top*K), CONV, bi2);
+#endif
+        BLASLONG ai2 = K * 8;
+#endif
+
 
         for (BLASLONG i=0; i<M/16; i+=1) {
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                bi2 += 1;
+
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 16;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 bi += 1;
 
@@ -947,6 +1125,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 ai += 16;
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m2(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -963,12 +1142,25 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 8 ) {
             gvl = __riscv_vsetvl_e16mf2(8);
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                bi2 += 1;
+
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 bi += 1;
 
@@ -976,6 +1168,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 ai += 8;
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -993,12 +1186,25 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16mf2(4);
 
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                bi2 += 1;
+
+                vfloat32m1_t A0 = __riscv_vlmul_ext_v_f32mf2_f32m1(__riscv_vle32_v_f32mf2(&CONV[ai2], gvl));
+                ai2 += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 bi += 1;
 
@@ -1006,6 +1212,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 ai += 4;
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -1022,14 +1229,25 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 2 ) {
             float result0 = 0;
             float result1 = 0;
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                ai2+=2;
+                bi2+=1;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 ai+=2;
                 bi+=1;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -1043,13 +1261,23 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             float result0 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                ai2+=1;
+                bi2+=1;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 ai+=1;
                 bi+=1;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -1060,7 +1288,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         n_top += 1;
     }
 #ifdef BF16_WIDEN_ONE
-    if (CONV) free(CONV);
+    free(CONV);
 #endif
     return 0;
 }
diff --git a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
index c062da045a..8cf6d64890 100644
--- a/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/sbgemm_kernel_8x8_zvl128b.c
@@ -1,7 +1,8 @@
 #include "common.h"
 #include <riscv_vector.h>
 
-#define BF16_WIDEN_ONE
+#define BF16_WIDEN_ONE  // Convert pre-hand and do operations in FP32
+#define USE_BF16_CVT    // Comment out for pre-RVA23 systems
 
 #ifdef BF16_WIDEN_ONE
 #define FORCEINLINE      inline __attribute__((always_inline))
@@ -14,7 +15,12 @@ static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
     count &= -B_UNROLL;
     while (count) {
         vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, B_UNROLL);
+#ifdef USE_BF16_CVT
         vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, B_UNROLL);
+#else
+        vfloat32m8_t B0 = __riscv_vreinterpret_v_u32m8_f32m8(__riscv_vsll_vx_u32m8(
+            __riscv_vwcvtu_x_x_v_u32m8(__riscv_vreinterpret_v_bf16m4_u16m4(B00), B_UNROLL), 16, B_UNROLL));
+#endif
         __riscv_vse32_v_f32m8(CONV, B0, B_UNROLL);
         BB += B_UNROLL;
         CONV += B_UNROLL;
@@ -23,7 +29,12 @@ static void FORCEINLINE B_CONV(__bf16 *BB, FLOAT *CONV, BLASLONG count)
     if (count2) {
         BLASLONG gvl2 = __riscv_vsetvl_e16m4(count2);
         vbfloat16m4_t B00 = __riscv_vle16_v_bf16m4(BB, gvl2);
+#ifdef USE_BF16_CVT
         vfloat32m8_t B0 = __riscv_vfwcvtbf16_f_f_v_f32m8(B00, gvl2);
+#else
+        vfloat32m8_t B0 = __riscv_vreinterpret_v_u32m8_f32m8(__riscv_vsll_vx_u32m8(
+            __riscv_vwcvtu_x_x_v_u32m8(__riscv_vreinterpret_v_bf16m4_u16m4(B00), gvl2), 16, gvl2));
+#endif
         __riscv_vse32_v_f32m8(CONV, B0, gvl2);
     }
 }
@@ -40,20 +51,19 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     BLASLONG gvl = 0;
     BLASLONG m_top = 0;
     BLASLONG n_top = 0;
+#if !defined(BF16_WIDEN_ONE) || !defined(BF16_DONT_CONV)
     __bf16 *BB = (__bf16 *)(B);
     __bf16 *AA = (__bf16 *)(A);
+#endif
 
 #ifdef BF16_WIDEN_ONE
-    FLOAT *CONV = NULL;
-    if ((M >= 4) && (N >= 4) && (K > 0)) {
-        CONV = (FLOAT *)(malloc((K * (8 + (M & -4))) * sizeof(FLOAT)));
-        if (!CONV) return 1;
+    FLOAT *CONV = (FLOAT *)(malloc((K * (8 + M)) * sizeof(FLOAT)));
+    if (!CONV) return 1;
 #ifndef BF16_DONT_CONV
-        B_CONV(AA, CONV + (K * 8), (M & -4) * K);
+    B_CONV(AA, CONV + (K * 8), K * M);
 #else
-        memset_zero(CONV, (K * (8 + (M & -4))) * sizeof(FLOAT), false);
+    memset_zero(CONV, (K * (8 + M)) * sizeof(FLOAT), false);
 #endif
-    }
 #endif
 
     // -- MAIN PASS
@@ -64,9 +74,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 8;
 #ifndef BF16_DONT_CONV
-        if (M >= 4) {
-          B_CONV(BB + (n_top*K), CONV, bi2);
-        }
+        B_CONV(BB + (n_top*K), CONV, bi2);
 #endif
         BLASLONG ai2 = K * 8;
 #endif
@@ -280,8 +288,12 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 2 ) {
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             float result0 = 0;
             float result1 = 0;
@@ -301,6 +313,26 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result15 = 0;
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                result2+=CONV[ai2+0]*CONV[bi2+1];
+                result3+=CONV[ai2+1]*CONV[bi2+1];
+                result4+=CONV[ai2+0]*CONV[bi2+2];
+                result5+=CONV[ai2+1]*CONV[bi2+2];
+                result6+=CONV[ai2+0]*CONV[bi2+3];
+                result7+=CONV[ai2+1]*CONV[bi2+3];
+                result8+=CONV[ai2+0]*CONV[bi2+4];
+                result9+=CONV[ai2+1]*CONV[bi2+4];
+                result10+=CONV[ai2+0]*CONV[bi2+5];
+                result11+=CONV[ai2+1]*CONV[bi2+5];
+                result12+=CONV[ai2+0]*CONV[bi2+6];
+                result13+=CONV[ai2+1]*CONV[bi2+6];
+                result14+=CONV[ai2+0]*CONV[bi2+7];
+                result15+=CONV[ai2+1]*CONV[bi2+7];
+                ai2+=2;
+                bi2+=8;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+1]);
@@ -319,6 +351,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result15+=(float)(AA[ai+1])*(float)(BB[bi+7]);
                 ai+=2;
                 bi+=8;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -353,10 +386,26 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result6 = 0;
             float result7 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+0]*CONV[bi2+1];
+                result2+=CONV[ai2+0]*CONV[bi2+2];
+                result3+=CONV[ai2+0]*CONV[bi2+3];
+                result4+=CONV[ai2+0]*CONV[bi2+4];
+                result5+=CONV[ai2+0]*CONV[bi2+5];
+                result6+=CONV[ai2+0]*CONV[bi2+6];
+                result7+=CONV[ai2+0]*CONV[bi2+7];
+                ai2+=1;
+                bi2+=8;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+2]);
@@ -367,6 +416,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result7+=(float)(AA[ai+0])*(float)(BB[bi+7]);
                 ai+=1;
                 bi+=8;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -392,9 +442,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 #ifdef BF16_WIDEN_ONE
         BLASLONG bi2 = K * 4;
 #ifndef BF16_DONT_CONV
-        if (M >= 4) {
-            B_CONV(BB + (n_top*K), CONV, bi2);
-        }
+        B_CONV(BB + (n_top*K), CONV, bi2);
 #endif
         BLASLONG ai2 = K * 8;
 #endif
@@ -538,8 +586,12 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 2 ) {
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             float result0 = 0;
             float result1 = 0;
@@ -551,6 +603,18 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result7 = 0;
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                result2+=CONV[ai2+0]*CONV[bi2+1];
+                result3+=CONV[ai2+1]*CONV[bi2+1];
+                result4+=CONV[ai2+0]*CONV[bi2+2];
+                result5+=CONV[ai2+1]*CONV[bi2+2];
+                result6+=CONV[ai2+0]*CONV[bi2+3];
+                result7+=CONV[ai2+1]*CONV[bi2+3];
+                ai2+=2;
+                bi2+=4;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+1]);
@@ -561,6 +625,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 result7+=(float)(AA[ai+1])*(float)(BB[bi+3]);
                 ai+=2;
                 bi+=4;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -583,16 +648,29 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result2 = 0;
             float result3 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+0]*CONV[bi2+1];
+                result2+=CONV[ai2+0]*CONV[bi2+2];
+                result3+=CONV[ai2+0]*CONV[bi2+3];
+                ai2+=1;
+                bi2+=4;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+2]);
                 result3+=(float)(AA[ai+0])*(float)(BB[bi+3]);
                 ai+=1;
                 bi+=4;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -610,15 +688,37 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     if ( N & 2 ) {
         gvl = __riscv_vsetvl_e16m1(8);
         m_top = 0;
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2 = K * 2;
+#ifndef BF16_DONT_CONV
+        B_CONV(BB + (n_top*K), CONV, bi2);
+#endif
+        BLASLONG ai2 = K * 8;
+#endif
 
         for (BLASLONG i=0; i<M/8; i+=1) {
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
             vfloat32m2_t result1 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                bi2 += 2;
+
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 bi += 2;
@@ -628,6 +728,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m2(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmaccbf16_vf_f32m2(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -648,13 +749,28 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16m1(4);
 
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
             vfloat32m1_t result1 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                float B1 = CONV[bi2+1];
+                bi2 += 2;
+
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+                result1 = __riscv_vfmacc_vf_f32m1(result1, B1, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 __bf16 B1 = BB[bi+1];
                 bi += 2;
@@ -664,6 +780,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
                 result1 = __riscv_vfwmaccbf16_vf_f32m1(result1, B1, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -684,8 +801,12 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 2 ) {
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             float result0 = 0;
             float result1 = 0;
@@ -693,12 +814,21 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result3 = 0;
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                result2+=CONV[ai2+0]*CONV[bi2+1];
+                result3+=CONV[ai2+1]*CONV[bi2+1];
+                ai2+=2;
+                bi2+=2;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 result2+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 result3+=(float)(AA[ai+1])*(float)(BB[bi+1]);
                 ai+=2;
                 bi+=2;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -716,14 +846,25 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
             float result0 = 0;
             float result1 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+0]*CONV[bi2+1];
+                ai2+=1;
+                bi2+=2;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+0])*(float)(BB[bi+1]);
                 ai+=1;
                 bi+=2;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -739,21 +880,42 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
     if ( N & 1 ) {
         gvl = __riscv_vsetvl_e16m1(8);
         m_top = 0;
+#ifdef BF16_WIDEN_ONE
+        BLASLONG bi2 = K * 1;
+#ifndef BF16_DONT_CONV
+        B_CONV(BB + (n_top*K), CONV, bi2);
+#endif
+        BLASLONG ai2 = K * 8;
+#endif
 
         for (BLASLONG i=0; i<M/8; i+=1) {
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m2_t result0 = __riscv_vfmv_v_f_f32m2(0.0f, gvl);
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                bi2 += 1;
+
+                vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&CONV[ai2], gvl);
+                ai2 += 8;
+
+                result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 bi += 1;
 
-                vbfloat16m1_t A0 = __riscv_vle16_v_bf16m1( &AA[ai+0*gvl], gvl );
+                vbfloat16mf2_t A0 = __riscv_vle16_v_bf16mf2( &AA[ai+0*gvl], gvl );
                 ai += 8;
 
-                result0 = __riscv_vfwmaccbf16_vf_f32m2(result0, B0, A0, gvl);
+                result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -771,12 +933,25 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         if ( M & 4 ) {
             gvl = __riscv_vsetvl_e16m1(4);
 
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
             BLASLONG ai=m_top*K;
             BLASLONG bi=n_top*K;
+#endif
 
             vfloat32m1_t result0 = __riscv_vfmv_v_f_f32m1(0.0f, gvl);
 
             for (BLASLONG k=0; k < K; ++k) {
+#ifdef BF16_WIDEN_ONE
+                float B0 = CONV[bi2+0];
+                bi2 += 1;
+
+                vfloat32m1_t A0 = __riscv_vle32_v_f32m1(&CONV[ai2], gvl);
+                ai2 += 4;
+
+                result0 = __riscv_vfmacc_vf_f32m1(result0, B0, A0, gvl);
+#else
                 __bf16 B0 = BB[bi+0];
                 bi += 1;
 
@@ -784,6 +959,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
                 ai += 4;
 
                 result0 = __riscv_vfwmaccbf16_vf_f32m1(result0, B0, A0, gvl);
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -799,17 +975,28 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
         if ( M & 2 ) {
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             float result0 = 0;
             float result1 = 0;
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                result1+=CONV[ai2+1]*CONV[bi2+0];
+                ai2+=2;
+                bi2+=1;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 result1+=(float)(AA[ai+1])*(float)(BB[bi+0]);
                 ai+=2;
                 bi+=1;
+#endif
             }
 
             BLASLONG ci=n_top*ldc+m_top;
@@ -823,13 +1010,23 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
 
             float result0 = 0;
 
-            BLASLONG ai = m_top * K;
-            BLASLONG bi = n_top * K;
+#ifdef BF16_WIDEN_ONE
+            bi2 = 0;
+#else
+            BLASLONG ai=m_top*K;
+            BLASLONG bi=n_top*K;
+#endif
 
             for (BLASLONG k=0; k<K; k++) {
+#ifdef BF16_WIDEN_ONE
+                result0+=CONV[ai2+0]*CONV[bi2+0];
+                ai2+=1;
+                bi2+=1;
+#else
                 result0+=(float)(AA[ai+0])*(float)(BB[bi+0]);
                 ai+=1;
                 bi+=1;
+#endif
             }
 
             BLASLONG ci = n_top * ldc + m_top;
@@ -840,7 +1037,7 @@ int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B,
         n_top += 1;
     }
 #ifdef BF16_WIDEN_ONE
-    if (CONV) free(CONV);
+    free(CONV);
 #endif
     return 0;
 }
diff --git a/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c b/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
index 4ec59f5e02..8bb466e338 100644
--- a/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
+++ b/kernel/riscv64/shgemm_kernel_16x8_zvl256b.c
@@ -1,7 +1,7 @@
 #include "common.h"
 #include <riscv_vector.h>
 
-#define FP16_NARROW
+#define FP16_NARROW     // Accumulate in FP16
 
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {
diff --git a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
index 915e1032ad..03bdcc240e 100644
--- a/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
+++ b/kernel/riscv64/shgemm_kernel_8x8_zvl128b.c
@@ -1,7 +1,7 @@
 #include "common.h"
 #include <riscv_vector.h>
 
-#define FP16_NARROW
+#define FP16_NARROW     // Accumulate in FP16
 
 int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc)
 {