Optimise masked attention

stduhpf · stduhpf · commit c19ea0f8cff2 · 2025-06-02T17:29:47.000+02:00
diff --git a/ggml_extend.hpp b/ggml_extend.hpp
@@ -956,7 +956,7 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_attention_ext(struct ggml_context*
         auto kq = ggml_mul_mat(ctx, k, q);  // [N * n_head, L_q, L_k]
         kq      = ggml_scale_inplace(ctx, kq, scale);
         if (mask) {
-            kq = ggml_add(ctx, kq, mask);
+            kq = ggml_add_inplace(ctx, kq, mask);
         }
         if (diag_mask_inf) {
             kq = ggml_diag_mask_inf_inplace(ctx, kq, 0);

Original file line number	Diff line number	Diff line change
`@@ -956,7 +956,7 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_attention_ext(struct ggml_context*`
`956`	`956`	`auto kq = ggml_mul_mat(ctx, k, q); // [N * n_head, L_q, L_k]`
`957`	`957`	`kq = ggml_scale_inplace(ctx, kq, scale);`
`958`	`958`	`if (mask) {`
`959`		`- kq = ggml_add(ctx, kq, mask);`
	`959`	`+ kq = ggml_add_inplace(ctx, kq, mask);`
`960`	`960`	`}`
`961`	`961`	`if (diag_mask_inf) {`
`962`	`962`	`kq = ggml_diag_mask_inf_inplace(ctx, kq, 0);`