xtensa: userspace: handle load/store ring exception

When a page can be accessed by both kernel and user threads,
the autofill DTLB may contain an entry for kernel thread.
This will result in load/store ring exception when it is
accessed by user thread later. In this case, we need to
invalidate all associated TLBs related to kernel access so
hardware can reload the page table the correct permission
for user thread.

Signed-off-by: Daniel Leung <daniel.leung@intel.com>

Author: dcpleung

arch/xtensa/core/ptables.c

@@ -11,6 +11,7 @@
 #include <zephyr/kernel/mm.h>
 #include <zephyr/toolchain.h>
 #include <xtensa/corebits.h>
+#include <xtensa_asm2_context.h>
 #include <xtensa_mmu_priv.h>
 
 #include <kernel_arch_func.h>
@@ -1172,6 +1173,53 @@ int arch_buffer_validate(const void *addr, size_t size, int write)
 	return mem_buffer_validate(addr, size, write, XTENSA_MMU_USER_RING);
 }
 
+bool xtensa_exc_load_store_ring_error_check(void *bsa_p)
+{
+	uintptr_t ring, vaddr;
+	_xtensa_irq_bsa_t *bsa = (_xtensa_irq_bsa_t *)bsa_p;
+
+	ring = (bsa->ps & XCHAL_PS_RING_MASK) >> XCHAL_PS_RING_SHIFT;
+
+	if (ring != XTENSA_MMU_USER_RING) {
+		return true;
+	}
+
+	vaddr = bsa->excvaddr;
+
+	if (arch_buffer_validate((void *)vaddr, sizeof(uint32_t), false) != 0) {
+		/* User thread DO NOT have access to this memory according to
+		 * page table. so this is a true access violation.
+		 */
+		return true;
+	}
+
+	/* User thread has access to this memory according to
+	 * page table. so this is not a true access violation.
+	 *
+	 * Now we need to find all associated auto-refilled DTLBs
+	 * and invalidate them. So that hardware can reload
+	 * from page table with correct permission for user
+	 * thread.
+	 */
+	while (true) {
+		uint32_t dtlb_entry = xtensa_dtlb_probe((void *)vaddr);
+
+		if ((dtlb_entry & XTENSA_MMU_PDTLB_HIT) != XTENSA_MMU_PDTLB_HIT) {
+			/* No more DTLB entry found. */
+			return false;
+		}
+
+		if ((dtlb_entry & XTENSA_MMU_PDTLB_WAY_MASK) >=
+				XTENSA_MMU_NUM_TLB_AUTOREFILL_WAYS) {
+			return false;
+		}
+
+		xtensa_dtlb_entry_invalidate_sync(dtlb_entry);
+	}
+
+	return false;
+}
+
 #ifdef CONFIG_XTENSA_MMU_FLUSH_AUTOREFILL_DTLBS_ON_SWAP
 /* This is only used when swapping page tables and auto-refill DTLBs
  * needing to be invalidated. Otherwise, SWAP_PAGE_TABLE assembly

arch/xtensa/core/vector_handlers.c

@@ -630,6 +630,13 @@ void *xtensa_excint1_c(void *esf)
 		xtensa_lazy_hifi_load(thread->arch.hifi_regs);
 		break;
 #endif /* CONFIG_XTENSA_LAZY_HIFI_SHARING */
+#if defined(CONFIG_XTENSA_MMU) && defined(CONFIG_USERSPACE)
+	case EXCCAUSE_LOAD_STORE_RING:
+		if (!xtensa_exc_load_store_ring_error_check(bsa)) {
+			break;
+		}
+		__fallthrough;
+#endif /* CONFIG_XTENSA_MMU && CONFIG_USERSPACE */
 	default:
 		reason = K_ERR_CPU_EXCEPTION;
 

arch/xtensa/include/xtensa_internal.h

@@ -74,6 +74,22 @@ void xtensa_userspace_enter(k_thread_entry_t user_entry,
  */
 bool xtensa_mem_kernel_has_access(const void *addr, size_t size, int write);
 
+/**
+ * @brief Check if it is a true load/store ring exception.
+ *
+ * When a page can be accessed by both kernel and user threads, the autofill DTLB
+ * may contain an entry for kernel thread. This will result in load/store ring
+ * exception when it is accessed by user thread later. In this case, this will
+ * invalidate all associated TLBs related to kernel access so hardware can reload
+ * the page table the correct permission for user thread.
+ *
+ * @param bsa_p Pointer to BSA struct.
+ *
+ * @retval True This is a true access violation.
+ * @retval False Access violation is due to incorrectly cached auto-refilled TLB.
+ */
+bool xtensa_exc_load_store_ring_error_check(void *bsa_p);
+
 /**
  * @}
  */

arch/xtensa/include/xtensa_mmu_priv.h

@@ -213,6 +213,15 @@
  */
 #define XTENSA_MMU_PDTLB_HIT			BIT(4)
 
+/**
+ * PDTLB WAY mask.
+ *
+ * For more information see
+ * Xtensa Instruction Set Architecture (ISA) Reference Manual
+ * 4.6.5.7 Formats for Probing MMU Option TLB Entries
+ */
+#define XTENSA_MMU_PDTLB_WAY_MASK		0xFU
+
 /**
  * Virtual address where the page table is mapped
  */