diff --git a/Cargo.toml b/Cargo.toml index 38494e937..e385fd259 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "examples/loopback", "examples/miotcp", "examples/mioudp", + "examples/nvme-test", "examples/polling", "examples/rftrace-example", "examples/stdin", diff --git a/examples/nvme-test/.gitignore b/examples/nvme-test/.gitignore new file mode 100644 index 000000000..0c142e93b --- /dev/null +++ b/examples/nvme-test/.gitignore @@ -0,0 +1,5 @@ +hermit-loader-x86_64 +hermit-loader-aarch64 +hermit-loader-riscv64 +opensbi* +nvme.img diff --git a/examples/nvme-test/Cargo.toml b/examples/nvme-test/Cargo.toml new file mode 100644 index 000000000..3225c09cc --- /dev/null +++ b/examples/nvme-test/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "nvme-test" +authors = ["Valentin Kunisch "] +edition = "2024" + +[target.'cfg(target_os = "hermit")'.dependencies] +hermit = { path = "../../hermit", default-features = false, features = ["nvme"] } + +[dependencies] +rand = "0.9.2" +vroom = { git = "https://github.com/valopok/vroom", branch = "main", default-features = false } diff --git a/examples/nvme-test/Makefile b/examples/nvme-test/Makefile new file mode 100644 index 000000000..e92b4ab6f --- /dev/null +++ b/examples/nvme-test/Makefile @@ -0,0 +1,73 @@ +.PHONY: run-x86_64 +run-x86_64: build-x86_64 disk-image + # https://github.com/hermit-os/loader?tab=readme-ov-file#x86-64 + qemu-system-x86_64 \ + -cpu qemu64,apic,fsgsbase,fxsr,rdrand,rdtscp,xsave,xsaveopt \ + -smp 1 \ + -m 128M \ + -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ + -display none \ + -serial stdio \ + -kernel hermit-loader-x86_64 \ + -initrd ../../target/x86_64-unknown-hermit/release/nvme-test \ + -drive file=nvme.img,format=raw,if=none,id=nvm \ + -device nvme,serial=deadbeef,drive=nvm + +.PHONY: run-aarch64 +run-aarch64: build-aarch64 disk-image + # https://github.com/hermit-os/loader?tab=readme-ov-file#aarch64 + qemu-system-aarch64 \ + -machine virt,gic-version=3 \ + -cpu cortex-a76 \ + -smp 1 \ + -m 512M \ + -semihosting \ + -display none \ + -serial stdio \ + -kernel hermit-loader-aarch64 \ + -device guest-loader,addr=0x48000000,initrd=../../target/aarch64-unknown-hermit/release/nvme-test \ + -drive file=nvme.img,format=raw,if=none,id=nvm \ + -device nvme,serial=deadbeef,drive=nvm + +# .PHONY: run-riscv64 +# run-riscv64: build-riscv64 disk-image +# # https://github.com/hermit-os/loader?tab=readme-ov-file#64-bit-risc-v +# qemu-system-riscv64 \ +# -machine virt \ +# -cpu rv64 \ +# -smp 1 \ +# -m 128M \ +# -display none \ +# -serial stdio \ +# -bios opensbi/share/opensbi/lp64/generic/firmware/fw_jump.bin \ +# -kernel hermit-loader-riscv64 \ +# -initrd ../../target/riscv64gc-unknown-hermit/release/nvme-test \ +# -drive file=nvme.img,format=raw,if=none,id=nvm \ +# -device nvme,serial=deadbeef,drive=nvm + +.PHONY: build-x86_64 +build-x86_64: + HERMIT_LOG_LEVEL_FILTER=ERROR \ + cargo build -Z build-std=std,core,alloc,panic_abort \ + --target x86_64-unknown-hermit --release + +.PHONY: build-aarch64 +build-aarch64: + HERMIT_LOG_LEVEL_FILTER=ERROR \ + cargo build -Z build-std=std,core,alloc,panic_abort \ + --target aarch64-unknown-hermit --release + +# .PHONY: build-riscv64 +# build-riscv64: +# HERMIT_LOG_LEVEL_FILTER=ERROR \ +# cargo build -Z build-std=std,core,alloc,panic_abort \ +# --target riscv64gc-unknown-hermit + +.PHONY: disk-image +disk-image: + qemu-img create nvme.img 4M + +.PHONY: clean +clean: + cargo clean + rm -f nvme.img diff --git a/examples/nvme-test/README.md b/examples/nvme-test/README.md new file mode 100644 index 000000000..32dadb3be --- /dev/null +++ b/examples/nvme-test/README.md @@ -0,0 +1,19 @@ +# An example program to test the NVMe driver + +> Note: PCI devices are not yet implemented for `riscv64`. + +Complete following steps to run the test with QEMU: + +- Install [QEMU](https://www.qemu.org) to get the commands +`qemu-system-x86_64`, `qemu-system-aarch64`. + +- Download the [hermit-loaders](https://github.com/hermit-os/loader/releases) for +`x86_64`, `aarch64` and place them into this nvme-test directory. + +> `riscv64` only: download [OpenSBI](https://github.com/riscv-software-src/opensbi/releases), +rename the directory to `opensbi` and place it into this nvme-test directory. + +- Execute `make run-x86_64` (default) or `make run-aarch64` to run the program. + +- Cleanup files with `make clean` afterwards. + diff --git a/examples/nvme-test/src/main.rs b/examples/nvme-test/src/main.rs new file mode 100644 index 000000000..593edba7c --- /dev/null +++ b/examples/nvme-test/src/main.rs @@ -0,0 +1,71 @@ +#[cfg(target_os = "hermit")] +use hermit as _; + +mod syscalls; +mod tests; +use syscalls::*; +use tests::run_tests; + +fn main() { + println!("Hello, NVMe!"); + // CAREFUL: this example writes to the NVMe drive + // match example() { + // Err(error) => eprintln!("{error:?}"), + // Ok(()) => println!("Success!"), + // } + run_tests(); +} + +#[allow(dead_code)] +fn example() -> Result<(), SysNvmeError> { + let namespace_ids = namespace_ids()?; + println!("Namespace IDs: {namespace_ids:?}."); + + let namespace_id = namespace_ids[0]; + let namespace = namespace(&namespace_id)?; + println!("Namespace: {namespace:?}"); + println!( + "Total namespace size: {}", + namespace.blocks * namespace.block_size + ); + + let maximum_transfer_size = maximum_transfer_size()?; + println!("Maximum transfer size: {maximum_transfer_size}."); + + let maximum_number_of_io_queue_pairs = maximum_number_of_io_queue_pairs()?; + println!("Maximum number of I/O queue pairs: {maximum_number_of_io_queue_pairs}."); + + let maximum_queue_entries_supported = maximum_queue_entries_supported()?; + println!("Maximum queue entries supported: {maximum_queue_entries_supported}."); + + let io_queue_pair_id = create_io_queue_pair(&namespace_id, maximum_queue_entries_supported)?; + println!( + "Created IO queue pair with ID {} and {} queue entries for namespace {}.", + io_queue_pair_id.0, maximum_queue_entries_supported, namespace_id.0 + ); + + let length = 16; + let mut buffer_1 = allocate_buffer(&io_queue_pair_id, length)?; + for i in 0..length { + buffer_1[i] = i as u8; + } + + let logical_block_address = 0; + write_to_io_queue_pair(&io_queue_pair_id, &buffer_1, logical_block_address)?; + println!("Wrote to IO queue pair with ID {io_queue_pair_id:?}."); + + let mut buffer_2 = allocate_buffer(&io_queue_pair_id, length)?; + read_from_io_queue_pair(&io_queue_pair_id, &mut buffer_2, logical_block_address)?; + println!("Read from IO queue pair with ID {io_queue_pair_id:?}."); + + println!("buffer_1: {:?}", &buffer_1[0..length]); + println!("buffer_2: {:?}", &buffer_2[0..length]); + + deallocate_buffer(&io_queue_pair_id, buffer_1)?; + deallocate_buffer(&io_queue_pair_id, buffer_2)?; + println!("Deallocated buffers."); + + delete_io_queue_pair(io_queue_pair_id)?; + println!("Deleted IO queue pair."); + Ok(()) +} diff --git a/examples/nvme-test/src/syscalls.rs b/examples/nvme-test/src/syscalls.rs new file mode 100644 index 000000000..ca3ee9136 --- /dev/null +++ b/examples/nvme-test/src/syscalls.rs @@ -0,0 +1,308 @@ +use vroom::{Dma, IoQueuePairId, Namespace, NamespaceId}; + +pub fn namespace_ids() -> Result, SysNvmeError> { + let mut number_of_namespaces: u32 = 0; + let exit_code = unsafe { sys_nvme_number_of_namespaces(&mut number_of_namespaces) }; + if exit_code != 0 { + return Err(SysNvmeError::from(exit_code)); + } + let mut namespace_ids: Vec = Vec::with_capacity(number_of_namespaces as usize); + let exit_code = + unsafe { sys_nvme_namespace_ids(namespace_ids.as_mut_ptr(), number_of_namespaces) }; + + let namespace_ids_pointer = namespace_ids.as_mut_ptr(); + core::mem::forget(namespace_ids); // prevents the Vec from being dropped and deallocated + let namespace_ids = unsafe { + Vec::from_raw_parts( + namespace_ids_pointer, + number_of_namespaces as usize, + number_of_namespaces as usize, + ) + }; + match exit_code { + 0 => Ok(namespace_ids), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn namespace(namespace_id: &NamespaceId) -> Result { + let mut namespace: Namespace = Namespace { + id: NamespaceId(0), + blocks: 0, + block_size: 0, + }; + let exit_code = unsafe { sys_nvme_namespace(namespace_id, &mut namespace) }; + match exit_code { + 0 => Ok(namespace), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn clear_namespace(namespace_id: &NamespaceId) -> Result<(), SysNvmeError> { + let exit_code = unsafe { sys_nvme_clear_namespace(namespace_id) }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn maximum_transfer_size() -> Result { + let mut maximum_transfer_size: usize = 0; + let exit_code = unsafe { sys_nvme_maximum_transfer_size(&mut maximum_transfer_size) }; + match exit_code { + 0 => Ok(maximum_transfer_size), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn maximum_number_of_io_queue_pairs() -> Result { + let mut maximum_number_of_io_queue_pairs: u16 = 0; + let exit_code = + unsafe { sys_nvme_maximum_number_of_io_queue_pairs(&mut maximum_number_of_io_queue_pairs) }; + match exit_code { + 0 => Ok(maximum_number_of_io_queue_pairs), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn maximum_queue_entries_supported() -> Result { + let mut maximum_queue_entries_supported: u32 = 0; + let exit_code = + unsafe { sys_nvme_maximum_queue_entries_supported(&mut maximum_queue_entries_supported) }; + match exit_code { + 0 => Ok(maximum_queue_entries_supported), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn create_io_queue_pair( + namespace_id: &NamespaceId, + number_of_entries: u32, +) -> Result { + let mut result: IoQueuePairId = IoQueuePairId(0); + let exit_code = + unsafe { sys_nvme_create_io_queue_pair(namespace_id, number_of_entries, &mut result) }; + match exit_code { + 0 => Ok(result), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn delete_io_queue_pair(io_queue_pair_id: IoQueuePairId) -> Result<(), SysNvmeError> { + let exit_code = unsafe { sys_nvme_delete_io_queue_pair(io_queue_pair_id) }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn allocate_buffer( + io_queue_pair_id: &IoQueuePairId, + number_of_elements: usize, +) -> Result, SysNvmeError> { + let mut result: Dma = unsafe { Dma::new_uninitialized() }; + let exit_code = + unsafe { sys_nvme_allocate_buffer(io_queue_pair_id, number_of_elements, &mut result) }; + match exit_code { + 0 => Ok(result), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn deallocate_buffer( + io_queue_pair_id: &IoQueuePairId, + mut buffer: Dma, +) -> Result<(), SysNvmeError> { + let exit_code = + unsafe { sys_nvme_deallocate_buffer(io_queue_pair_id, &mut buffer as *mut Dma) }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn read_from_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: &mut Dma, + logical_block_address: u64, +) -> Result<(), SysNvmeError> { + let exit_code = unsafe { + sys_nvme_read_from_io_queue_pair(io_queue_pair_id, buffer, logical_block_address) + }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn write_to_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: &Dma, + logical_block_address: u64, +) -> Result<(), SysNvmeError> { + let exit_code = + unsafe { sys_nvme_write_to_io_queue_pair(io_queue_pair_id, buffer, logical_block_address) }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn submit_read_to_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: &mut Dma, + logical_block_address: u64, +) -> Result<(), SysNvmeError> { + let exit_code = unsafe { + sys_nvme_submit_read_to_io_queue_pair(io_queue_pair_id, buffer, logical_block_address) + }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn submit_write_to_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: &Dma, + logical_block_address: u64, +) -> Result<(), SysNvmeError> { + let exit_code = unsafe { + sys_nvme_submit_write_to_io_queue_pair(io_queue_pair_id, buffer, logical_block_address) + }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +pub fn complete_io_with_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, +) -> Result<(), SysNvmeError> { + let exit_code = unsafe { sys_nvme_complete_io_with_io_queue_pair(io_queue_pair_id) }; + match exit_code { + 0 => Ok(()), + n => Err(SysNvmeError::from(n)), + } +} + +unsafe extern "C" { + #[link_name = "sys_nvme_number_of_namespaces"] + fn sys_nvme_number_of_namespaces(result: *mut u32) -> usize; + + #[link_name = "sys_nvme_namespace_ids"] + fn sys_nvme_namespace_ids(vec_pointer: *mut NamespaceId, length: u32) -> usize; + + #[link_name = "sys_nvme_namespace"] + fn sys_nvme_namespace(namespace_id: &NamespaceId, result: *mut Namespace) -> usize; + + #[link_name = "sys_nvme_clear_namespace"] + fn sys_nvme_clear_namespace(namespace_id: &NamespaceId) -> usize; + + #[link_name = "sys_nvme_maximum_transfer_size"] + fn sys_nvme_maximum_transfer_size(result: *mut usize) -> usize; + + #[link_name = "sys_nvme_maximum_number_of_io_queue_pairs"] + fn sys_nvme_maximum_number_of_io_queue_pairs(result: *mut u16) -> usize; + + #[link_name = "sys_nvme_maximum_queue_entries_supported"] + fn sys_nvme_maximum_queue_entries_supported(result: *mut u32) -> usize; + + #[link_name = "sys_nvme_create_io_queue_pair"] + fn sys_nvme_create_io_queue_pair( + namespace_id: &NamespaceId, + number_of_entries: u32, + resulting_io_queue_pair_id: *mut IoQueuePairId, + ) -> usize; + + #[link_name = "sys_nvme_delete_io_queue_pair"] + fn sys_nvme_delete_io_queue_pair(io_queue_pair_id: IoQueuePairId) -> usize; + + #[link_name = "sys_nvme_allocate_buffer"] + fn sys_nvme_allocate_buffer( + io_queue_pair_id: &IoQueuePairId, + size: usize, + resulting_buffer: *mut Dma, + ) -> usize; + + #[link_name = "sys_nvme_deallocate_buffer"] + fn sys_nvme_deallocate_buffer(io_queue_pair_id: &IoQueuePairId, buffer: *mut Dma) -> usize; + + #[link_name = "sys_nvme_read_from_io_queue_pair"] + fn sys_nvme_read_from_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: *mut Dma, + logical_block_address: u64, + ) -> usize; + + #[link_name = "sys_nvme_write_to_io_queue_pair"] + fn sys_nvme_write_to_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: *const Dma, + logical_block_address: u64, + ) -> usize; + + #[link_name = "sys_nvme_submit_read_to_io_queue_pair"] + fn sys_nvme_submit_read_to_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: *mut Dma, + logical_block_address: u64, + ) -> usize; + + #[link_name = "sys_nvme_submit_write_to_io_queue_pair"] + fn sys_nvme_submit_write_to_io_queue_pair( + io_queue_pair_id: &IoQueuePairId, + buffer: *const Dma, + logical_block_address: u64, + ) -> usize; + + #[link_name = "sys_nvme_complete_io_with_io_queue_pair"] + fn sys_nvme_complete_io_with_io_queue_pair(io_queue_pair_id: &IoQueuePairId) -> usize; +} + +#[derive(Debug, Clone, Copy)] +pub enum SysNvmeError { + UnknownError = 0, + ZeroPointerParameter = 1, + DeviceDoesNotExist = 2, + CouldNotIdentifyNamespaces = 3, + NamespaceDoesNotExist = 4, + MaxNumberOfQueuesReached = 5, + CouldNotCreateIoQueuePair = 6, + CouldNotDeleteIoQueuePair = 7, + CouldNotFindIoQueuePair = 8, + BufferIsZero = 9, + BufferTooBig = 10, + BufferIncorrectlySized = 11, + CouldNotAllocateMemory = 12, + CouldNotAllocateBuffer = 13, + CouldNotDeallocateBuffer = 14, + CouldNotReadFromIoQueuePair = 15, + CouldNotWriteToIoQueuePair = 16, + CouldNotClearNamespace = 17, +} + +impl From for SysNvmeError { + fn from(value: usize) -> Self { + match value { + 1 => SysNvmeError::ZeroPointerParameter, + 2 => SysNvmeError::DeviceDoesNotExist, + 3 => SysNvmeError::CouldNotIdentifyNamespaces, + 4 => SysNvmeError::NamespaceDoesNotExist, + 5 => SysNvmeError::MaxNumberOfQueuesReached, + 6 => SysNvmeError::CouldNotCreateIoQueuePair, + 7 => SysNvmeError::CouldNotDeleteIoQueuePair, + 8 => SysNvmeError::CouldNotFindIoQueuePair, + 9 => SysNvmeError::BufferIsZero, + 10 => SysNvmeError::BufferTooBig, + 11 => SysNvmeError::BufferIncorrectlySized, + 12 => SysNvmeError::CouldNotAllocateMemory, + 13 => SysNvmeError::CouldNotAllocateBuffer, + 14 => SysNvmeError::CouldNotDeallocateBuffer, + 15 => SysNvmeError::CouldNotReadFromIoQueuePair, + 16 => SysNvmeError::CouldNotWriteToIoQueuePair, + 17 => SysNvmeError::CouldNotClearNamespace, + _ => SysNvmeError::UnknownError, + } + } +} diff --git a/examples/nvme-test/src/tests.rs b/examples/nvme-test/src/tests.rs new file mode 100644 index 000000000..956d3e2a0 --- /dev/null +++ b/examples/nvme-test/src/tests.rs @@ -0,0 +1,692 @@ +use std::thread; +use std::time::{Duration, Instant}; + +use vroom::NamespaceId; + +use crate::syscalls::*; + +pub fn run_tests() { + println!("Start running tests."); + test_namespaces(); + // test_maximum_transfer_size(); + // test_maximum_number_of_io_queue_pairs(); + // test_maximum_queue_entries_supported(); + // test_io_queue_pair_creation(); + // test_submit_write_read().unwrap(); + // test_full_namespace_write_read().unwrap(); + // clear_namespaces(); + // fill_namespaces(); + // benchmark_seqr_1_thread(Duration::from_secs(60), 32); + // benchmark_randr_1_thread(Duration::from_secs(120), 32); + // benchmark_randw_1_thread(Duration::from_secs(900), 32); + // benchmark_randw_n_threads(Duration::from_secs(5), 2, 32); // Does not work properly + println!("Tests ran successfully."); +} + +#[allow(dead_code)] +fn test_namespaces() { + let result = namespace_ids(); + assert!( + result.is_ok(), + "Could not get namespace IDs. Please verify that an NVMe device is available." + ); + + let namespace_ids = result.unwrap(); + namespace_ids.iter().for_each(|namespace_id| { + let namespace = namespace(&namespace_id); + dbg!(&namespace); + assert!(namespace.is_ok()); + }); + + let invalid_namespace_id = + NamespaceId(namespace_ids.iter().max().map(|id| id.0 + 1).unwrap_or(0)); + let invalid = namespace(&invalid_namespace_id); + assert!(invalid.is_err()); +} + +#[allow(dead_code)] +fn test_maximum_transfer_size() { + let result = maximum_transfer_size(); + dbg!(&result); + assert!( + result.is_ok(), + "Could not get the maximum transfer size. Please verify that an NVMe device is available." + ); +} + +#[allow(dead_code)] +fn test_maximum_number_of_io_queue_pairs() { + let result = maximum_number_of_io_queue_pairs(); + dbg!(&result); + assert!( + result.is_ok(), + "Could not get the maximum number of I/O queue pairs. Please verify that an NVMe device is available." + ); +} + +#[allow(dead_code)] +fn test_maximum_queue_entries_supported() { + let result = maximum_queue_entries_supported(); + assert!( + result.is_ok(), + "Could not get the maximum number of supported queue entries. Please verify that an NVMe device is available." + ); + let maximum_queue_entries_supported = result.unwrap(); + dbg!(&maximum_queue_entries_supported); + assert!(maximum_queue_entries_supported >= 2); +} + +#[allow(dead_code)] +fn test_io_queue_pair_creation() { + let namespace_ids: Vec = namespace_ids().unwrap(); + let max_entries: u32 = maximum_queue_entries_supported().unwrap(); + for namespace_id in namespace_ids { + let result = create_io_queue_pair(&namespace_id, 0); + assert!(result.is_err()); + let result = create_io_queue_pair(&namespace_id, 1); + assert!(result.is_err()); + + let result = create_io_queue_pair(&namespace_id, 2); + assert!(result.is_ok()); + let result = delete_io_queue_pair(result.unwrap()); + assert!(result.is_ok()); + + let result = create_io_queue_pair(&namespace_id, (max_entries / 2).min(2)); + assert!(result.is_ok()); + let result = delete_io_queue_pair(result.unwrap()); + assert!(result.is_ok()); + + let result = create_io_queue_pair(&namespace_id, max_entries); + assert!(result.is_ok()); + let result = delete_io_queue_pair(result.unwrap()); + assert!(result.is_ok()); + + if max_entries < u32::MAX { + let result = create_io_queue_pair(&namespace_id, max_entries + 1); + assert!(result.is_err()); + let result = create_io_queue_pair(&namespace_id, u32::MAX); + assert!(result.is_err()); + } + + let max_number_of_queue_pairs = maximum_number_of_io_queue_pairs().unwrap(); + let mut queue_pairs = Vec::new(); + + (0..max_number_of_queue_pairs).for_each(|_| { + let result = create_io_queue_pair(&namespace_id, max_entries); + assert!(result.is_ok()); + queue_pairs.push(result.unwrap()) + }); + + let result = create_io_queue_pair(&namespace_id, max_entries); + assert!(result.is_err()); + + queue_pairs.into_iter().for_each(|queue_pair| { + let result = delete_io_queue_pair(queue_pair); + assert!(result.is_ok()); + }); + } +} + +#[allow(dead_code)] +fn test_submit_write_read() -> Result<(), SysNvmeError> { + let namespace_ids: Vec = namespace_ids()?; + let namespace_id: &NamespaceId = namespace_ids.first().unwrap(); + let namespace = namespace(namespace_id)?; + let buffer_length: usize = 4096; + let max_entries = maximum_queue_entries_supported()?; + let io_queue_pair_id = create_io_queue_pair(namespace_id, max_entries)?; + + let mut source_buffer = allocate_buffer(&io_queue_pair_id, buffer_length)?; + (0..buffer_length).for_each(|i| source_buffer[i] = rand::random::()); + + let mut dest_buffer = allocate_buffer(&io_queue_pair_id, buffer_length)?; + (0..buffer_length).for_each(|i| dest_buffer[i] = rand::random::()); + + let alignment = buffer_length as u64 / namespace.block_size; + let max_logical_block_address = namespace.blocks - alignment; + let logical_block_address = + rand::random_range(0..(max_logical_block_address / alignment)) * alignment; + + submit_write_to_io_queue_pair(&io_queue_pair_id, &source_buffer, logical_block_address)?; + complete_io_with_io_queue_pair(&io_queue_pair_id)?; + + submit_read_to_io_queue_pair(&io_queue_pair_id, &mut dest_buffer, logical_block_address)?; + complete_io_with_io_queue_pair(&io_queue_pair_id)?; + + for i in 0..buffer_length { + assert!(source_buffer[i] == dest_buffer[i]); + } + + deallocate_buffer(&io_queue_pair_id, source_buffer)?; + deallocate_buffer(&io_queue_pair_id, dest_buffer)?; + delete_io_queue_pair(io_queue_pair_id)?; + Ok(()) +} + +#[allow(dead_code)] +fn test_full_namespace_write_read() -> Result<(), SysNvmeError> { + let namespace_ids: Vec = namespace_ids()?; + let namespace_id: &NamespaceId = namespace_ids.first().unwrap(); + let namespace = namespace(namespace_id)?; + let max_entries: u32 = maximum_queue_entries_supported()?; + let io_queue_pair_id = create_io_queue_pair(namespace_id, max_entries)?; + + let page_size = 4096; + let block_size = namespace.block_size; + let blocks = namespace.blocks; + let namespace_size = blocks * block_size; + + let buffer_length = 8192; + let required_operations = namespace_size / buffer_length as u64; + let remainder = (namespace_size % buffer_length as u64) as usize; + let blocks_per_operation = buffer_length as u64 / block_size; + let remainder_operations = remainder / page_size; + + let mut source_buffer = allocate_buffer(&io_queue_pair_id, buffer_length)?; + (0..buffer_length).for_each(|i| source_buffer[i] = rand::random::()); + + let mut destination_buffer = allocate_buffer(&io_queue_pair_id, buffer_length)?; + (0..buffer_length).for_each(|i| destination_buffer[i] = rand::random::()); + + let mut random_buffer = allocate_buffer(&io_queue_pair_id, buffer_length)?; + (0..buffer_length).for_each(|i| random_buffer[i] = rand::random::()); + + println!( + "Testing full write and read of name space {} with {blocks} blocks and a block size of {block_size}.", + namespace_id.0 + ); + println!( + "{required_operations} operations are required with a buffer size of {buffer_length}." + ); + println!("{remainder_operations} operations are required for the remainder with a buffer size of {page_size}."); + let start_time = Instant::now(); + for i in 0..required_operations { + for j in 0..buffer_length { + destination_buffer[j] = random_buffer[j]; + } + let logical_block_address = i * blocks_per_operation; + write_to_io_queue_pair(&io_queue_pair_id, &source_buffer, logical_block_address)?; + read_from_io_queue_pair( + &io_queue_pair_id, + &mut destination_buffer, + logical_block_address, + )?; + for j in 0..buffer_length { + assert!(source_buffer[j] == destination_buffer[j]); + } + } + + deallocate_buffer(&io_queue_pair_id, source_buffer)?; + deallocate_buffer(&io_queue_pair_id, destination_buffer)?; + + if remainder != 0 { + let mut source_buffer = allocate_buffer(&io_queue_pair_id, page_size)?; + (0..page_size).for_each(|i| source_buffer[i] = rand::random::()); + + let mut destination_buffer = allocate_buffer(&io_queue_pair_id, page_size)?; + (0..page_size).for_each(|i| destination_buffer[i] = rand::random::()); + + for i in 0..page_size { + destination_buffer[i] = random_buffer[i]; + } + for i in 0..remainder_operations as u64 { + let logical_block_address = + required_operations * blocks_per_operation + i * page_size as u64; + dbg!(logical_block_address); + write_to_io_queue_pair(&io_queue_pair_id, &source_buffer, logical_block_address)?; + read_from_io_queue_pair( + &io_queue_pair_id, + &mut destination_buffer, + logical_block_address, + )?; + for j in 0..page_size { + assert!(source_buffer[j] == destination_buffer[j]); + } + } + deallocate_buffer(&io_queue_pair_id, source_buffer)?; + deallocate_buffer(&io_queue_pair_id, destination_buffer)?; + } + let elapsed_time = start_time.elapsed(); + println!( + "Finished testing name space {} in {:.2} seconds", + namespace_id.0, + elapsed_time.as_secs_f64() + ); + + deallocate_buffer(&io_queue_pair_id, random_buffer)?; + + Ok(()) +} + +#[allow(dead_code)] +fn clear_namespaces() { + for namespace_id in namespace_ids().unwrap() { + clear_namespace(&namespace_id).unwrap(); + } +} + +#[allow(dead_code)] +fn fill_namespaces() { + let namespace_ids: Vec = namespace_ids().unwrap(); + let buffer_length: usize = 4096; + let max_entries = maximum_queue_entries_supported().unwrap(); + + for namespace_id in namespace_ids { + let namespace = namespace(&namespace_id).unwrap(); + let io_queue_pair_id = create_io_queue_pair(&namespace_id, max_entries).unwrap(); + let mut buffer = allocate_buffer(&io_queue_pair_id, buffer_length).unwrap(); + (0..buffer_length).for_each(|i| buffer[i] = rand::random::()); + + let alignment = buffer_length as u64 / namespace.block_size; + let max_logical_block_address = namespace.blocks - alignment; + + println!( + "Started filling name space {} with {} blocks", + namespace_id.0, namespace.blocks + ); + let start_time = Instant::now(); + for i in 0..(max_logical_block_address / alignment) { + let logical_block_address = i * alignment; + write_to_io_queue_pair(&io_queue_pair_id, &buffer, logical_block_address).unwrap(); + } + let elapsed_time = start_time.elapsed(); + + println!( + "Finished filling name space {} in {:.2} seconds", + namespace_id.0, + elapsed_time.as_secs_f64() + ); + + deallocate_buffer(&io_queue_pair_id, buffer).unwrap(); + delete_io_queue_pair(io_queue_pair_id).unwrap(); + } +} + +#[allow(dead_code)] +fn benchmark_seqr_1_thread(duration: Duration, queue_depth: u32) { + let namespace_ids: Vec = namespace_ids().unwrap(); + let namespace_id: &NamespaceId = namespace_ids.first().unwrap(); + let namespace = namespace(namespace_id).unwrap(); + let buffer_length: usize = 4096; + let max_entries = maximum_queue_entries_supported().unwrap(); + assert!(queue_depth <= max_entries); + let io_queue_pair_id = create_io_queue_pair(namespace_id, max_entries).unwrap(); + + let mut buffer = allocate_buffer(&io_queue_pair_id, buffer_length).unwrap(); + (0..buffer_length).for_each(|i| buffer[i] = rand::random::()); + + let mut active_submissions: u32 = 0; + let mut io_operations = 0; + // let mut last_measurement_time = Duration::ZERO; + // let mut vec_operations = Vec::new(); + let alignment = buffer_length as u64 / namespace.block_size; + let max_logical_block_address = namespace.blocks - alignment; + + println!( + "Benchmark sequential read for {} s; queue depth {queue_depth}", + duration.as_secs() + ); + let start_time = Instant::now(); + for i in 0..(max_logical_block_address / alignment) { + if start_time.elapsed() >= duration { + break; + } + let logical_block_address = i * alignment; + while let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + if active_submissions < queue_depth { + submit_read_to_io_queue_pair(&io_queue_pair_id, &mut buffer, logical_block_address) + .unwrap(); + active_submissions += 1; + } + // let elapsed_time = start_time.elapsed(); + // if elapsed_time.as_secs() > last_measurement_time.as_secs() { + // vec_operations.push((elapsed_time.as_secs_f64(), io_operations)); + // last_measurement_time = elapsed_time; + // } + } + while active_submissions > 0 { + if let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + } + let elapsed_time = start_time.elapsed(); + let iops = io_operations as f64 / elapsed_time.as_secs_f64(); + let mbps = iops * buffer_length as f64 / 1_000_000_f64; + println!("Elapsed time: {:.2} s", elapsed_time.as_secs_f64()); + println!("Read operations: {io_operations}"); + println!("IOPS: {iops:.2}"); + println!("MB/s: {mbps:.2}"); + + // println!("(seconds,kIOPS)"); + // let absolute_times: Vec = vec_operations + // .iter() + // .skip(1) + // .map(|(time, _)| *time) + // .collect(); + // let delta_times: Vec = vec_operations + // .windows(2) + // .map(|slice| slice[1].0 - slice[0].0) + // .collect(); + // let delta_operations: Vec = vec_operations + // .windows(2) + // .map(|slice| slice[1].1 - slice[0].1) + // .collect(); + // let average_points = 4; + // for i in 0..absolute_times.len() { + // let previous_points = (average_points - 1).min(i); + // let mut total_delta_time = 0f64; + // let mut total_delta_operations = 0u64; + // for j in 0..previous_points + 1 { + // total_delta_time += delta_times[i - j]; + // total_delta_operations += delta_operations[i - j]; + // } + // let average_delta_time = total_delta_time / (previous_points + 1) as f64; + // let average_delta_operations = total_delta_operations as f64 / (previous_points + 1) as f64; + + // let elapsed_time = absolute_times[i]; + // let average_kiops = (average_delta_operations / average_delta_time) / 1000f64; + // println!("({elapsed_time:.0},{average_kiops:.3})"); + // } + + deallocate_buffer(&io_queue_pair_id, buffer).unwrap(); + delete_io_queue_pair(io_queue_pair_id).unwrap(); +} + +#[allow(dead_code)] +fn benchmark_randr_1_thread(duration: Duration, queue_depth: u32) { + let namespace_ids: Vec = namespace_ids().unwrap(); + let namespace_id: &NamespaceId = namespace_ids.first().unwrap(); + let namespace = namespace(namespace_id).unwrap(); + let buffer_length: usize = 4096; + let max_entries = maximum_queue_entries_supported().unwrap(); + assert!(queue_depth <= max_entries); + let io_queue_pair_id = create_io_queue_pair(namespace_id, max_entries).unwrap(); + + let mut buffer = allocate_buffer(&io_queue_pair_id, buffer_length).unwrap(); + (0..buffer_length).for_each(|i| buffer[i] = rand::random::()); + + let mut active_submissions: u32 = 0; + let mut io_operations = 0; + let mut last_measurement_time = Duration::ZERO; + let mut vec_operations = Vec::new(); + let alignment = buffer_length as u64 / namespace.block_size; + let max_logical_block_address = namespace.blocks - alignment; + + println!( + "Benchmark random read for {} s; queue depth {queue_depth}", + duration.as_secs() + ); + let start_time = Instant::now(); + while start_time.elapsed() < duration { + let logical_block_address = + rand::random_range(0..(max_logical_block_address / alignment)) * alignment; + while let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + if active_submissions < queue_depth { + submit_read_to_io_queue_pair(&io_queue_pair_id, &mut buffer, logical_block_address) + .unwrap(); + active_submissions += 1; + } + let elapsed_time = start_time.elapsed(); + if elapsed_time.as_secs() > last_measurement_time.as_secs() { + vec_operations.push((elapsed_time.as_secs_f64(), io_operations)); + last_measurement_time = elapsed_time; + } + } + while active_submissions > 0 { + if let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + } + let elapsed_time = start_time.elapsed(); + let iops = io_operations as f64 / elapsed_time.as_secs_f64(); + println!("Elapsed time: {:.2} s", elapsed_time.as_secs_f64()); + println!("Read operations: {io_operations}"); + println!("IOPS: {iops:.2}"); + + println!("(seconds,kIOPS)"); + let absolute_times: Vec = vec_operations + .iter() + .skip(1) + .map(|(time, _)| *time) + .collect(); + let delta_times: Vec = vec_operations + .windows(2) + .map(|slice| slice[1].0 - slice[0].0) + .collect(); + let delta_operations: Vec = vec_operations + .windows(2) + .map(|slice| slice[1].1 - slice[0].1) + .collect(); + let average_points = 4; + for i in 0..absolute_times.len() { + let previous_points = (average_points - 1).min(i); + let mut total_delta_time = 0f64; + let mut total_delta_operations = 0u64; + for j in 0..previous_points + 1 { + total_delta_time += delta_times[i - j]; + total_delta_operations += delta_operations[i - j]; + } + let average_delta_time = total_delta_time / (previous_points + 1) as f64; + let average_delta_operations = total_delta_operations as f64 / (previous_points + 1) as f64; + + let elapsed_time = absolute_times[i]; + let average_kiops = (average_delta_operations / average_delta_time) / 1000f64; + println!("({elapsed_time:.0},{average_kiops:.3})"); + } + + deallocate_buffer(&io_queue_pair_id, buffer).unwrap(); + delete_io_queue_pair(io_queue_pair_id).unwrap(); +} + +/// CAUTION: this will overwrite the NVMe device with random data! +#[allow(dead_code)] +fn benchmark_randw_1_thread(duration: Duration, queue_depth: u32) { + let namespace_ids: Vec = namespace_ids().unwrap(); + let namespace_id: &NamespaceId = namespace_ids.first().unwrap(); + let namespace = namespace(namespace_id).unwrap(); + let buffer_length: usize = 4096; + let max_entries = maximum_queue_entries_supported().unwrap(); + assert!(queue_depth <= max_entries); + let io_queue_pair_id = create_io_queue_pair(namespace_id, max_entries).unwrap(); + + let mut buffer = allocate_buffer(&io_queue_pair_id, buffer_length).unwrap(); + (0..buffer_length).for_each(|i| buffer[i] = rand::random::()); + + let mut active_submissions: u32 = 0; + let mut io_operations: u64 = 0; + let mut last_measurement_time = Duration::ZERO; + let mut vec_operations = Vec::new(); + let alignment = buffer_length as u64 / namespace.block_size; + let max_logical_block_address = namespace.blocks - alignment; + + clear_namespace(namespace_id).unwrap(); + + println!( + "Benchmark random write for {} s; queue depth {queue_depth}", + duration.as_secs() + ); + let start_time = Instant::now(); + while start_time.elapsed() < duration { + let logical_block_address = + rand::random_range(0..(max_logical_block_address / alignment)) * alignment; + while let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + if active_submissions < queue_depth { + submit_write_to_io_queue_pair(&io_queue_pair_id, &mut buffer, logical_block_address) + .unwrap(); + active_submissions += 1; + } + let elapsed_time = start_time.elapsed(); + if elapsed_time.as_secs() > last_measurement_time.as_secs() { + vec_operations.push((elapsed_time.as_secs_f64(), io_operations)); + last_measurement_time = elapsed_time; + } + } + while active_submissions > 0 { + if let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + } + let elapsed_time = start_time.elapsed(); + let iops = io_operations as f64 / elapsed_time.as_secs_f64(); + println!("Elapsed time: {:.2} s", elapsed_time.as_secs_f64()); + println!("Write operations: {io_operations}"); + println!("IOPS: {iops:.2}"); + + println!("(seconds,kIOPS)"); + let absolute_times: Vec = vec_operations + .iter() + .skip(1) + .map(|(time, _)| *time) + .collect(); + let delta_times: Vec = vec_operations + .windows(2) + .map(|slice| slice[1].0 - slice[0].0) + .collect(); + let delta_operations: Vec = vec_operations + .windows(2) + .map(|slice| slice[1].1 - slice[0].1) + .collect(); + let average_points = 6; + for i in 0..absolute_times.len() { + let previous_points = (average_points - 1).min(i); + let mut total_delta_time = 0f64; + let mut total_delta_operations = 0u64; + for j in 0..previous_points + 1 { + total_delta_time += delta_times[i - j]; + total_delta_operations += delta_operations[i - j]; + } + let average_delta_time = total_delta_time / (previous_points + 1) as f64; + let average_delta_operations = total_delta_operations as f64 / (previous_points + 1) as f64; + + let elapsed_time = absolute_times[i]; + let average_kiops = (average_delta_operations / average_delta_time) / 1000f64; + println!("({elapsed_time:.0},{average_kiops:.3})"); + } + + deallocate_buffer(&io_queue_pair_id, buffer).unwrap(); + delete_io_queue_pair(io_queue_pair_id).unwrap(); +} + +/// CAUTION: this will overwrite the NVMe device with random data! +/// Hermit uses a cooperative scheduler that does not play nicely with +/// parallelizing these workloads to many threads. +/// The threads need to manually yield which seems to cause big overhead. +#[allow(dead_code)] +fn benchmark_randw_n_threads(duration: Duration, number_of_threads: u8, queue_depth: u32) { + println!( + "Benchmark random write for {} s; threads: {number_of_threads}, queue depth {queue_depth}", + duration.as_secs() + ); + let mut io_queue_pair_ids = Vec::new(); + let mut threads = Vec::new(); + + let namespace_ids: Vec = namespace_ids().unwrap(); + let namespace_id: &NamespaceId = namespace_ids.first().unwrap(); + let namespace = namespace(namespace_id).unwrap(); + let buffer_length: usize = 4096; + let alignment = buffer_length as u64 / namespace.block_size; + let max_logical_block_address = namespace.blocks - alignment; + let max_entries = maximum_queue_entries_supported().unwrap(); + assert!(queue_depth <= max_entries); + + clear_namespace(namespace_id).unwrap(); + + for _ in 0..number_of_threads { + let io_queue_pair_id = create_io_queue_pair(&namespace_id, max_entries).unwrap(); + io_queue_pair_ids.push(io_queue_pair_id); + } + println!("{io_queue_pair_ids:?}"); + for i in 0..number_of_threads { + let io_queue_pair_id = io_queue_pair_ids[i as usize]; + let mut buffer = allocate_buffer(&io_queue_pair_id, buffer_length).unwrap(); + (0..buffer_length).for_each(|i| buffer[i] = rand::random::()); + + // returns operations and IOPS + let handle = thread::spawn(move || -> (u64, f64) { + println!("T{i}: spawned"); + let mut active_submissions: u32 = 0; + let mut io_operations = 0; + let mut yield_counter: u64 = 0; + let mut last_yield_time = Duration::ZERO; + let yield_time_delta = Duration::from_millis(1000); + + thread::yield_now(); + + let start_time = Instant::now(); + while start_time.elapsed() < duration { + let logical_block_address = + rand::random_range(0..(max_logical_block_address / alignment)) * alignment; + while let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + if active_submissions < queue_depth { + submit_write_to_io_queue_pair( + &io_queue_pair_id, + &mut buffer, + logical_block_address, + ) + .unwrap(); + active_submissions += 1; + } + if start_time.elapsed() > last_yield_time + yield_time_delta { + last_yield_time = start_time.elapsed(); + yield_counter += 1; + thread::yield_now(); + } + } + while active_submissions > 0 { + if let Ok(_) = complete_io_with_io_queue_pair(&io_queue_pair_id) { + active_submissions -= 1; + io_operations += 1; + } + } + let elapsed_time = start_time.elapsed(); + println!("T{i}: yield counter {yield_counter}"); + println!("T{i}: elapsed time {:.2}", elapsed_time.as_secs_f64()); + + deallocate_buffer(&io_queue_pair_id, buffer).unwrap(); + println!("T{i}: deallocated buffer"); + + let io_operations_per_second = io_operations as f64 / elapsed_time.as_secs_f64(); + (io_operations, io_operations_per_second) + }); + threads.push(handle); + } + + let global_start_time = Instant::now(); + + let (io_operations, io_operations_per_second) = + threads.into_iter().fold((0, 0.), |accumulator, thread| { + let result = thread + .join() + .expect("The thread creation or execution failed!"); + (accumulator.0 + result.0, accumulator.1 + result.1) + }); + + let globally_elapsed_time = global_start_time.elapsed(); + println!( + "Globally elapsed seconds: {:.2}", + globally_elapsed_time.as_secs_f64() + ); + + for i in 0..number_of_threads { + delete_io_queue_pair(io_queue_pair_ids[i as usize]).unwrap(); + println!("T{i}: deleted IOQPID {:?}", io_queue_pair_ids[i as usize]); + } + + println!("Write operations: {io_operations}"); + println!("IOPS: {io_operations_per_second:.2}"); +} diff --git a/hermit/Cargo.toml b/hermit/Cargo.toml index 3e28567d2..4c1319d20 100644 --- a/hermit/Cargo.toml +++ b/hermit/Cargo.toml @@ -53,6 +53,7 @@ instrument = [] libc = [] mmap = [] net = [] +nvme = [] pci = [] pci-ids = ["pci"] diff --git a/hermit/build.rs b/hermit/build.rs index 35aeb96f6..0b4d7f86c 100644 --- a/hermit/build.rs +++ b/hermit/build.rs @@ -130,6 +130,7 @@ impl KernelSrc { "log-target", "mmap", "net", + "nvme", "pci", "pci-ids", "rtl8139",