diff --git a/esp-hal/src/aes/mod.rs b/esp-hal/src/aes/mod.rs index 07b09150591..2d8096f6531 100644 --- a/esp-hal/src/aes/mod.rs +++ b/esp-hal/src/aes/mod.rs @@ -382,8 +382,8 @@ pub mod dma { DmaError, DmaRxBuffer, DmaTxBuffer, - InternalMemoryCachelineAligned, NoBuffer, + aligned::InternalMemory, prepare_for_rx, prepare_for_tx, }, @@ -761,7 +761,7 @@ pub mod dma { #[cfg(dma_can_access_psram)] unaligned_data_buffers: [Option; 2], - descriptors: InternalMemoryCachelineAligned<[DmaDescriptor; OUT_DESCR_COUNT + 1]>, + descriptors: InternalMemory<[DmaDescriptor; OUT_DESCR_COUNT + 1]>, } // The DMA descriptors prevent auto-implementing Sync and Send, but they can be treated as Send @@ -801,9 +801,7 @@ pub mod dma { #[cfg(dma_can_access_psram)] unaligned_data_buffers: [const { None }; 2], - descriptors: InternalMemoryCachelineAligned::new( - [DmaDescriptor::EMPTY; OUT_DESCR_COUNT + 1], - ), + descriptors: InternalMemory::new([DmaDescriptor::EMPTY; OUT_DESCR_COUNT + 1]), } } @@ -944,7 +942,7 @@ pub mod dma { #[cfg(dma_can_access_psram)] for buffer in self.unaligned_data_buffers.iter_mut() { // Avoid copying the write_back buffer - if let Some(buffer) = buffer.as_ref() { + if let Some(buffer) = buffer.as_mut() { buffer.write_back(); } *buffer = None; @@ -988,7 +986,7 @@ pub mod dma { work_item: &mut AesOperation, ) -> Result<(), AesDma<'d>> { let input_len = work_item.buffers.input.len(); - let (input_dscr, output_dscr) = self.descriptors.get_mut().split_at_mut(1); + let (input_dscr, output_dscr) = self.descriptors.get_mut().into_inner().split_at_mut(1); let (input_buffer, data_len) = unsafe { // This unwrap is infallible as AES-DMA devices don't have TX DMA alignment // requirements. diff --git a/esp-hal/src/dma/aligned.rs b/esp-hal/src/dma/aligned.rs new file mode 100644 index 00000000000..2db052fa7f5 --- /dev/null +++ b/esp-hal/src/dma/aligned.rs @@ -0,0 +1,205 @@ +//! Helper types for DMA buffers. + +use core::ops::{Deref, DerefMut}; + +use procmacros::doc_replace; + +#[cfg(dma_can_access_psram)] +use crate::soc::is_valid_psram_address; +use crate::{ + dma::{DmaAlignmentError, DmaBufError}, + soc::is_valid_ram_address, +}; + +/// DMA appropriate wrapper type for internal memory values. +/// +/// The value wrapped in this type is guaranteed to be safely useable +/// as a DMA buffer or descriptor array, meaning DMA or cache management +/// operations will not corrupt surrounding data. +/// +/// [`DmaAlignedMut`] is a reference type that carries this guarantee. +// ESP32-P4 internal memory is cached, enforce alignment to avoid memory +// corruption. Technically only needed for IN buffers and descriptor lists. +#[derive(Debug)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] +#[cfg_attr(soc_internal_memory_cached, repr(C, align(64)))] // dcache cache line +#[cfg_attr(not(soc_internal_memory_cached), repr(C, align(4)))] // Worst-case word alignment +#[instability::unstable] +pub struct InternalMemory(T); + +impl InternalMemory { + /// Creates a new value aligned for DMA operations in internal memory. + #[instability::unstable] + pub const fn new(init: T) -> Self { + Self(init) + } + + /// Returns a [`DmaAlignedMut`] to the underlying value. + /// + /// # Panics + /// + /// Panics if the value is not located at a valid internal RAM address. + #[instability::unstable] + pub fn get_mut(&mut self) -> DmaAlignedMut<'_, T> { + assert!(is_valid_ram_address(&raw const self.0 as usize)); + DmaAlignedMut(&mut self.0) + } +} + +/// A mutable reference to an [`InternalMemory`] object. +#[derive(Debug)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] +#[instability::unstable] +pub struct DmaAlignedMut<'a, T>(&'a mut T) +where + T: ?Sized; + +impl<'a, T: ?Sized> DmaAlignedMut<'a, T> { + #[doc_replace("align_req" => { + cfg(soc_internal_memory_cached) => "64", + _ => "4" + })] + /// Creates a new [`DmaAlignedMut`] from a mutable variable, if it's + /// provably compatible. + /// + /// In internal memory, the address and size of the variable + /// must be at least __align_req__ byte aligned. + #[instability::unstable] + pub fn new(ptr: &'a mut T) -> Result { + if core::mem::size_of_val(ptr) == 0 { + return Ok(Self(ptr)); + } + + let addr = ptr as *mut T as *mut () as usize; + + if is_valid_ram_address(addr) { + let alignment = core::mem::align_of::>(); + + if !core::mem::size_of_val(ptr).is_multiple_of(alignment) { + return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Size)); + } + if !addr.is_multiple_of(alignment) { + return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Address)); + } + + return Ok(Self(ptr)); + } + + #[cfg(dma_can_access_psram)] + if is_valid_psram_address(addr) { + let alignment = cfg_select! { + // TODO(esp32p4): PSRAM is cached through the L2 cache, whose line size is + // configurable (64 or 128 bytes) and is not encoded anywhere yet. Assume the + // larger, always-safe value until the L2 line size is available. + soc_internal_memory_cached => 128, + any(esp32, esp32c5, esp32c61) => 32, // TODO: fixed 32-bytes, metadata-ify + _ => crate::soc::CONFIG_DATA_CACHE_LINE_SIZE, + }; + + if !core::mem::size_of_val(ptr).is_multiple_of(alignment) { + return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Size)); + } + if !addr.is_multiple_of(alignment) { + return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Address)); + } + + return Ok(Self(ptr)); + } + + Err(DmaBufError::UnsupportedMemoryRegion) + } + + /// Creates a new [`DmaAlignedMut`] from *any* mutable slice. + /// + /// # Safety + /// + /// The caller must ensure that the reference is properly aligned for the memory region it + /// occupies and the cachelines occupied by the reference do not overlap with any other data + /// that can be corrupted by a cacheline invalidation operation. + #[instability::unstable] + pub const unsafe fn new_unchecked(ptr: &'a mut T) -> Self { + Self(ptr) + } + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + fn do_cache_op(&self) -> bool { + // Do not do cache operations on zero-sized values. + if core::mem::size_of_val(self.0) == 0 { + return false; + } + + let mut in_cached_region = false; + let address = self.0 as *const T as *const () as usize; + + #[cfg(soc_internal_memory_cached)] + { + in_cached_region |= is_valid_ram_address(address); + } + #[cfg(dma_can_access_psram)] + { + in_cached_region |= is_valid_psram_address(address); + } + + in_cached_region + } + + /// Writes back the data from the cache to memory. + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + #[instability::unstable] + pub fn writeback(&mut self) { + if !self.do_cache_op() { + return; + } + + // SAFETY: we own the cachelines and can't trash anything else + unsafe { + crate::soc::cache_writeback_addr( + self.0 as *const T as *const () as u32, + core::mem::size_of_val(self.0) as u32, + ); + } + } + + /// Invalidates the cache lines for this data. + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + #[instability::unstable] + pub fn invalidate(&mut self) { + if !self.do_cache_op() { + return; + } + + // SAFETY: we own the cachelines and can't trash anything else + unsafe { + crate::soc::cache_invalidate_addr( + self.0 as *const T as *const () as u32, + core::mem::size_of_val(self.0) as u32, + ); + } + } + + /// Converts this object into a mutable reference. + pub fn into_inner(self) -> &'a mut T { + self.0 + } +} + +impl<'a, T, const N: usize> DmaAlignedMut<'a, [T; N]> { + /// Converts this object into a slice reference. + pub fn unsize(self) -> DmaAlignedMut<'a, [T]> { + DmaAlignedMut(self.0) + } +} + +impl<'a, T: ?Sized> Deref for DmaAlignedMut<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a, T: ?Sized> DerefMut for DmaAlignedMut<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + } +} diff --git a/esp-hal/src/dma/buffers/mod.rs b/esp-hal/src/dma/buffers/mod.rs index cd7ba5eb873..8be8c45c522 100644 --- a/esp-hal/src/dma/buffers/mod.rs +++ b/esp-hal/src/dma/buffers/mod.rs @@ -1,16 +1,16 @@ #[cfg(dma_can_access_psram)] -use core::ops::Range; +use core::{mem::MaybeUninit, ops::Range}; use core::{ ops::{Deref, DerefMut}, ptr::{NonNull, null_mut}, }; use super::*; -use crate::soc::is_slice_in_dram; #[cfg(dma_can_access_psram)] +use crate::soc::{is_slice_in_psram, is_valid_psram_address, is_valid_ram_address}; use crate::{ - dma::InternalMemoryBuffer, - soc::{is_slice_in_psram, is_valid_psram_address, is_valid_ram_address}, + dma::aligned::{DmaAlignedMut, InternalMemory}, + soc::is_slice_in_dram, }; pub(crate) mod scoped; @@ -367,9 +367,6 @@ pub struct Preparation { /// The descriptor the DMA will start from. pub start: *mut DmaDescriptor, - /// The direction of the DMA transfer. - pub direction: TransferDirection, - /// Must be `true` if any of the DMA descriptors contain data in PSRAM. #[cfg(dma_can_access_psram)] pub accesses_psram: bool, @@ -505,18 +502,11 @@ pub struct DmaTxBuf(ScopedDmaTxBuf<'static>); impl DmaTxBuf { /// Creates a new [DmaTxBuf] from some descriptors and a buffer. - /// - /// There must be enough descriptors for the provided buffer. - /// Depending on alignment requirements, each descriptor can handle at most - /// 4095 bytes worth of buffer. - /// - /// Both the descriptors and buffer must be in DMA-capable memory. - /// Only DRAM is supported for descriptors. pub fn new( - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, ) -> Result { - ScopedDmaTxBuf::new_with_config(descriptors, buffer, BurstConfig::default()).map(Self) + ScopedDmaTxBuf::new(descriptors, buffer).map(Self) } /// Creates a new [DmaTxBuf] from some descriptors and a buffer. @@ -528,8 +518,8 @@ impl DmaTxBuf { /// Both the descriptors and buffer must be in DMA-capable memory. /// Only DRAM is supported for descriptors. pub fn new_with_config( - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, config: impl Into, ) -> Result { ScopedDmaTxBuf::new_with_config(descriptors, buffer, config).map(Self) @@ -541,7 +531,12 @@ impl DmaTxBuf { } /// Consume the buf, returning the descriptors and buffer. - pub fn split(self) -> (&'static mut [DmaDescriptor], &'static mut [u8]) { + pub fn split( + self, + ) -> ( + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [u8]>, + ) { self.0.split() } @@ -618,17 +613,11 @@ pub struct DmaRxBuf(ScopedDmaRxBuf<'static>); impl DmaRxBuf { /// Creates a new [DmaRxBuf] from some descriptors and a buffer. - /// - /// There must be enough descriptors for the provided buffer. - /// Each descriptor can handle 4092 bytes worth of buffer. - /// - /// Both the descriptors and buffer must be in DMA-capable memory. - /// Only DRAM is supported. pub fn new( - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, ) -> Result { - ScopedDmaRxBuf::new_with_config(descriptors, buffer, BurstConfig::default()).map(Self) + ScopedDmaRxBuf::new(descriptors, buffer).map(Self) } /// Creates a new [DmaRxBuf] from some descriptors and a buffer. @@ -640,8 +629,8 @@ impl DmaRxBuf { /// Both the descriptors and buffer must be in DMA-capable memory. /// Only DRAM is supported for descriptors. pub fn new_with_config( - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, config: impl Into, ) -> Result { ScopedDmaRxBuf::new_with_config(descriptors, buffer, config).map(Self) @@ -653,7 +642,12 @@ impl DmaRxBuf { } /// Consume the buf, returning the descriptors and buffer. - pub fn split(self) -> (&'static mut [DmaDescriptor], &'static mut [u8]) { + pub fn split( + self, + ) -> ( + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [u8]>, + ) { self.0.split() } @@ -742,22 +736,16 @@ unsafe impl DmaRxBuffer for DmaRxBuf { pub struct DmaRxTxBuf { rx_descriptors: DescriptorSet<'static>, tx_descriptors: DescriptorSet<'static>, - buffer: &'static mut [u8], + buffer: DmaAlignedMut<'static, [u8]>, burst: BurstConfig, } impl DmaRxTxBuf { /// Creates a new [DmaRxTxBuf] from some descriptors and a buffer. - /// - /// There must be enough descriptors for the provided buffer. - /// Each descriptor can handle 4092 bytes worth of buffer. - /// - /// Both the descriptors and buffer must be in DMA-capable memory. - /// Only DRAM is supported. pub fn new( - rx_descriptors: &'static mut [DmaDescriptor], - tx_descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + rx_descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + tx_descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, ) -> Result { let mut buf = Self { rx_descriptors: DescriptorSet::new(rx_descriptors)?, @@ -780,14 +768,12 @@ impl DmaRxTxBuf { let burst = burst.into(); self.set_length_fallible(length, burst)?; - self.rx_descriptors.link_with_buffer( - self.buffer, - burst.max_chunk_size_for(self.buffer, TransferDirection::In), - )?; - self.tx_descriptors.link_with_buffer( - self.buffer, - burst.max_chunk_size_for(self.buffer, TransferDirection::Out), - )?; + let max_chunk_size_in = burst.max_chunk_size_for(&self.buffer, TransferDirection::In); + let max_chunk_size_out = burst.max_chunk_size_for(&self.buffer, TransferDirection::Out); + self.rx_descriptors + .link_with_buffer(&mut self.buffer, max_chunk_size_in)?; + self.tx_descriptors + .link_with_buffer(&mut self.buffer, max_chunk_size_out)?; self.burst = burst; @@ -802,12 +788,13 @@ impl DmaRxTxBuf { /// Consume the buf, returning the rx descriptors, tx descriptors and /// buffer. + #[allow(clippy::type_complexity)] pub fn split( self, ) -> ( - &'static mut [DmaDescriptor], - &'static mut [DmaDescriptor], - &'static mut [u8], + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [u8]>, ) { ( self.rx_descriptors.into_inner(), @@ -832,12 +819,12 @@ impl DmaRxTxBuf { /// Returns the entire buf as a slice than can be read. pub fn as_slice(&self) -> &[u8] { - self.buffer + &self.buffer } /// Returns the entire buf as a slice than can be written. pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.buffer + &mut self.buffer } fn set_length_fallible(&mut self, len: usize, burst: BurstConfig) -> Result<(), DmaBufError> { @@ -847,14 +834,10 @@ impl DmaRxTxBuf { burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::In)?; burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::Out)?; - self.rx_descriptors.set_rx_length( - len, - burst.max_chunk_size_for(self.buffer, TransferDirection::In), - )?; - self.tx_descriptors.set_tx_length( - len, - burst.max_chunk_size_for(self.buffer, TransferDirection::Out), - )?; + let max_chunk_size_in = burst.max_chunk_size_for(&self.buffer, TransferDirection::In); + let max_chunk_size_out = burst.max_chunk_size_for(&self.buffer, TransferDirection::Out); + self.rx_descriptors.set_rx_length(len, max_chunk_size_in)?; + self.tx_descriptors.set_tx_length(len, max_chunk_size_out)?; Ok(()) } @@ -879,24 +862,14 @@ unsafe impl DmaTxBuffer for DmaRxTxBuf { desc.reset_for_tx(desc.next.is_null()); } - cfg_if::cfg_if! { - if #[cfg(dma_can_access_psram)] { - // Optimization: avoid locking for PSRAM range. - let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); - if is_data_in_psram || cfg!(soc_internal_memory_cached) { - unsafe { - crate::soc::cache_writeback_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } - } + #[cfg(dma_can_access_psram)] + let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + self.buffer.writeback(); Preparation { start: self.tx_descriptors.head(), - direction: TransferDirection::Out, #[cfg(dma_can_access_psram)] accesses_psram: is_data_in_psram, burst_transfer: self.burst, @@ -940,7 +913,6 @@ unsafe impl DmaRxBuffer for DmaRxTxBuf { Preparation { start: self.rx_descriptors.head(), - direction: TransferDirection::In, #[cfg(dma_can_access_psram)] accesses_psram: is_data_in_psram, burst_transfer: self.burst, @@ -1001,8 +973,8 @@ unsafe impl DmaRxBuffer for DmaRxTxBuf { #[derive(Debug)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct DmaRxStreamBuf { - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, burst: BurstConfig, } @@ -1010,16 +982,9 @@ impl DmaRxStreamBuf { /// Creates a new [DmaRxStreamBuf] evenly distributing the buffer between /// the provided descriptors. pub fn new( - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + mut descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + mut buffer: DmaAlignedMut<'static, [u8]>, ) -> Result { - if !is_slice_in_dram(descriptors) { - return Err(DmaBufError::UnsupportedMemoryRegion); - } - if !is_slice_in_dram(buffer) { - return Err(DmaBufError::UnsupportedMemoryRegion); - } - // see https://github.com/esp-rs/esp-hal/issues/2269#issuecomment-4397953660 // we can lift that requirement once we sort out this issue if descriptors.len() < 4 { @@ -1057,7 +1022,12 @@ impl DmaRxStreamBuf { } /// Consume the buf, returning the descriptors and buffer. - pub fn split(self) -> (&'static mut [DmaDescriptor], &'static mut [u8]) { + pub fn split( + self, + ) -> ( + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [u8]>, + ) { (self.descriptors, self.buffer) } } @@ -1075,9 +1045,12 @@ unsafe impl DmaRxBuffer for DmaRxStreamBuf { desc.reset_for_rx(); } + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + self.descriptors.writeback(); + Preparation { start: self.descriptors.as_mut_ptr(), - direction: TransferDirection::In, #[cfg(dma_can_access_psram)] accesses_psram: false, burst_transfer: self.burst, @@ -1303,8 +1276,8 @@ impl DmaRxStreamBufView { #[derive(Debug)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct DmaTxStreamBuf { - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, burst: BurstConfig, pre_filled: Option, view_descriptor_idx: usize, @@ -1315,19 +1288,14 @@ impl DmaTxStreamBuf { /// Creates a new [DmaTxStreamBuf] evenly distributing the buffer between /// the provided descriptors. pub fn new( - descriptors: &'static mut [DmaDescriptor], - buffer: &'static mut [u8], + mut descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + mut buffer: DmaAlignedMut<'static, [u8]>, ) -> Result { - match () { - _ if !is_slice_in_dram(descriptors) => Err(DmaBufError::UnsupportedMemoryRegion)?, - _ if !is_slice_in_dram(buffer) => Err(DmaBufError::UnsupportedMemoryRegion)?, - _ if descriptors.len() < 4 => { - // see https://github.com/esp-rs/esp-hal/issues/2269#issuecomment-4397953660 - // we can lift that requirement once we sort out this issue - Err(DmaBufError::InsufficientDescriptors)? - } - _ => (), - }; + if descriptors.len() < 4 { + // see https://github.com/esp-rs/esp-hal/issues/2269#issuecomment-4397953660 + // we can lift that requirement once we sort out this issue + return Err(DmaBufError::InsufficientDescriptors); + } // Evenly distribute the buffer between the descriptors. let chunk_size = Some(buffer.len() / descriptors.len()) @@ -1363,7 +1331,12 @@ impl DmaTxStreamBuf { } /// Consume the buf, returning the descriptors and buffer. - pub fn split(self) -> (&'static mut [DmaDescriptor], &'static mut [u8]) { + pub fn split( + self, + ) -> ( + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [u8]>, + ) { (self.descriptors, self.buffer) } @@ -1394,7 +1367,7 @@ impl DmaTxStreamBuf { self.view_descriptor_offset = 0; let pre_filled = self.pre_filled.unwrap_or(self.buffer.len()); mark_tx_stream_descriptors_ready( - self.descriptors, + &mut self.descriptors, &mut self.view_descriptor_idx, &mut self.view_descriptor_offset, pre_filled, @@ -1407,7 +1380,7 @@ impl DmaTxStreamBuf { /// the `next` pointers because the linked list must remain intact until the /// transfer is running. fn mark_tx_stream_descriptors_ready( - descriptors: &mut [DmaDescriptor], + descriptors: &mut DmaAlignedMut<'_, [DmaDescriptor]>, descriptor_idx: &mut usize, descriptor_offset: &mut usize, bytes_pushed: usize, @@ -1419,6 +1392,9 @@ fn mark_tx_stream_descriptors_ready( let mut bytes_filled = 0; let num_descriptors = descriptors.len(); + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + descriptors.invalidate(); + for i in 0..num_descriptors { let d = (*descriptor_idx + i) % num_descriptors; let desc = &mut descriptors[d]; @@ -1440,10 +1416,13 @@ fn mark_tx_stream_descriptors_ready( desc.set_length(desc.size()); desc.set_suc_eof(true); } + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + descriptors.writeback(); } fn advance_tx_stream_descriptors( - descriptors: &mut [DmaDescriptor], + descriptors: &mut DmaAlignedMut<'_, [DmaDescriptor]>, descriptor_idx: &mut usize, descriptor_offset: &mut usize, bytes_pushed: usize, @@ -1455,6 +1434,9 @@ fn advance_tx_stream_descriptors( let mut bytes_filled = 0; let num_descriptors = descriptors.len(); + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + descriptors.invalidate(); + for i in 0..num_descriptors { let d = (*descriptor_idx + i) % num_descriptors; let desc = &mut descriptors[d]; @@ -1478,6 +1460,9 @@ fn advance_tx_stream_descriptors( prev.next = desc; } } + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + descriptors.writeback(); } unsafe impl DmaTxBuffer for DmaTxStreamBuf { @@ -1494,12 +1479,13 @@ unsafe impl DmaTxBuffer for DmaTxStreamBuf { desc.set_owner(Owner::Dma); next = desc; } + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + self.descriptors.writeback(); self.setup_view_state(); Preparation { start: self.descriptors.as_mut_ptr(), - direction: TransferDirection::Out, #[cfg(dma_can_access_psram)] accesses_psram: false, burst_transfer: self.burst, @@ -1560,7 +1546,7 @@ impl DmaTxStreamBufView { /// Advances the first `n` bytes from the available data pub fn advance(&mut self, bytes_pushed: usize) { advance_tx_stream_descriptors( - self.buf.descriptors, + &mut self.buf.descriptors, &mut self.descriptor_idx, &mut self.descriptor_offset, bytes_pushed, @@ -1589,7 +1575,7 @@ impl DmaTxStreamBufView { } } -static mut EMPTY: [DmaDescriptor; 1] = [DmaDescriptor::EMPTY]; +static mut EMPTY: InternalMemory<[DmaDescriptor; 1]> = InternalMemory::new([DmaDescriptor::EMPTY]); /// An empty buffer that can be used when you don't need to transfer any data. pub struct EmptyBuf; @@ -1600,16 +1586,13 @@ unsafe impl DmaTxBuffer for EmptyBuf { fn prepare(&mut self) -> Preparation { #[cfg(soc_internal_memory_cached)] + #[allow(static_mut_refs)] unsafe { - crate::soc::cache_writeback_addr( - (&raw const EMPTY) as u32, - core::mem::size_of::() as u32, - ); + EMPTY.get_mut().writeback(); } Preparation { start: (&raw mut EMPTY).cast(), - direction: TransferDirection::Out, #[cfg(dma_can_access_psram)] accesses_psram: false, burst_transfer: BurstConfig::default(), @@ -1638,16 +1621,13 @@ unsafe impl DmaRxBuffer for EmptyBuf { fn prepare(&mut self) -> Preparation { #[cfg(soc_internal_memory_cached)] + #[allow(static_mut_refs)] unsafe { - crate::soc::cache_writeback_addr( - (&raw const EMPTY) as u32, - core::mem::size_of::() as u32, - ); + EMPTY.get_mut().writeback(); } Preparation { start: (&raw mut EMPTY).cast(), - direction: TransferDirection::In, #[cfg(dma_can_access_psram)] accesses_psram: false, burst_transfer: BurstConfig::default(), @@ -1679,40 +1659,41 @@ unsafe impl DmaRxBuffer for EmptyBuf { /// it does reading the buffer, which may leave it unable to keep up with the /// bandwidth requirements of some peripherals at high frequencies. pub struct DmaLoopBuf { - descriptor: &'static mut DmaDescriptor, - buffer: &'static mut [u8], + descriptor: DmaAlignedMut<'static, [DmaDescriptor]>, + buffer: DmaAlignedMut<'static, [u8]>, } impl DmaLoopBuf { /// Create a new [DmaLoopBuf]. pub fn new( - descriptor: &'static mut DmaDescriptor, - buffer: &'static mut [u8], + mut descriptors: DmaAlignedMut<'static, [DmaDescriptor]>, + mut buffer: DmaAlignedMut<'static, [u8]>, ) -> Result { - if !is_slice_in_dram(buffer) { - return Err(DmaBufError::UnsupportedMemoryRegion); - } - if !is_slice_in_dram(core::slice::from_ref(descriptor)) { - return Err(DmaBufError::UnsupportedMemoryRegion); - } - - if buffer.len() > BurstConfig::default().max_chunk_size_for(buffer, TransferDirection::Out) + if buffer.len() > BurstConfig::default().max_chunk_size_for(&buffer, TransferDirection::Out) { return Err(DmaBufError::InsufficientDescriptors); } - descriptor.set_owner(Owner::Dma); // Doesn't matter - descriptor.set_suc_eof(false); - descriptor.set_length(buffer.len()); - descriptor.set_size(buffer.len()); - descriptor.buffer = buffer.as_mut_ptr(); - descriptor.next = descriptor; + descriptors[0].set_owner(Owner::Dma); // Doesn't matter + descriptors[0].set_suc_eof(false); + descriptors[0].set_length(buffer.len()); + descriptors[0].set_size(buffer.len()); + descriptors[0].buffer = buffer.as_mut_ptr(); + descriptors[0].next = descriptors.as_mut_ptr(); - Ok(Self { descriptor, buffer }) + Ok(Self { + descriptor: descriptors, + buffer, + }) } /// Consume the buf, returning the descriptor and buffer. - pub fn split(self) -> (&'static mut DmaDescriptor, &'static mut [u8]) { + pub fn split( + self, + ) -> ( + DmaAlignedMut<'static, [DmaDescriptor]>, + DmaAlignedMut<'static, [u8]>, + ) { (self.descriptor, self.buffer) } } @@ -1723,10 +1704,9 @@ unsafe impl DmaTxBuffer for DmaLoopBuf { fn prepare(&mut self) -> Preparation { Preparation { - start: self.descriptor, + start: self.descriptor.as_mut_ptr(), #[cfg(dma_can_access_psram)] accesses_psram: false, - direction: TransferDirection::Out, burst_transfer: BurstConfig::default(), // The DMA must not check the owner bit, as it is never set. check_owner: Some(false), @@ -1749,13 +1729,13 @@ impl Deref for DmaLoopBuf { type Target = [u8]; fn deref(&self) -> &Self::Target { - self.buffer + &self.buffer } } impl DerefMut for DmaLoopBuf { fn deref_mut(&mut self) -> &mut Self::Target { - self.buffer + &mut self.buffer } } @@ -1769,7 +1749,6 @@ impl NoBuffer { fn prep(&self) -> Preparation { Preparation { start: self.0.start, - direction: self.0.direction, #[cfg(dma_can_access_psram)] accesses_psram: self.0.accesses_psram, burst_transfer: self.0.burst_transfer, @@ -1851,6 +1830,7 @@ pub(crate) unsafe fn prepare_for_tx( } } + let descriptors = unsafe { DmaAlignedMut::new_unchecked(descriptors) }; let mut descriptors = unwrap!(DescriptorSet::new(descriptors)); // TODO: it would be best if this function returned the amount of data that could be linked // up. @@ -1862,17 +1842,11 @@ pub(crate) unsafe fn prepare_for_tx( } #[cfg(soc_internal_memory_cached)] - unsafe { - crate::soc::cache_writeback_addr( - descriptors.head() as u32, - core::mem::size_of_val(descriptors.descriptors) as u32, - ); - } + descriptors.descriptors.writeback(); Ok(( NoBuffer(Preparation { start: descriptors.head(), - direction: TransferDirection::Out, burst_transfer: BurstConfig::DEFAULT, check_owner: None, auto_write_back: false, @@ -1913,6 +1887,7 @@ pub(crate) unsafe fn prepare_for_rx( } } + let descriptors = unsafe { DmaAlignedMut::new_unchecked(descriptors) }; let mut descriptors = unwrap!(DescriptorSet::new(descriptors)); let data_len = if data_in_psram { cfg_if::cfg_if! { @@ -1931,21 +1906,6 @@ pub(crate) unsafe fn prepare_for_rx( crate::soc::cache_invalidate_addr(data_addr as u32, consumed_bytes as u32); } - // On chips with cached internal memory, the ManualWritebackBuffer alignment - // buffers live in cached DRAM. Flush the entire struct (including - // dst_address and n_bytes) to memory now so that the post-DMA - // cache_invalidate_addr in write_back() can safely discard the stale - // cache without losing those metadata fields. - #[cfg(soc_internal_memory_cached)] - for buf in align_buffers.iter().flatten() { - unsafe { - crate::soc::cache_writeback_addr( - buf as *const ManualWritebackBuffer as u32, - core::mem::size_of::() as u32, - ); - } - } - consumed_bytes } else { unreachable!() @@ -1973,17 +1933,11 @@ pub(crate) unsafe fn prepare_for_rx( } #[cfg(soc_internal_memory_cached)] - unsafe { - crate::soc::cache_writeback_addr( - descriptors.head() as u32, - core::mem::size_of_val(descriptors.descriptors) as u32, - ); - } + descriptors.descriptors.writeback(); ( NoBuffer(Preparation { start: descriptors.head(), - direction: TransferDirection::In, burst_transfer: BurstConfig::DEFAULT, check_owner: None, auto_write_back: true, @@ -2006,7 +1960,7 @@ fn build_descriptor_list_for_psram( let min_alignment = ExternalBurstConfig::DEFAULT.min_psram_alignment(TransferDirection::In); let chunk_size = 4096 - min_alignment; - let mut desciptor_iter = DescriptorChainingIter::new(descriptors.descriptors); + let mut desciptor_iter = DescriptorChainingIter::new(&mut descriptors.descriptors); let mut copy_buffer_iter = copy_buffers.iter_mut(); // MIN_LAST_DMA_LEN could make this really annoying, so we're just allocating a bit larger @@ -2042,12 +1996,13 @@ fn build_descriptor_list_for_psram( let copy_buffer = unwrap!(copy_buffer_iter.next()); let buffer = copy_buffer.insert(ManualWritebackBuffer::new(get_range(data, 0..head_to_copy))); + buffer.prepare_for_dma(); let Some(descriptor) = desciptor_iter.next() else { return consumed; }; descriptor.set_size(head_to_copy); - descriptor.buffer = buffer.buffer_ptr(); + descriptor.buffer = buffer.mut_buffer_ptr(); consumed += head_to_copy; }; @@ -2072,12 +2027,13 @@ fn build_descriptor_list_for_psram( data, data.len() - tail_to_copy..data.len(), ))); + buffer.prepare_for_dma(); let Some(descriptor) = desciptor_iter.next() else { return consumed; }; descriptor.set_size(tail_to_copy); - descriptor.buffer = buffer.buffer_ptr(); + descriptor.buffer = buffer.mut_buffer_ptr(); consumed += tail_to_copy; } @@ -2137,7 +2093,7 @@ const BUF_LEN: usize = 16 + 2 * (MIN_LAST_DMA_LEN - 1); // 2x makes aligning sho /// the CPU back to PSRAM. #[cfg(dma_can_access_psram)] pub(crate) struct ManualWritebackBuffer { - buffer: InternalMemoryBuffer, + buffer: InternalMemory>, dst_address: NonNull, n_bytes: u8, } @@ -2147,27 +2103,35 @@ impl ManualWritebackBuffer { pub fn new(ptr: NonNull<[u8]>) -> Self { assert!(ptr.len() <= BUF_LEN); Self { - buffer: InternalMemoryBuffer::new(), + buffer: InternalMemory::new(MaybeUninit::uninit()), dst_address: ptr.cast(), n_bytes: ptr.len() as u8, } } - pub fn write_back(&self) { - unsafe { - // The DMA wrote its data directly to memory, bypassing the CPU cache. - // Invalidate the cache lines covering the alignment buffer so the CPU - // reads the fresh DMA data rather than the stale zeros from new(). - #[cfg(soc_internal_memory_cached)] - crate::soc::cache_invalidate_addr(self.buffer_ptr() as u32, BUF_LEN as u32); + pub fn prepare_for_dma(&mut self) { + // Ensure our cache line is not dirty. A dirty cacheline + // evicted during DMA operation can clobber received data. + #[cfg(soc_internal_memory_cached)] + self.buffer.get_mut().invalidate(); + } + pub fn write_back(&mut self) { + // The DMA wrote its data directly to memory, bypassing the CPU cache. + // Invalidate the cache lines covering the alignment buffer so the CPU + // reads the fresh DMA data rather than the stale zeros from new(). + #[cfg(soc_internal_memory_cached)] + self.buffer.get_mut().invalidate(); + + let src = self.mut_buffer_ptr().cast_const(); + unsafe { self.dst_address .as_ptr() - .copy_from(self.buffer_ptr().cast_const(), self.n_bytes as usize); + .copy_from(src, self.n_bytes as usize); } } - pub fn buffer_ptr(&self) -> *mut u8 { - self.buffer.get() as *const [u8] as *mut u8 + pub fn mut_buffer_ptr(&mut self) -> *mut u8 { + self.buffer.get_mut().as_mut_ptr().cast::() } } diff --git a/esp-hal/src/dma/buffers/scoped.rs b/esp-hal/src/dma/buffers/scoped.rs index e684b73d9b8..ff52e0374d5 100644 --- a/esp-hal/src/dma/buffers/scoped.rs +++ b/esp-hal/src/dma/buffers/scoped.rs @@ -4,17 +4,25 @@ use super::*; /// /// This is a contiguous buffer linked together by DMA descriptors of length /// 4095 at most. It can only be used for transmitting data to a peripheral's -/// FIFO. See [DmaRxBuf] for receiving data. +/// FIFO. See [ScopedDmaRxBuf] for receiving data. #[derive(Debug)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] pub(crate) struct ScopedDmaTxBuf<'a> { descriptors: DescriptorSet<'a>, - buffer: &'a mut [u8], + buffer: DmaAlignedMut<'a, [u8]>, burst: BurstConfig, } impl<'a> ScopedDmaTxBuf<'a> { /// Creates a new [ScopedDmaTxBuf] from some descriptors and a buffer. + pub fn new( + descriptors: DmaAlignedMut<'a, [DmaDescriptor]>, + buffer: DmaAlignedMut<'a, [u8]>, + ) -> Result { + Self::new_with_config(descriptors, buffer, BurstConfig::default()) + } + + /// Creates a new [ScopedDmaTxBuf] with a caller-chosen minimum alignment. /// /// There must be enough descriptors for the provided buffer. /// Depending on alignment requirements, each descriptor can handle at most @@ -23,8 +31,8 @@ impl<'a> ScopedDmaTxBuf<'a> { /// Both the descriptors and buffer must be in DMA-capable memory. /// Only DRAM is supported for descriptors. pub fn new_with_config( - descriptors: &'a mut [DmaDescriptor], - buffer: &'a mut [u8], + descriptors: DmaAlignedMut<'a, [DmaDescriptor]>, + buffer: DmaAlignedMut<'a, [u8]>, config: impl Into, ) -> Result { let mut buf = Self { @@ -47,10 +55,9 @@ impl<'a> ScopedDmaTxBuf<'a> { let burst = burst.into(); self.set_length_fallible(length, burst)?; - self.descriptors.link_with_buffer( - self.buffer, - burst.max_chunk_size_for(self.buffer, TransferDirection::Out), - )?; + let max_chunk_size = burst.max_chunk_size_for(&self.buffer, TransferDirection::Out); + self.descriptors + .link_with_buffer(&mut self.buffer, max_chunk_size)?; self.burst = burst; Ok(()) @@ -63,8 +70,8 @@ impl<'a> ScopedDmaTxBuf<'a> { } /// Consume the buf, returning the descriptors and buffer. - pub fn split(self) -> (&'a mut [DmaDescriptor], &'a mut [u8]) { - (self.descriptors.into_inner(), self.buffer) + pub fn split(self) -> (DmaAlignedMut<'a, [DmaDescriptor]>, DmaAlignedMut<'a, [u8]>) { + (self.descriptors.descriptors, self.buffer) } /// Returns the size of the underlying buffer @@ -87,10 +94,8 @@ impl<'a> ScopedDmaTxBuf<'a> { } burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::Out)?; - self.descriptors.set_tx_length( - len, - burst.max_chunk_size_for(self.buffer, TransferDirection::Out), - )?; + let max_chunk_size = burst.max_chunk_size_for(&self.buffer, TransferDirection::Out); + self.descriptors.set_tx_length(len, max_chunk_size)?; // This only needs to be done once (after every significant length change) as // Self::prepare sets Preparation::auto_write_back to false. @@ -124,12 +129,12 @@ impl<'a> ScopedDmaTxBuf<'a> { /// Returns the buf as a mutable slice than can be written. pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.buffer + &mut self.buffer } /// Returns the buf as a slice than can be read. pub fn as_slice(&self) -> &[u8] { - self.buffer + &self.buffer } } @@ -139,37 +144,16 @@ unsafe impl<'a> DmaTxBuffer for ScopedDmaTxBuf<'a> { fn prepare(&mut self) -> Preparation { #[cfg(soc_internal_memory_cached)] - unsafe { - crate::soc::cache_writeback_addr( - self.descriptors.head() as u32, - core::mem::size_of_val(self.descriptors.descriptors) as u32, - ); - } + self.descriptors.descriptors.writeback(); - cfg_if::cfg_if! { - if #[cfg(dma_can_access_psram)] { - let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); - if is_data_in_psram || cfg!(soc_internal_memory_cached) { - unsafe { - crate::soc::cache_writeback_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } else if #[cfg(soc_internal_memory_cached)] { - unsafe { - crate::soc::cache_writeback_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } + #[cfg(dma_can_access_psram)] + let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + self.buffer.writeback(); Preparation { start: self.descriptors.head(), - direction: TransferDirection::Out, #[cfg(dma_can_access_psram)] accesses_psram: is_data_in_psram, burst_transfer: self.burst, @@ -196,11 +180,19 @@ unsafe impl<'a> DmaTxBuffer for ScopedDmaTxBuf<'a> { #[cfg_attr(feature = "defmt", derive(defmt::Format))] pub(crate) struct ScopedDmaRxBuf<'a> { descriptors: DescriptorSet<'a>, - buffer: &'a mut [u8], + buffer: DmaAlignedMut<'a, [u8]>, burst: BurstConfig, } impl<'a> ScopedDmaRxBuf<'a> { + /// Creates a new [ScopedDmaRxBuf] from some descriptors and a buffer. + pub fn new( + descriptors: DmaAlignedMut<'a, [DmaDescriptor]>, + buffer: DmaAlignedMut<'a, [u8]>, + ) -> Result { + Self::new_with_config(descriptors, buffer, BurstConfig::default()) + } + /// Creates a new [ScopedDmaRxBuf] from some descriptors and a buffer. /// /// There must be enough descriptors for the provided buffer. @@ -210,8 +202,8 @@ impl<'a> ScopedDmaRxBuf<'a> { /// Both the descriptors and buffer must be in DMA-capable memory. /// Only DRAM is supported for descriptors. pub fn new_with_config( - descriptors: &'a mut [DmaDescriptor], - buffer: &'a mut [u8], + descriptors: DmaAlignedMut<'a, [DmaDescriptor]>, + buffer: DmaAlignedMut<'a, [u8]>, config: impl Into, ) -> Result { let mut buf = Self { @@ -233,10 +225,9 @@ impl<'a> ScopedDmaRxBuf<'a> { let burst = burst.into(); self.set_length_fallible(length, burst)?; - self.descriptors.link_with_buffer( - self.buffer, - burst.max_chunk_size_for(self.buffer, TransferDirection::In), - )?; + let max_chunk_size = burst.max_chunk_size_for(&self.buffer, TransferDirection::In); + self.descriptors + .link_with_buffer(&mut self.buffer, max_chunk_size)?; self.burst = burst; Ok(()) @@ -249,8 +240,8 @@ impl<'a> ScopedDmaRxBuf<'a> { } /// Consume the buf, returning the descriptors and buffer. - pub fn split(self) -> (&'a mut [DmaDescriptor], &'a mut [u8]) { - (self.descriptors.into_inner(), self.buffer) + pub fn split(self) -> (DmaAlignedMut<'a, [DmaDescriptor]>, DmaAlignedMut<'a, [u8]>) { + (self.descriptors.descriptors, self.buffer) } /// Returns the size of the underlying buffer @@ -291,12 +282,12 @@ impl<'a> ScopedDmaRxBuf<'a> { /// Returns the entire underlying buffer as a slice than can be read. pub fn as_slice(&self) -> &[u8] { - self.buffer + &self.buffer } /// Returns the entire underlying buffer as a slice than can be written. pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.buffer + &mut self.buffer } /// Return the number of bytes that was received by this buf. @@ -351,38 +342,16 @@ unsafe impl<'a> DmaRxBuffer for ScopedDmaRxBuf<'a> { } #[cfg(soc_internal_memory_cached)] - unsafe { - crate::soc::cache_writeback_addr( - self.descriptors.head() as u32, - core::mem::size_of_val(self.descriptors.descriptors) as u32, - ); - } + self.descriptors.descriptors.writeback(); - cfg_if::cfg_if! { - if #[cfg(dma_can_access_psram)] { - // Optimization: avoid locking for PSRAM range. - let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); - if is_data_in_psram || cfg!(soc_internal_memory_cached) { - unsafe { - crate::soc::cache_invalidate_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } else if #[cfg(soc_internal_memory_cached)] { - unsafe { - crate::soc::cache_invalidate_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } + #[cfg(dma_can_access_psram)] + let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); + + #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))] + self.buffer.invalidate(); Preparation { start: self.descriptors.head(), - direction: TransferDirection::In, #[cfg(dma_can_access_psram)] accesses_psram: is_data_in_psram, burst_transfer: self.burst, diff --git a/esp-hal/src/dma/engine/mod.rs b/esp-hal/src/dma/engine/mod.rs index 760e0a7766c..a97a35f84d3 100644 --- a/esp-hal/src/dma/engine/mod.rs +++ b/esp-hal/src/dma/engine/mod.rs @@ -20,19 +20,19 @@ for_each_dma_engine! { }; ("COPY_DMA") => { mod copy; - pub use copy::{CopyDmaChannel, CopyDmaRxChannel, CopyDmaTxChannel}; + pub use copy::*; }; ("CRYPTO_DMA") => { mod crypto; - pub use crypto::{CryptoDmaChannel, CryptoDmaRxChannel, CryptoDmaTxChannel}; + pub use crypto::*; }; ("I2S_DMA") => { mod i2s; - pub use i2s::{I2sDmaChannel, I2sDmaRxChannel, I2sDmaTxChannel}; + pub use i2s::*; }; ("SPI_DMA") => { mod spi; - pub use spi::{SpiDmaChannel, SpiDmaRxChannel, SpiDmaTxChannel}; + pub use spi::*; }; } diff --git a/esp-hal/src/dma/m2m.rs b/esp-hal/src/dma/m2m.rs index bcbac5f8298..65abc712d09 100644 --- a/esp-hal/src/dma/m2m.rs +++ b/esp-hal/src/dma/m2m.rs @@ -26,6 +26,7 @@ use crate::{ DmaTxBuf, DmaTxBuffer, DmaTxInterrupt, + aligned::DmaAlignedMut, }, }; @@ -601,8 +602,8 @@ where enum State<'d, Dm: DriverMode> { Idle( Mem2Mem<'d, Dm>, - &'d mut [DmaDescriptor], - &'d mut [DmaDescriptor], + DmaAlignedMut<'d, [DmaDescriptor]>, + DmaAlignedMut<'d, [DmaDescriptor]>, ), Active( Mem2MemRxTransfer<'d, DmaRxBuf, Dm>, @@ -625,6 +626,12 @@ where if rx_descriptors.is_empty() || tx_descriptors.is_empty() { return Err(DmaError::OutOfDescriptors); } + + // Safety: descriptors are aligned to what the DMA requires, and we don't call invalidate on + // these slices. + let rx_descriptors = unsafe { DmaAlignedMut::new_unchecked(rx_descriptors) }; + let tx_descriptors = unsafe { DmaAlignedMut::new_unchecked(tx_descriptors) }; + Ok(Self { state: State::Idle(mem2mem, rx_descriptors, tx_descriptors), config, @@ -642,7 +649,7 @@ where rx_buffer: &mut [u8], tx_buffer: &[u8], ) -> Result, DmaError> { - let State::Idle(mem2mem, rx_descriptors, tx_descriptors) = + let State::Idle(mem2mem, mut rx_descriptors, mut tx_descriptors) = core::mem::replace(&mut self.state, State::InUse) else { panic!("SimpleMem2MemTransfer was forgotten with core::mem::forget or similar"); @@ -652,15 +659,26 @@ where // the user calls core::mem::forget on SimpleMem2MemTransfer. This is // just the unfortunate consequence of doing DMA without enforcing // 'static. - let rx_buffer = - unsafe { core::slice::from_raw_parts_mut(rx_buffer.as_mut_ptr(), rx_buffer.len()) }; - let tx_buffer = - unsafe { core::slice::from_raw_parts_mut(tx_buffer.as_ptr() as _, tx_buffer.len()) }; + let rx_buffer = DmaAlignedMut::new(unsafe { + core::slice::from_raw_parts_mut(rx_buffer.as_mut_ptr(), rx_buffer.len()) + })?; + let tx_buffer = unsafe { + DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut( + tx_buffer.as_ptr() as _, + tx_buffer.len(), + )) + }; let rx_descriptors = unsafe { - core::slice::from_raw_parts_mut(rx_descriptors.as_mut_ptr(), rx_descriptors.len()) + DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut( + rx_descriptors.as_mut_ptr(), + rx_descriptors.len(), + )) }; let tx_descriptors = unsafe { - core::slice::from_raw_parts_mut(tx_descriptors.as_mut_ptr(), tx_descriptors.len()) + DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut( + tx_descriptors.as_mut_ptr(), + tx_descriptors.len(), + )) }; // Note: The ESP32-S2 insists that RX is started before TX. Contrary to the TRM diff --git a/esp-hal/src/dma/mod.rs b/esp-hal/src/dma/mod.rs index a7aab3e4304..87d77510204 100644 --- a/esp-hal/src/dma/mod.rs +++ b/esp-hal/src/dma/mod.rs @@ -48,10 +48,9 @@ //! ⚠️ Note: Descriptors should be sized as `(max_transfer_size + CHUNK_SIZE - 1) / CHUNK_SIZE`. //! I.e., to transfer buffers of size `1..=CHUNK_SIZE`, you need 1 descriptor. //! -//! ⚠️ Note: For chips that support DMA to/from PSRAM (ESP32-S3) DMA transfers to/from PSRAM +//! ⚠️ Note: For chips that support DMA to/from PSRAM DMA transfers to/from PSRAM //! have extra alignment requirements. The address and size of the buffer pointed to by -//! each descriptor must be a multiple of the cache line (block) size. This is 32 bytes -//! on ESP32-S3. +//! each descriptor must be a multiple of the cache line (block) size. //! //! For convenience you can use the [crate::dma_buffers] macro. @@ -66,11 +65,12 @@ use crate::{ Async, Blocking, DriverMode, + dma::aligned::DmaAlignedMut, interrupt::InterruptHandler, - soc::is_slice_in_dram, system::{Cpu, PeripheralGuard}, }; +pub mod aligned; mod buffers; #[cfg(dma_supports_mem2mem)] mod m2m; @@ -320,7 +320,7 @@ macro_rules! dma_buffers { $crate::dma_buffers_chunk_size!($rx_size, $tx_size, $crate::dma::CHUNK_SIZE) }; ($size:expr) => { - $crate::dma_buffers_chunk_size!($size, $crate::dma::CHUNK_SIZE) + $crate::dma_buffers!($size, $size) }; } @@ -346,7 +346,7 @@ macro_rules! dma_descriptors { }; ($size:expr) => { - $crate::dma_descriptors_chunk_size!($size, $size, $crate::dma::CHUNK_SIZE) + $crate::dma_descriptors!($size, $size) }; } @@ -369,7 +369,16 @@ macro_rules! dma_descriptors { #[cfg(feature = "unstable")] #[cfg_attr(docsrs, doc(cfg(feature = "unstable")))] macro_rules! dma_buffers_chunk_size { - ($rx_size:expr, $tx_size:expr, $chunk_size:expr) => {{ $crate::dma_buffers_impl!($rx_size, $tx_size, $chunk_size) }}; + ($rx_size:expr, $tx_size:expr, $chunk_size:expr) => {{ + let (rx_buf, rx_desc, tx_buf, tx_desc) = + $crate::dma_buffers_impl!($rx_size, $tx_size, $chunk_size); + ( + rx_buf.into_inner(), + rx_desc.into_inner(), + tx_buf.into_inner(), + tx_desc.into_inner(), + ) + }}; ($size:expr, $chunk_size:expr) => { $crate::dma_buffers_chunk_size!($size, $size, $chunk_size) @@ -393,60 +402,16 @@ macro_rules! dma_buffers_chunk_size { #[cfg(feature = "unstable")] #[cfg_attr(docsrs, doc(cfg(feature = "unstable")))] macro_rules! dma_descriptors_chunk_size { - ($rx_size:expr, $tx_size:expr, $chunk_size:expr) => {{ $crate::dma_descriptors_impl!($rx_size, $tx_size, $chunk_size) }}; + ($rx_size:expr, $tx_size:expr, $chunk_size:expr) => {{ + let (rx, tx) = $crate::dma_descriptors_impl!($rx_size, $tx_size, $chunk_size); + (rx.into_inner(), tx.into_inner()) + }}; ($size:expr, $chunk_size:expr) => { $crate::dma_descriptors_chunk_size!($size, $size, $chunk_size) }; } -// ESP32-P4 internal memory is cached, enforce alignment to avoid memory -// corruption. Technically only needed for IN buffers and descriptor lists. -#[cfg_attr(soc_internal_memory_cached, repr(C, align(64)))] // dcache cache line -#[doc(hidden)] -pub struct InternalMemoryCachelineAligned(T); - -impl InternalMemoryCachelineAligned { - pub const fn new(init: T) -> Self { - Self(init) - } - - pub const fn get(&self) -> &T { - &self.0 - } - - pub const fn get_mut(&mut self) -> &mut T { - &mut self.0 - } -} - -// ESP32 requires word alignment for DMA buffers. -// ESP32-S2 technically supports byte-aligned DMA buffers, but the -// transfer ends up writing out of bounds. -#[cfg_attr(not(soc_internal_memory_cached), repr(C, align(4)))] -#[doc(hidden)] -pub struct InternalMemoryBuffer(InternalMemoryCachelineAligned<[u8; N]>); - -impl Default for InternalMemoryBuffer { - fn default() -> Self { - Self::new() - } -} - -impl InternalMemoryBuffer { - pub const fn new() -> Self { - Self(InternalMemoryCachelineAligned::new([0u8; N])) - } - - pub const fn get(&self) -> &[u8] { - self.0.get() - } - - pub const fn get_mut(&mut self) -> &mut [u8] { - self.0.get_mut() - } -} - #[doc(hidden)] #[macro_export] #[cfg(feature = "unstable")] @@ -463,11 +428,11 @@ macro_rules! dma_buffers_impl { ( { #[allow(unused_braces)] - static mut BUFFER: $crate::dma::InternalMemoryBuffer<{ $size }> = - $crate::dma::InternalMemoryBuffer::new(); + static mut BUFFER: $crate::dma::aligned::InternalMemory<[u8; { $size }]> = + $crate::dma::aligned::InternalMemory::new([0; $size]); // SAFETY: The ConstStaticCell in the descriptor part ensures there will only // be a single mutable reference to this buffer. - unsafe { BUFFER.get_mut() } + unsafe { BUFFER.get_mut().unsize() } }, $crate::dma_descriptors_impl!($size, $chunk_size), ) @@ -496,18 +461,17 @@ macro_rules! dma_descriptors_impl { ($size:expr, $chunk_size:expr) => {{ use $crate::{ __macro_implementation::static_cell::ConstStaticCell, - dma::{DmaDescriptor, InternalMemoryCachelineAligned}, + dma::{DmaDescriptor, aligned::InternalMemory}, }; - const COUNT: usize = $crate::dma_descriptor_count!($size, $chunk_size); - + const __DMA_DESCRIPTOR_COUNT: usize = $crate::dma_descriptor_count!($size, $chunk_size); static DESCRIPTORS: ConstStaticCell< - InternalMemoryCachelineAligned<[DmaDescriptor; COUNT]>, - > = ConstStaticCell::new(InternalMemoryCachelineAligned::new( - [DmaDescriptor::EMPTY; COUNT], + InternalMemory<[DmaDescriptor; __DMA_DESCRIPTOR_COUNT]>, + > = ConstStaticCell::new(InternalMemory::new( + [DmaDescriptor::EMPTY; __DMA_DESCRIPTOR_COUNT], )); - DESCRIPTORS.take().get_mut() + DESCRIPTORS.take().get_mut().unsize() }}; } @@ -531,6 +495,30 @@ macro_rules! dma_descriptor_count { }}; } +#[procmacros::doc_replace] +/// Convenience macro to create a DmaRxBuf from buffer size. The buffer and +/// descriptors are statically allocated and used to create the `DmaRxBuf`. +/// +/// ## Usage +/// ```rust,no_run +/// # {before_snippet} +/// use esp_hal::dma_rx_buffer; +/// +/// let rx_buf = dma_rx_buffer!(32000)?; +/// # {after_snippet} +/// ``` +#[macro_export] +#[cfg(feature = "unstable")] +#[cfg_attr(docsrs, doc(cfg(feature = "unstable")))] +macro_rules! dma_rx_buffer { + ($rx_size:expr) => {{ + let (rx_buffer, rx_descriptors) = $crate::dma_buffers_impl!($rx_size); + + // TODO: this macro should be infallible + $crate::dma::DmaRxBuf::new(rx_descriptors, rx_buffer) + }}; +} + #[procmacros::doc_replace] /// Convenience macro to create a DmaTxBuf from buffer size. The buffer and /// descriptors are statically allocated and used to create the `DmaTxBuf`. @@ -540,7 +528,7 @@ macro_rules! dma_descriptor_count { /// # {before_snippet} /// use esp_hal::dma_tx_buffer; /// -/// let tx_buf = dma_tx_buffer!(32000); +/// let tx_buf = dma_tx_buffer!(32000)?; /// # {after_snippet} /// ``` #[macro_export] @@ -550,6 +538,7 @@ macro_rules! dma_tx_buffer { ($tx_size:expr) => {{ let (tx_buffer, tx_descriptors) = $crate::dma_buffers_impl!($tx_size); + // TODO: this macro should be infallible $crate::dma::DmaTxBuf::new(tx_descriptors, tx_buffer) }}; } @@ -639,7 +628,7 @@ macro_rules! dma_loop_buffer { let (buffer, descriptors) = $crate::dma_buffers_impl!($size, $size); - $crate::dma::DmaLoopBuf::new(&mut descriptors[0], buffer).unwrap() + $crate::dma::DmaLoopBuf::new(descriptors, buffer).unwrap() }}; } @@ -735,26 +724,27 @@ pub const fn descriptor_count(buffer_size: usize, chunk_size: usize) -> usize { return 1; } + // TODO: we could round up to cacheline size to reduce memory waste on + // soc_internal_memory_cached devices. buffer_size.div_ceil(chunk_size) } #[derive(Debug)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] struct DescriptorSet<'a> { - descriptors: &'a mut [DmaDescriptor], + descriptors: DmaAlignedMut<'a, [DmaDescriptor]>, } impl<'a> DescriptorSet<'a> { /// Creates a new `DescriptorSet` from a slice of descriptors and associates /// them with the given buffer. - fn new(descriptors: &'a mut [DmaDescriptor]) -> Result { - if !is_slice_in_dram(descriptors) { + fn new(descriptors: DmaAlignedMut<'a, [DmaDescriptor]>) -> Result { + #[cfg(not(esp32p4))] // P4 can read descriptors from PSRAM + if !crate::soc::is_slice_in_dram(&descriptors) { return Err(DmaBufError::UnsupportedMemoryRegion); } - descriptors.fill(DmaDescriptor::EMPTY); - - Ok(unsafe { Self::new_unchecked(descriptors) }) + Ok(unsafe { Self::new_unchecked(descriptors.into_inner()) }) } /// Creates a new `DescriptorSet` from a slice of descriptors and associates @@ -765,11 +755,14 @@ impl<'a> DescriptorSet<'a> { /// The caller must ensure that the descriptors are located in a supported /// memory region. unsafe fn new_unchecked(descriptors: &'a mut [DmaDescriptor]) -> Self { - Self { descriptors } + descriptors.fill(DmaDescriptor::EMPTY); + Self { + descriptors: unsafe { DmaAlignedMut::new_unchecked(descriptors) }, + } } /// Consumes the `DescriptorSet` and returns the inner slice of descriptors. - fn into_inner(self) -> &'a mut [DmaDescriptor] { + fn into_inner(self) -> DmaAlignedMut<'a, [DmaDescriptor]> { self.descriptors } @@ -814,7 +807,7 @@ impl<'a> DescriptorSet<'a> { buffer: &mut [u8], chunk_size: usize, ) -> Result<(), DmaBufError> { - Self::set_up_buffer_ptrs(buffer, self.descriptors, chunk_size) + Self::set_up_buffer_ptrs(buffer, &mut self.descriptors, chunk_size) } /// Prepares descriptors for transferring `len` bytes of data. @@ -826,7 +819,7 @@ impl<'a> DescriptorSet<'a> { chunk_size: usize, prepare: fn(&mut DmaDescriptor, usize), ) -> Result<(), DmaBufError> { - Self::set_up_descriptors(self.descriptors, len, chunk_size, prepare) + Self::set_up_descriptors(&mut self.descriptors, len, chunk_size, prepare) } /// Prepares descriptors for reading `len` bytes of data. @@ -1046,8 +1039,6 @@ where preparation: Preparation, peri: DmaPeripheral, ) -> Result<(), DmaError> { - debug_assert_eq!(preparation.direction, TransferDirection::In); - debug!("Preparing RX transfer {:?}", preparation); trace!("First descriptor {:?}", unsafe { &*preparation.start }); @@ -1272,8 +1263,6 @@ where preparation: Preparation, peri: DmaPeripheral, ) -> Result<(), DmaError> { - debug_assert_eq!(preparation.direction, TransferDirection::Out); - debug!("Preparing TX transfer {:?}", preparation); trace!("First descriptor {:?}", unsafe { &*preparation.start }); diff --git a/esp-hal/src/i2s/parallel.rs b/esp-hal/src/i2s/parallel.rs index f4cb21c6bc0..28229d75669 100644 --- a/esp-hal/src/i2s/parallel.rs +++ b/esp-hal/src/i2s/parallel.rs @@ -39,7 +39,7 @@ //! ```rust, no_run //! # {before_snippet} //! # use esp_hal::dma::DmaTxBuf; -//! # use esp_hal::dma_buffers; +//! # use esp_hal::dma_tx_buffer; //! # use esp_hal::delay::Delay; //! # use esp_hal::i2s::parallel::{I2sParallel, TxEightBits}; //! @@ -61,11 +61,11 @@ //! peripherals.GPIO14, //! ); //! -//! let (_, _, tx_buffer, tx_descriptors) = dma_buffers!(0, BUFFER_SIZE); //! let mut parallel = //! I2sParallel::new(i2s, dma_channel, Rate::from_mhz(1), pins, clock).into_async(); //! -//! for (i, data) in tx_buffer.chunks_mut(4).enumerate() { +//! let mut tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); +//! for (i, data) in tx_buf.as_mut_slice().chunks_mut(4).enumerate() { //! let offset = i * 4; //! // i2s parallel driver expects the buffer to be interleaved //! data[0] = (offset + 2) as u8; @@ -74,9 +74,6 @@ //! data[3] = (offset + 1) as u8; //! } //! -//! let mut tx_buf: DmaTxBuf = -//! DmaTxBuf::new(tx_descriptors, tx_buffer).expect("DmaTxBuf::new failed"); -//! //! // Sending 256 bytes. //! loop { //! let xfer = match parallel.send(tx_buf) { diff --git a/esp-hal/src/parl_io.rs b/esp-hal/src/parl_io.rs index 118017d7d6b..4a94dd89945 100644 --- a/esp-hal/src/parl_io.rs +++ b/esp-hal/src/parl_io.rs @@ -17,14 +17,13 @@ //! ```rust, no_run //! # {before_snippet} //! # use esp_hal::delay::Delay; -//! # use esp_hal::dma_buffers; +//! # use esp_hal::dma_rx_buffer; //! # use esp_hal::dma::DmaRxBuf; //! # use esp_hal::gpio::NoPin; //! # use esp_hal::parl_io::{BitPackOrder, ParlIo, RxConfig, RxFourBits}; //! //! // Initialize DMA buffer and descriptors for data reception -//! let (rx_buffer, rx_descriptors, _, _) = dma_buffers!(32000, 0); -//! let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer)?; +//! let mut dma_rx_buf = dma_rx_buffer!(32000)?; //! let dma_channel = peripherals.DMA_CH0; //! //! // Configure the 4-bit input pins and clock pin diff --git a/esp-hal/src/soc/esp32s2/mod.rs b/esp-hal/src/soc/esp32s2/mod.rs index 153abd0077d..167a0b51105 100644 --- a/esp-hal/src/soc/esp32s2/mod.rs +++ b/esp-hal/src/soc/esp32s2/mod.rs @@ -50,6 +50,25 @@ pub unsafe fn cache_invalidate_addr(addr: u32, size: u32) { } } +pub(crate) const CONFIG_INSTRUCTION_CACHE_SIZE: usize = cfg_select! { + instruction_cache_size_8kb => 0, + instruction_cache_size_16kb => 1, +}; +pub(crate) const CONFIG_INSTRUCTION_CACHE_LINE_SIZE: usize = cfg_select! { + instruction_cache_line_size_16b => 0, + instruction_cache_line_size_32b => 1, +}; + +pub(crate) const CONFIG_DATA_CACHE_SIZE: usize = cfg_select! { + data_cache_size_0kb => 0, // doesn't matter according to esp-idf + data_cache_size_8kb => 0, + data_cache_size_16kb => 1, +}; +pub(crate) const CONFIG_DATA_CACHE_LINE_SIZE: usize = cfg_select! { + data_cache_line_size_16b => 0, + data_cache_line_size_32b => 1, +}; + #[crate::ram] pub(crate) unsafe fn configure_cpu_caches() { // Set up caches. Doesn't work when put in `configure_cpu_caches`. @@ -107,29 +126,6 @@ pub(crate) unsafe fn configure_cpu_caches() { const CACHE_4WAYS_ASSOC: u32 = 0; - const CONFIG_ESP32S2_INSTRUCTION_CACHE_SIZE: u32 = match () { - _ if cfg!(instruction_cache_size_8kb) => 0, - _ if cfg!(instruction_cache_size_16kb) => 1, - _ => core::unreachable!(), - }; - const CONFIG_ESP32S2_INSTRUCTION_CACHE_LINE_SIZE: u32 = match () { - _ if cfg!(instruction_cache_line_size_16b) => 0, - _ if cfg!(instruction_cache_line_size_32b) => 1, - _ => core::unreachable!(), - }; - - const CONFIG_ESP32S2_DATA_CACHE_SIZE: u32 = match () { - _ if cfg!(data_cache_size_0kb) => 0, // doesn't matter according to esp-idf - _ if cfg!(data_cache_size_8kb) => 0, - _ if cfg!(data_cache_size_16kb) => 1, - _ => core::unreachable!(), - }; - const CONFIG_ESP32S2_DATA_CACHE_LINE_SIZE: u32 = match () { - _ if cfg!(data_cache_line_size_16b) => 0, - _ if cfg!(data_cache_line_size_32b) => 1, - _ => core::unreachable!(), - }; - #[derive(Clone, Copy, Debug)] enum CacheLayout { Invalid = 0, @@ -171,17 +167,17 @@ pub(crate) unsafe fn configure_cpu_caches() { unsafe { Cache_Set_ICache_Mode( - CONFIG_ESP32S2_INSTRUCTION_CACHE_SIZE, + CONFIG_INSTRUCTION_CACHE_SIZE as u32, CACHE_4WAYS_ASSOC, - CONFIG_ESP32S2_INSTRUCTION_CACHE_LINE_SIZE, + CONFIG_INSTRUCTION_CACHE_LINE_SIZE as u32, ); Cache_Invalidate_ICache_All(); Cache_Resume_ICache(0); Cache_Set_DCache_Mode( - CONFIG_ESP32S2_DATA_CACHE_SIZE, + CONFIG_DATA_CACHE_SIZE as u32, CACHE_4WAYS_ASSOC, - CONFIG_ESP32S2_DATA_CACHE_LINE_SIZE, + CONFIG_DATA_CACHE_LINE_SIZE as u32, ); Cache_Invalidate_DCache_All(); Cache_Enable_DCache(0); diff --git a/esp-hal/src/soc/esp32s3/mod.rs b/esp-hal/src/soc/esp32s3/mod.rs index ac0fecacdf9..3b80062ba77 100644 --- a/esp-hal/src/soc/esp32s3/mod.rs +++ b/esp-hal/src/soc/esp32s3/mod.rs @@ -23,6 +23,34 @@ pub(crate) fn i2s_sclk_frequency() -> u32 { clocks::pll_160m_frequency() } +pub(crate) const CONFIG_INSTRUCTION_CACHE_SIZE: usize = cfg_select! { + instruction_cache_size_32kb => 0x8000, + instruction_cache_size_16kb => 0x4000, +}; +pub(crate) const CONFIG_ICACHE_ASSOCIATED_WAYS: usize = cfg_select! { + icache_associated_ways_8 => 8, + icache_associated_ways_4 => 4, +}; +pub(crate) const CONFIG_INSTRUCTION_CACHE_LINE_SIZE: usize = cfg_select! { + instruction_cache_line_size_32b => 32, + instruction_cache_line_size_16b => 16, +}; + +pub(crate) const CONFIG_DATA_CACHE_SIZE: usize = cfg_select! { + data_cache_size_64kb => 0x10000, + data_cache_size_32kb => 0x8000, + data_cache_size_16kb => 0x4000, +}; +pub(crate) const CONFIG_DCACHE_ASSOCIATED_WAYS: usize = cfg_select! { + dcache_associated_ways_8 => 8, + dcache_associated_ways_4 => 4, +}; +pub(crate) const CONFIG_DATA_CACHE_LINE_SIZE: usize = cfg_select! { + data_cache_line_size_64b => 64, + data_cache_line_size_32b => 32, + data_cache_line_size_16b => 16, +}; + #[unsafe(link_section = ".rwtext")] pub(crate) unsafe fn configure_cpu_caches() { // this is just the bare minimum we need to run code from flash @@ -47,46 +75,12 @@ pub(crate) unsafe fn configure_cpu_caches() { ); } - const CONFIG_ESP32S3_INSTRUCTION_CACHE_SIZE: u32 = match () { - _ if cfg!(instruction_cache_size_32kb) => 0x8000, - _ if cfg!(instruction_cache_size_16kb) => 0x4000, - _ => core::unreachable!(), - }; - const CONFIG_ESP32S3_ICACHE_ASSOCIATED_WAYS: u8 = match () { - _ if cfg!(icache_associated_ways_8) => 8, - _ if cfg!(icache_associated_ways_4) => 4, - _ => core::unreachable!(), - }; - const CONFIG_ESP32S3_INSTRUCTION_CACHE_LINE_SIZE: u8 = match () { - _ if cfg!(instruction_cache_line_size_32b) => 32, - _ if cfg!(instruction_cache_line_size_16b) => 16, - _ => core::unreachable!(), - }; - - const CONFIG_ESP32S3_DATA_CACHE_SIZE: u32 = match () { - _ if cfg!(data_cache_size_64kb) => 0x10000, - _ if cfg!(data_cache_size_32kb) => 0x8000, - _ if cfg!(data_cache_size_16kb) => 0x4000, - _ => core::unreachable!(), - }; - const CONFIG_ESP32S3_DCACHE_ASSOCIATED_WAYS: u8 = match () { - _ if cfg!(dcache_associated_ways_8) => 8, - _ if cfg!(dcache_associated_ways_4) => 4, - _ => core::unreachable!(), - }; - const CONFIG_ESP32S3_DATA_CACHE_LINE_SIZE: u8 = match () { - _ if cfg!(data_cache_line_size_64b) => 64, - _ if cfg!(data_cache_line_size_32b) => 32, - _ if cfg!(data_cache_line_size_16b) => 16, - _ => core::unreachable!(), - }; - // Configure the mode of instruction cache: cache size, cache line size. unsafe { rom_config_instruction_cache_mode( - CONFIG_ESP32S3_INSTRUCTION_CACHE_SIZE, - CONFIG_ESP32S3_ICACHE_ASSOCIATED_WAYS, - CONFIG_ESP32S3_INSTRUCTION_CACHE_LINE_SIZE, + CONFIG_INSTRUCTION_CACHE_SIZE as u32, + CONFIG_ICACHE_ASSOCIATED_WAYS as u8, + CONFIG_INSTRUCTION_CACHE_LINE_SIZE as u8, ); } @@ -94,9 +88,9 @@ pub(crate) unsafe fn configure_cpu_caches() { unsafe { Cache_Suspend_DCache(); rom_config_data_cache_mode( - CONFIG_ESP32S3_DATA_CACHE_SIZE, - CONFIG_ESP32S3_DCACHE_ASSOCIATED_WAYS, - CONFIG_ESP32S3_DATA_CACHE_LINE_SIZE, + CONFIG_DATA_CACHE_SIZE as u32, + CONFIG_DCACHE_ASSOCIATED_WAYS as u8, + CONFIG_DATA_CACHE_LINE_SIZE as u8, ); Cache_Resume_DCache(0); } diff --git a/esp-hal/src/spi/master/dma.rs b/esp-hal/src/spi/master/dma.rs index b10d2a526ca..ab7239df21f 100644 --- a/esp-hal/src/spi/master/dma.rs +++ b/esp-hal/src/spi/master/dma.rs @@ -15,8 +15,6 @@ use enumset::EnumSet; use procmacros::ram; use super::*; -#[cfg(all(spi_master_version = "1", spi_address_workaround))] -use crate::dma::InternalMemoryBuffer; use crate::{ RegisterToggle, dma::{ @@ -28,11 +26,11 @@ use crate::{ DmaRxBuffer, DmaTxBuf, DmaTxBuffer, - InternalMemoryCachelineAligned, NoBuffer, ScopedDmaRxBuf, ScopedDmaTxBuf, TransferDirection, + aligned::{DmaAlignedMut, InternalMemory}, asynch::DmaRxFuture, prepare_for_rx, prepare_for_tx, @@ -116,7 +114,8 @@ impl<'d> Spi<'d, Blocking> { /// # {before_snippet} /// use esp_hal::{ /// dma::{DmaRxBuf, DmaTxBuf}, -/// dma_buffers, +/// dma_rx_buffer, +/// dma_tx_buffer, /// spi::{ /// Mode, /// master::{Config, Spi}, @@ -124,10 +123,8 @@ impl<'d> Spi<'d, Blocking> { /// }; /// /// // Optional: create and set up copy buffers. -/// let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(32000); -/// -/// let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer)?; -/// let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer)?; +/// let dma_rx_buf = dma_rx_buffer!(32000)?; +/// let dma_tx_buf = dma_tx_buffer!(32000)?; /// /// let mut spi = Spi::new( /// peripherals.SPI2, @@ -264,19 +261,30 @@ impl<'d> SpiDma<'d, Blocking> { state.tx_transfer_in_progress.set(false); state.rx_transfer_in_progress.set(false); - let descriptors = unsafe { (&mut *state.descriptors.get()).get_mut() }; - descriptors.fill(DmaDescriptor::EMPTY); - - let (tx_descriptors, rx_descriptors) = descriptors.split_at_mut(1); + // Safety: The descriptors occupy their own shared cache line and are updated in a + // synchronised fashion. + let (tx_descriptors, rx_descriptors) = unsafe { + let descriptors = (&mut *state.descriptors.get()).get_mut().into_inner(); + descriptors.fill(DmaDescriptor::EMPTY); + let (tx_descriptors, rx_descriptors) = descriptors.split_at_mut(1); + ( + DmaAlignedMut::new_unchecked(tx_descriptors), + DmaAlignedMut::new_unchecked(rx_descriptors), + ) + }; let tx_buffer = cfg_select! { all(spi_master_version = "1", spi_address_workaround) => unsafe { - (&mut *state.default_tx_buffer.get()).get_mut() + (&mut *state.default_tx_buffer.get()).get_mut().unsize() }, - _ => &mut [] + _ => unsafe { + DmaAlignedMut::new_unchecked(&mut [][..]) + } }; - let rx_buffer = unwrap!(DmaRxBuf::new(rx_descriptors, &mut [])); + let rx_buffer = unwrap!(DmaRxBuf::new(rx_descriptors, unsafe { + DmaAlignedMut::new_unchecked(&mut []) + })); let tx_buffer = unwrap!(DmaTxBuf::new(tx_descriptors, tx_buffer)); // The buffers must be set up when creating the driver. @@ -888,7 +896,7 @@ impl<'a> MaybeCopyRxBuf<'a> { #[cfg(dma_can_access_psram)] for buffer in align_buffer.iter_mut() { - if let Some(buffer) = buffer.as_ref() { + if let Some(buffer) = buffer.as_mut() { buffer.write_back(); } *buffer = None; @@ -1946,10 +1954,10 @@ struct DmaState { rx_buffer: UnsafeCell>>, tx_buffer: UnsafeCell>>, - descriptors: UnsafeCell>, + descriptors: UnsafeCell>, #[cfg(all(spi_master_version = "1", spi_address_workaround))] - default_tx_buffer: UnsafeCell>, + default_tx_buffer: UnsafeCell>, } impl DmaState { @@ -1999,9 +2007,9 @@ for_each_spi_master!( rx_buffer: UnsafeCell::new(MaybeUninit::uninit()), tx_buffer: UnsafeCell::new(MaybeUninit::uninit()), - descriptors: UnsafeCell::new(InternalMemoryCachelineAligned::new([DmaDescriptor::EMPTY; 2])), + descriptors: UnsafeCell::new(InternalMemory::new([DmaDescriptor::EMPTY; 2])), #[cfg(all(spi_master_version = "1", spi_address_workaround))] - default_tx_buffer: UnsafeCell::new(InternalMemoryBuffer::new()), + default_tx_buffer: UnsafeCell::new(InternalMemory::new([0; 4])), }; &DMA_STATE diff --git a/esp-hal/src/spi/slave.rs b/esp-hal/src/spi/slave.rs index 395953b83ea..81b1d3cd506 100644 --- a/esp-hal/src/spi/slave.rs +++ b/esp-hal/src/spi/slave.rs @@ -23,8 +23,8 @@ ```rust, no_run # {before_snippet} -# use esp_hal::dma_buffers; -# use esp_hal::dma::{DmaRxBuf, DmaTxBuf}; +# use esp_hal::dma_rx_buffer; +# use esp_hal::dma_tx_buffer; # use esp_hal::spi::Mode; # use esp_hal::spi::slave::Spi; let sclk = peripherals.GPIO0; @@ -32,9 +32,8 @@ let miso = peripherals.GPIO1; let mosi = peripherals.GPIO2; let cs = peripherals.GPIO3; -let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(32000); -let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); -let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); +let dma_rx_buf = dma_rx_buffer!(32000).unwrap(); +let dma_tx_buf = dma_tx_buffer!(32000).unwrap(); let mut spi = Spi::new(peripherals.SPI2, Mode::_0) .with_sck(sclk) .with_mosi(mosi) diff --git a/esp-hal/src/uart/uhci.rs b/esp-hal/src/uart/uhci.rs index 2fa24fa991c..8bb27b1e4c3 100644 --- a/esp-hal/src/uart/uhci.rs +++ b/esp-hal/src/uart/uhci.rs @@ -12,7 +12,8 @@ //! use esp_hal::{ //! clock::CpuClock, //! dma::{DmaRxBuf, DmaTxBuf}, -//! dma_buffers, +//! dma_rx_buffer, +//! dma_tx_buffer, //! main, //! rom::software_reset, //! uart, @@ -23,8 +24,6 @@ //! fn main() -> ! { //! let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); //! let peripherals = esp_hal::init(config); -//! let config = esp_hal::Config::default().with_cpu_clock(CpuClock::max()); -//! let peripherals = esp_hal::init(config); //! //! let config = uart::Config::default() //! .with_rx(RxConfig::default().with_fifo_full_threshold(64)) @@ -35,9 +34,8 @@ //! .with_tx(peripherals.GPIO2) //! .with_rx(peripherals.GPIO3); //! -//! let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4092); -//! let dma_rx = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); -//! let mut dma_tx = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); +//! let dma_rx = dma_rx_buffer!(4092).unwrap(); +//! let mut dma_tx = dma_tx_buffer!(4092).unwrap(); //! //! let mut uhci = Uhci::new(uart, peripherals.UHCI0, peripherals.DMA_CH0); //! uhci.apply_rx_config( diff --git a/esp-metadata/src/lib.rs b/esp-metadata/src/lib.rs index edcb73ad037..6e6988af985 100644 --- a/esp-metadata/src/lib.rs +++ b/esp-metadata/src/lib.rs @@ -903,7 +903,7 @@ fn generate_for_each_macro(name: &str, branches: &[Branch<'_>]) -> TokenStream { // } // } // ``` - #( #inner!( (#repeat_names #( (#repeat_branches) ),*) ); )* + #( #inner!( (#repeat_names #( (#repeat_branches) ),*) ); )* }; } } diff --git a/examples/async/embassy_spi/src/main.rs b/examples/async/embassy_spi/src/main.rs index cbf0bc7a575..796d68c247e 100644 --- a/examples/async/embassy_spi/src/main.rs +++ b/examples/async/embassy_spi/src/main.rs @@ -21,8 +21,8 @@ use embassy_executor::Spawner; use embassy_time::{Duration, Timer}; use esp_backtrace as _; use esp_hal::{ - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, interrupt::software::SoftwareInterruptControl, spi::{ Mode, @@ -56,9 +56,8 @@ async fn main(_spawner: Spawner) { _ => peripherals.DMA_CH0, }; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(32000); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(32000).unwrap(); + let dma_tx_buf = dma_tx_buffer!(32000).unwrap(); let mut spi = Spi::new( peripherals.SPI2, diff --git a/examples/peripheral/spi/loopback_dma_psram/src/main.rs b/examples/peripheral/spi/loopback_dma_psram/src/main.rs index 2a0df715549..b37bb9eb69e 100644 --- a/examples/peripheral/spi/loopback_dma_psram/src/main.rs +++ b/examples/peripheral/spi/loopback_dma_psram/src/main.rs @@ -22,7 +22,8 @@ extern crate alloc; use esp_backtrace as _; use esp_hal::{ delay::Delay, - dma::{DmaRxBuf, DmaTxBuf, ExternalBurstConfig}, + dma::ExternalBurstConfig, + dma_rx_buffer, main, spi::{ Mode, @@ -34,23 +35,30 @@ use log::*; esp_bootloader_esp_idf::esp_app_desc!(); -macro_rules! dma_alloc_buffer { +macro_rules! dma_alloc_tx_buffer { ($size:expr, $align:expr) => {{ - let layout = core::alloc::Layout::from_size_align($size, $align).unwrap(); - unsafe { + let layout = core::alloc::Layout::from_size_align($size, $align as usize).unwrap(); + let buffer = unsafe { let ptr = alloc::alloc::alloc(layout); if ptr.is_null() { error!("dma_alloc_buffer: alloc failed"); alloc::alloc::handle_alloc_error(layout); } core::slice::from_raw_parts_mut(ptr, $size) - } + }; + + const DMA_CHUNK_SIZE: usize = 4096 - $align as usize; + let descriptors = esp_hal::dma_descriptors_impl!($size, DMA_CHUNK_SIZE); + esp_hal::dma::DmaTxBuf::new_with_config( + descriptors, + unsafe { esp_hal::dma::aligned::DmaAlignedMut::new_unchecked(buffer) }, + $align, + ) }}; } const DMA_BUFFER_SIZE: usize = 8192; const DMA_ALIGNMENT: ExternalBurstConfig = ExternalBurstConfig::Size64; -const DMA_CHUNK_SIZE: usize = 4096 - DMA_ALIGNMENT as usize; #[main] fn main() -> ! { @@ -72,25 +80,8 @@ fn main() -> ! { _ => peripherals.DMA_CH0, }; - let (_, tx_descriptors) = - esp_hal::dma_descriptors_chunk_size!(0, DMA_BUFFER_SIZE, DMA_CHUNK_SIZE); - let tx_buffer = dma_alloc_buffer!(DMA_BUFFER_SIZE, DMA_ALIGNMENT as usize); - info!( - "TX: {:p} len {} ({} descripters)", - tx_buffer.as_ptr(), - tx_buffer.len(), - tx_descriptors.len() - ); - let mut dma_tx_buf = - DmaTxBuf::new_with_config(tx_descriptors, tx_buffer, DMA_ALIGNMENT).unwrap(); - let (rx_buffer, rx_descriptors, _, _) = esp_hal::dma_buffers!(DMA_BUFFER_SIZE, 0); - info!( - "RX: {:p} len {} ({} descripters)", - rx_buffer.as_ptr(), - rx_buffer.len(), - rx_descriptors.len() - ); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); + let mut dma_tx_buf = dma_alloc_tx_buffer!(DMA_BUFFER_SIZE, DMA_ALIGNMENT).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); // Need to set miso first so that mosi can overwrite the // output connection (because we are using the same pin to loop back) let mut spi = Spi::new( diff --git a/examples/peripheral/spi/slave_dma/src/main.rs b/examples/peripheral/spi/slave_dma/src/main.rs index 5141ce0c51d..b4175b1a0c2 100644 --- a/examples/peripheral/spi/slave_dma/src/main.rs +++ b/examples/peripheral/spi/slave_dma/src/main.rs @@ -32,8 +32,8 @@ use esp_backtrace as _; use esp_hal::{ delay::Delay, - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, gpio::{Input, InputConfig, Level, Output, OutputConfig, Pull}, main, spi::{Mode, slave::Spi}, @@ -65,9 +65,8 @@ fn main() -> ! { _ => peripherals.DMA_CH0, }; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(32000); - let mut slave_receive = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut slave_send = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut slave_receive = dma_rx_buffer!(32000).unwrap(); + let mut slave_send = dma_tx_buffer!(32000).unwrap(); let mut spi = Spi::new(peripherals.SPI2, Mode::_0) .with_sck(slave_sclk) diff --git a/hil-test/src/bin/crypto/aes.rs b/hil-test/src/bin/crypto/aes.rs index 781595a3b39..b3798b8696a 100644 --- a/hil-test/src/bin/crypto/aes.rs +++ b/hil-test/src/bin/crypto/aes.rs @@ -364,7 +364,8 @@ mod tests { use esp_hal::{ aes::dma::AesDma, dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, }; struct Context<'a> { @@ -421,9 +422,8 @@ mod tests { const DMA_BUFFER_SIZE: usize = 16; - let (output, rx_descriptors, input, tx_descriptors) = dma_buffers!(DMA_BUFFER_SIZE); - let output = DmaRxBuf::new(rx_descriptors, output).unwrap(); - let input = DmaTxBuf::new(tx_descriptors, input).unwrap(); + let input = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let output = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); let peripherals = esp_hal::init(Config::default().with_cpu_clock(CpuClock::max())); diff --git a/hil-test/src/bin/embassy_timers_executors.rs b/hil-test/src/bin/embassy_timers_executors.rs index 36216580e74..72eb5778dcc 100644 --- a/hil-test/src/bin/embassy_timers_executors.rs +++ b/hil-test/src/bin/embassy_timers_executors.rs @@ -428,14 +428,14 @@ mod interrupt_executor { } } -#[cfg(any(esp32, esp32c3, esp32c6, esp32h2, esp32s2, esp32s3))] +#[cfg(any(soc_has_spi3, soc_has_i2s0))] #[embedded_test::tests(default_timeout = 3, executor = hil_test::Executor::new())] mod interrupt_spi_dma { use embassy_time::{Duration, Instant, Timer}; use esp_hal::{ Blocking, - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, interrupt::{Priority, software::SoftwareInterruptControl}, spi::{ Mode, @@ -451,12 +451,11 @@ mod interrupt_spi_dma { static STOP_INTERRUPT_TASK: AtomicBool = AtomicBool::new(false); static INTERRUPT_TASK_WORKING: AtomicBool = AtomicBool::new(false); - #[cfg(any(esp32, esp32s2, esp32s3))] + #[cfg(soc_has_spi3)] #[embassy_executor::task] async fn interrupt_driven_task(spi: esp_hal::spi::master::SpiDma<'static, Blocking>) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(128); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(128).unwrap(); + let dma_tx_buf = dma_tx_buffer!(128).unwrap(); let mut spi = spi.with_buffers(dma_rx_buf, dma_tx_buf).into_async(); @@ -475,7 +474,7 @@ mod interrupt_spi_dma { } } - #[cfg(not(any(esp32, esp32s2, esp32s3)))] + #[cfg(not(soc_has_spi3))] #[embassy_executor::task] async fn interrupt_driven_task(i2s_tx: esp_hal::i2s::master::I2s<'static, Blocking>) { use esp_hal::dma_tx_stream_buffer; @@ -511,12 +510,14 @@ mod interrupt_spi_dma { any(feature = "esp32", feature = "esp32s2") => { (peripherals.DMA_SPI2, peripherals.DMA_SPI3) } + feature = "esp32p4" => { + (peripherals.DMA_AXI_CH0, peripherals.DMA_AXI_CH1) + } _ => (peripherals.DMA_CH0, peripherals.DMA_CH1), }; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(1024); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(1024).unwrap(); + let dma_tx_buf = dma_tx_buffer!(1024).unwrap(); let (_, mosi) = hil_test::common_test_pins!(peripherals); @@ -533,7 +534,7 @@ mod interrupt_spi_dma { .with_buffers(dma_rx_buf, dma_tx_buf) .into_async(); - #[cfg(any(esp32, esp32s2, esp32s3))] + #[cfg(soc_has_spi3)] let other_peripheral = Spi::new( peripherals.SPI3, Config::default() @@ -543,7 +544,7 @@ mod interrupt_spi_dma { .unwrap() .with_dma(dma_channel2); - #[cfg(not(any(esp32, esp32s2, esp32s3)))] + #[cfg(not(soc_has_spi3))] let other_peripheral = esp_hal::i2s::master::I2s::new( peripherals.I2S0, dma_channel2, @@ -620,9 +621,8 @@ mod interrupt_spi_dma { peripherals: SpiPeripherals, finished: &'static Signal, ) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(3200); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(3200).unwrap(); + let dma_tx_buf = dma_tx_buffer!(3200).unwrap(); let mut spi = Spi::new( peripherals.spi, @@ -652,15 +652,9 @@ mod interrupt_spi_dma { esp_rtos::start(timg0.timer0, sw_int.software_interrupt0); let dma_channel = cfg_select! { - spi_master_dma_engine = "SPI_DMA" => { - peripherals.DMA_SPI2 - }, - spi_master_dma_engine = "AHB_GDMA" => { - peripherals.DMA_CH0 - }, - spi_master_dma_engine = "AXI_GDMA" => { - peripherals.DMA_AXI_CH0 - }, + spi_master_dma_engine = "SPI_DMA" => peripherals.DMA_SPI2, + spi_master_dma_engine = "AHB_GDMA" => peripherals.DMA_CH0, + spi_master_dma_engine = "AXI_GDMA" => peripherals.DMA_AXI_CH0, }; let transfer_finished = &*mk_static!(Signal, Signal::new()); diff --git a/hil-test/src/bin/gpio.rs b/hil-test/src/bin/gpio.rs index f4e51910e7d..535d3ea4780 100644 --- a/hil-test/src/bin/gpio.rs +++ b/hil-test/src/bin/gpio.rs @@ -6,7 +6,7 @@ #![no_std] #![no_main] -#![cfg_attr(xtensa, feature(asm_experimental_arch))] +#![cfg_attr(esp32s2, feature(asm_experimental_arch))] use esp_hal::gpio::{AnyPin, Input, InputConfig, Level, Output, OutputConfig, Pin, Pull}; use hil_test as _; diff --git a/hil-test/src/bin/i2s.rs b/hil-test/src/bin/i2s.rs index b72bc312bc8..15fa33c9f8d 100644 --- a/hil-test/src/bin/i2s.rs +++ b/hil-test/src/bin/i2s.rs @@ -14,7 +14,7 @@ mod tests { use esp_hal::{ Async, delay::Delay, - dma::{DmaDescriptor, DmaRxBuf, DmaTxBuf, DmaTxStreamBuf}, + dma::{DmaDescriptor, DmaRxBuf, DmaTxBuf, DmaTxStreamBuf, aligned::InternalMemory}, dma_rx_stream_buffer, dma_tx_stream_buffer, gpio::{AnyPin, NoPin, Pin}, @@ -371,9 +371,14 @@ mod tests { #[test] fn test_i2s_read_one_shot_multiple(ctx: Context) { - let buffer = hil_test::mk_static!([u8; 8000], [0u8; 8000]); - let descr = hil_test::mk_static!([DmaDescriptor; 4], [DmaDescriptor::EMPTY; 4]); - let mut rx_buffer = DmaRxBuf::new(descr, buffer).unwrap(); + let buffer = + hil_test::mk_static!(InternalMemory<[u8; 8000]>, InternalMemory::new([0; 8000])); + let descr = hil_test::mk_static!( + InternalMemory<[DmaDescriptor; 4]>, + InternalMemory::new([DmaDescriptor::EMPTY; 4]) + ); + let mut rx_buffer = + DmaRxBuf::new(descr.get_mut().unsize(), buffer.get_mut().unsize()).unwrap(); let i2s = I2s::new( ctx.i2s, @@ -423,9 +428,14 @@ mod tests { #[test] async fn test_i2s_read_one_shot_multiple_async(ctx: Context) { - let buffer = hil_test::mk_static!([u8; 8000], [0u8; 8000]); - let descr = hil_test::mk_static!([DmaDescriptor; 4], [DmaDescriptor::EMPTY; 4]); - let mut rx_buffer = DmaRxBuf::new(descr, buffer).unwrap(); + let buffer = + hil_test::mk_static!(InternalMemory<[u8; 8000]>, InternalMemory::new([0; 8000])); + let descr = hil_test::mk_static!( + InternalMemory<[DmaDescriptor; 4]>, + InternalMemory::new([DmaDescriptor::EMPTY; 4]) + ); + let mut rx_buffer = + DmaRxBuf::new(descr.get_mut().unsize(), buffer.get_mut().unsize()).unwrap(); let i2s = I2s::new( ctx.i2s, @@ -480,9 +490,13 @@ mod tests { // write completes. #[test] fn test_i2s_write_one_shot(ctx: Context) { - let buffer = hil_test::mk_static!([u8; 8000], [1u8; 8000]); - let descr = hil_test::mk_static!([DmaDescriptor; 4], [DmaDescriptor::EMPTY; 4]); - let tx_buffer = DmaTxBuf::new(descr, buffer).unwrap(); + let buffer = + hil_test::mk_static!(InternalMemory<[u8; 8000]>, InternalMemory::new([1u8; 8000])); + let descr = hil_test::mk_static!( + InternalMemory<[DmaDescriptor; 4]>, + InternalMemory::new([DmaDescriptor::EMPTY; 4]) + ); + let tx_buffer = DmaTxBuf::new(descr.get_mut().unsize(), buffer.get_mut().unsize()).unwrap(); let i2s = I2s::new( ctx.i2s, @@ -518,9 +532,13 @@ mod tests { // write completes. #[test] async fn test_i2s_write_one_shot_async(ctx: Context) { - let buffer = hil_test::mk_static!([u8; 8000], [1u8; 8000]); - let descr = hil_test::mk_static!([DmaDescriptor; 4], [DmaDescriptor::EMPTY; 4]); - let tx_buffer = DmaTxBuf::new(descr, buffer).unwrap(); + let buffer = + hil_test::mk_static!(InternalMemory<[u8; 8000]>, InternalMemory::new([1u8; 8000])); + let descr = hil_test::mk_static!( + InternalMemory<[DmaDescriptor; 4]>, + InternalMemory::new([DmaDescriptor::EMPTY; 4]) + ); + let tx_buffer = DmaTxBuf::new(descr.get_mut().unsize(), buffer.get_mut().unsize()).unwrap(); let i2s = I2s::new( ctx.i2s, @@ -558,9 +576,13 @@ mod tests { // write completes. #[test] async fn test_i2s_write_one_shot_multiple(ctx: Context) { - let buffer = hil_test::mk_static!([u8; 4], [1u8; 4]); - let descr = hil_test::mk_static!([DmaDescriptor; 4], [DmaDescriptor::EMPTY; 4]); - let mut tx_buffer = DmaTxBuf::new(descr, buffer).unwrap(); + let buffer = hil_test::mk_static!(InternalMemory<[u8; 4]>, InternalMemory::new([1u8; 4])); + let descr = hil_test::mk_static!( + InternalMemory<[DmaDescriptor; 4]>, + InternalMemory::new([DmaDescriptor::EMPTY; 4]) + ); + let mut tx_buffer = + DmaTxBuf::new(descr.get_mut().unsize(), buffer.get_mut().unsize()).unwrap(); let i2s = I2s::new( ctx.i2s, @@ -602,9 +624,13 @@ mod tests { // write completes. #[test] async fn test_i2s_write_one_shot_multiple_async(ctx: Context) { - let buffer = hil_test::mk_static!([u8; 4], [1u8; 4]); - let descr = hil_test::mk_static!([DmaDescriptor; 4], [DmaDescriptor::EMPTY; 4]); - let mut tx_buffer = DmaTxBuf::new(descr, buffer).unwrap(); + let buffer = hil_test::mk_static!(InternalMemory<[u8; 4]>, InternalMemory::new([1u8; 4])); + let descr = hil_test::mk_static!( + InternalMemory<[DmaDescriptor; 4]>, + InternalMemory::new([DmaDescriptor::EMPTY; 4]) + ); + let mut tx_buffer = + DmaTxBuf::new(descr.get_mut().unsize(), buffer.get_mut().unsize()).unwrap(); let i2s = I2s::new( ctx.i2s, diff --git a/hil-test/src/bin/lcd_cam.rs b/hil-test/src/bin/lcd_cam.rs index f600b89b045..bd3ab8858f1 100644 --- a/hil-test/src/bin/lcd_cam.rs +++ b/hil-test/src/bin/lcd_cam.rs @@ -9,7 +9,8 @@ use esp_hal::{ Async, Blocking, dma::{DmaChannel, DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, gpio::Level, lcd_cam::{ BitOrder, @@ -75,8 +76,7 @@ mod async_tests { async fn init() -> AsyncContext<'static> { let peripherals = esp_hal::init(esp_hal::Config::default()); let lcd_cam = LcdCam::new(peripherals.LCD_CAM).into_async(); - let (_, _, tx_buffer, tx_desc) = dma_buffers!(0, DATA_SIZE); - let dma_buf = DmaTxBuf::new(tx_desc, tx_buffer).unwrap(); + let dma_buf = dma_tx_buffer!(DATA_SIZE).unwrap(); AsyncContext { lcd_cam, @@ -117,8 +117,7 @@ mod blocking_tests { let peripherals = esp_hal::init(esp_hal::Config::default()); let lcd_cam = LcdCam::new(peripherals.LCD_CAM); let pcnt = Pcnt::new(peripherals.PCNT); - let (_, _, tx_buffer, tx_desc) = dma_buffers!(0, DATA_SIZE); - let dma_buf = DmaTxBuf::new(tx_desc, tx_buffer).unwrap(); + let dma_buf = dma_tx_buffer!(DATA_SIZE).unwrap(); BlockingContext { lcd_cam, @@ -351,9 +350,8 @@ mod camera_tests { #[init] fn init() -> CameraContext { let peripherals = esp_hal::init(esp_hal::Config::default()); - let (rx_buf, rx_desc, tx_buf, tx_desc) = dma_buffers!(50 * 50); - let dma_rx_buf = DmaRxBuf::new(rx_desc, rx_buf).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_desc, tx_buf).unwrap(); + let dma_rx_buf = dma_rx_buffer!(2500).unwrap(); + let dma_tx_buf = dma_tx_buffer!(2500).unwrap(); CameraContext { peripherals, diff --git a/hil-test/src/bin/misc_non_drivers/dma_buffers.rs b/hil-test/src/bin/misc_non_drivers/dma_buffers.rs index 4031d7db72e..00f0f5ddb11 100644 --- a/hil-test/src/bin/misc_non_drivers/dma_buffers.rs +++ b/hil-test/src/bin/misc_non_drivers/dma_buffers.rs @@ -70,7 +70,7 @@ mod tests { fn with_tx_view(all_cpu_owned: bool, f: impl FnOnce(DmaTxStreamBufView)) { let mut buf = dma_tx_stream_buffer!(BUFFER_SIZE, CHUNK_SIZE); buf.prepare(); - let (descriptors, buffer) = buf.split(); + let (mut descriptors, buffer) = buf.split(); if all_cpu_owned { for desc in descriptors.iter_mut() { desc.set_owner(Owner::Cpu); @@ -107,7 +107,7 @@ mod tests { } #[test] - fn test_dma_rx_stream_buf_insufficient_descriptors() { + fn test_dma_rx_stream_buf_insufficient_descriptors_require_at_least_4() { let (buffer, descriptors) = esp_hal::dma_buffers_impl!(BUFFER_SIZE, CHUNK_SIZE * 2); match DmaRxStreamBuf::new(descriptors, buffer) { Err(DmaBufError::InsufficientDescriptors) => (), @@ -116,7 +116,7 @@ mod tests { } #[test] - fn test_dma_tx_stream_buf_insufficient_descriptors() { + fn test_dma_tx_stream_buf_insufficient_descriptors_require_at_least_4() { let (buffer, descriptors) = esp_hal::dma_buffers_impl!(BUFFER_SIZE, CHUNK_SIZE * 2); match DmaTxStreamBuf::new(descriptors, buffer) { Err(DmaBufError::InsufficientDescriptors) => (), @@ -339,7 +339,7 @@ mod tests { core::assert_eq!(view.available_bytes(), 0); let buf = ::from_view(view); - let (descriptors, buffer) = buf.split(); + let (mut descriptors, buffer) = buf.split(); // DMA processed first descriptor, now it should be owned by CPU descriptors[0].set_owner(Owner::Cpu); diff --git a/hil-test/src/bin/misc_non_drivers/dma_mem2mem.rs b/hil-test/src/bin/misc_non_drivers/dma_mem2mem.rs index 92773434b58..80639fb9124 100644 --- a/hil-test/src/bin/misc_non_drivers/dma_mem2mem.rs +++ b/hil-test/src/bin/misc_non_drivers/dma_mem2mem.rs @@ -35,12 +35,12 @@ mod tests { .with_descriptors(rx_descriptors, tx_descriptors, Default::default()) .unwrap(); - for i in 0..core::mem::size_of_val(tx_buffer) { + for i in 0..tx_buffer.len() { tx_buffer[i] = (i % 256) as u8; } let dma_wait = mem2mem.start_transfer(rx_buffer, tx_buffer).unwrap(); dma_wait.wait().unwrap(); - for i in 0..core::mem::size_of_val(tx_buffer) { + for i in 0..tx_buffer.len() { assert_eq!(rx_buffer[i], tx_buffer[i]); } } diff --git a/hil-test/src/bin/parl_io.rs b/hil-test/src/bin/parl_io.rs index 43a3d97a4bd..b57b97c36f2 100644 --- a/hil-test/src/bin/parl_io.rs +++ b/hil-test/src/bin/parl_io.rs @@ -13,8 +13,8 @@ mod tests { #[cfg(esp32c6)] use esp_hal::parl_io::{RxEightBits, TxEightBits}; use esp_hal::{ - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, gpio::{AnyPin, Pin}, parl_io::{ BitPackOrder, @@ -129,9 +129,8 @@ mod tests { fn test_parl_io_rx_can_read_tx_in_1_bit_mode(ctx: Context) { const BUFFER_SIZE: usize = 64; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); let (clock_rx, clock_tx) = unsafe { ctx.clock_pin.split() }; let (valid_rx, valid_tx) = unsafe { ctx.valid_pin.split() }; @@ -192,9 +191,8 @@ mod tests { fn test_parl_io_rx_can_read_tx_in_2_bit_mode(ctx: Context) { const BUFFER_SIZE: usize = 64; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); let (clock_rx, clock_tx) = unsafe { ctx.clock_pin.split() }; let (valid_rx, valid_tx) = unsafe { ctx.valid_pin.split() }; @@ -255,9 +253,8 @@ mod tests { fn test_parl_io_rx_can_read_tx_in_4_bit_mode(ctx: Context) { const BUFFER_SIZE: usize = 64; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); let (clock_rx, clock_tx) = unsafe { ctx.clock_pin.split() }; let (valid_rx, valid_tx) = unsafe { ctx.valid_pin.split() }; @@ -329,9 +326,8 @@ mod tests { fn test_parl_io_rx_can_read_tx_in_8_bit_mode(ctx: Context) { const BUFFER_SIZE: usize = 250; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); let (clock_rx, clock_tx) = unsafe { ctx.clock_pin.split() }; let (valid_rx, valid_tx) = unsafe { ctx.valid_pin.split() }; diff --git a/hil-test/src/bin/spi_full_duplex.rs b/hil-test/src/bin/spi_full_duplex.rs index e0c94e3d36f..a5b61a94f32 100644 --- a/hil-test/src/bin/spi_full_duplex.rs +++ b/hil-test/src/bin/spi_full_duplex.rs @@ -27,8 +27,10 @@ cfg_select! { #[cfg(spi_master_supports_dma)] use esp_hal::{ gpio::{Level, NoPin}, - dma::{DmaDescriptor, DmaRxBuf, DmaTxBuf}, + dma::{DmaDescriptor, DmaRxBuf, DmaTxBuf, aligned::DmaAlignedMut}, dma_buffers, + dma_rx_buffer, + dma_tx_buffer, spi::master::SpiDma, }; @@ -531,9 +533,8 @@ mod tests { fn test_dma_read_dma_write_pcnt(ctx: Context) { const DMA_BUFFER_SIZE: usize = 8; const TRANSFER_SIZE: usize = 5; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(DMA_BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); let mut spi = ctx.spi.with_dma(ctx.dma_channel); @@ -565,9 +566,8 @@ mod tests { fn test_dma_read_dma_transfer_pcnt(ctx: Context) { const DMA_BUFFER_SIZE: usize = 8; const TRANSFER_SIZE: usize = 5; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(DMA_BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); let mut spi = ctx.spi.with_dma(ctx.dma_channel); @@ -599,8 +599,16 @@ mod tests { fn test_symmetric_dma_transfer(ctx: Context) { // This test case sends a large amount of data, multiple times to verify that // https://github.com/esp-rs/esp-hal/issues/2151 is and remains fixed. - let mut dma_rx_buf = DmaRxBuf::new(ctx.rx_descriptors, ctx.rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(ctx.tx_descriptors, ctx.tx_buffer).unwrap(); + let mut dma_rx_buf = DmaRxBuf::new( + DmaAlignedMut::new(ctx.rx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.rx_buffer).unwrap(), + ) + .unwrap(); + let mut dma_tx_buf = DmaTxBuf::new( + DmaAlignedMut::new(ctx.tx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.tx_buffer).unwrap(), + ) + .unwrap(); for (i, v) in dma_tx_buf.as_mut_slice().iter_mut().enumerate() { *v = (i % 255) as u8; @@ -630,9 +638,8 @@ mod tests { fn test_asymmetric_dma_transfer(ctx: Context) { const WRITE_SIZE: usize = 4; const READ_SIZE: usize = 2; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4, 4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(4).unwrap(); dma_tx_buf.fill(&[0xde, 0xad, 0xbe, 0xef]); @@ -689,9 +696,8 @@ mod tests { } // Set up SPI - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let dma_tx_buf = dma_tx_buffer!(4).unwrap(); let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); @@ -712,9 +718,8 @@ mod tests { #[test] #[cfg(all(spi_master_supports_dma, feature = "unstable"))] fn test_dma_bus_symmetric_transfer(ctx: Context) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(128); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(128).unwrap(); + let dma_tx_buf = dma_tx_buffer!(128).unwrap(); let mut spi = ctx .spi @@ -775,9 +780,8 @@ mod tests { #[test] #[cfg(all(pcnt_driver_supported, spi_master_supports_dma, feature = "unstable"))] async fn test_async_dma_read_dma_write_pcnt(ctx: Context) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let dma_tx_buf = dma_tx_buffer!(4).unwrap(); let mut spi = ctx .spi .with_dma(ctx.dma_channel) @@ -819,9 +823,8 @@ mod tests { #[test] #[cfg(all(pcnt_driver_supported, spi_master_supports_dma, feature = "unstable"))] async fn test_async_dma_read_dma_transfer_pcnt(ctx: Context) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let dma_tx_buf = dma_tx_buffer!(4).unwrap(); let mut spi = ctx .spi .with_dma(ctx.dma_channel) @@ -870,9 +873,8 @@ mod tests { .with_miso(Level::High) .with_dma(ctx.dma_channel); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(4).unwrap(); dma_tx_buf.fill(&[0xde, 0xad, 0xbe, 0xef]); @@ -959,8 +961,16 @@ mod tests { .unwrap(); // Set up a large buffer that would trigger a timeout - let dma_rx_buf = DmaRxBuf::new(ctx.rx_descriptors, ctx.rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(ctx.tx_descriptors, ctx.tx_buffer).unwrap(); + let dma_rx_buf = DmaRxBuf::new( + DmaAlignedMut::new(ctx.rx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.rx_buffer).unwrap(), + ) + .unwrap(); + let dma_tx_buf = DmaTxBuf::new( + DmaAlignedMut::new(ctx.tx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.tx_buffer).unwrap(), + ) + .unwrap(); let spi = ctx.spi.with_dma(ctx.dma_channel); @@ -977,13 +987,22 @@ mod tests { #[cfg(all(spi_master_supports_dma, feature = "unstable"))] fn can_transmit_after_cancel(mut ctx: Context) { // Slow down. At 80kHz, the transfer is supposed to take a bit over 3 seconds. + ctx.spi .apply_config(&Config::default().with_frequency(Rate::from_khz(80))) .unwrap(); // Set up a large buffer that would trigger a timeout - let mut dma_rx_buf = DmaRxBuf::new(ctx.rx_descriptors, ctx.rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(ctx.tx_descriptors, ctx.tx_buffer).unwrap(); + let mut dma_rx_buf = DmaRxBuf::new( + DmaAlignedMut::new(ctx.rx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.rx_buffer).unwrap(), + ) + .unwrap(); + let mut dma_tx_buf = DmaTxBuf::new( + DmaAlignedMut::new(ctx.tx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.tx_buffer).unwrap(), + ) + .unwrap(); let mut spi = ctx.spi.with_dma(ctx.dma_channel); @@ -1015,8 +1034,16 @@ mod tests { #[cfg(all(spi_master_supports_dma, feature = "unstable"))] async fn cancelling_an_awaited_transfer_does_nothing(ctx: Context) { // Set up a large buffer that would trigger a timeout - let dma_rx_buf = DmaRxBuf::new(ctx.rx_descriptors, ctx.rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(ctx.tx_descriptors, ctx.tx_buffer).unwrap(); + let dma_rx_buf = DmaRxBuf::new( + DmaAlignedMut::new(ctx.rx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.rx_buffer).unwrap(), + ) + .unwrap(); + let dma_tx_buf = DmaTxBuf::new( + DmaAlignedMut::new(ctx.tx_descriptors).unwrap(), + DmaAlignedMut::new(ctx.tx_buffer).unwrap(), + ) + .unwrap(); let spi = ctx.spi.with_dma(ctx.dma_channel).into_async(); @@ -1103,9 +1130,8 @@ mod tests { .channel0 .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let dma_tx_buf = dma_tx_buffer!(4).unwrap(); let mut spi = ctx .spi @@ -1128,9 +1154,8 @@ mod tests { .channel0 .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let dma_tx_buf = dma_tx_buffer!(4).unwrap(); let mut spi = ctx .spi @@ -1190,9 +1215,8 @@ mod tests { ) .unwrap(); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(4).unwrap(); + let dma_tx_buf = dma_tx_buffer!(4).unwrap(); let mut spi = spi .with_dma(ctx.dma_channel) diff --git a/hil-test/src/bin/spi_half_duplex_slave_qspi.rs b/hil-test/src/bin/spi_half_duplex_slave_qspi.rs index 22bab6cd0bd..e7dd9bf4385 100644 --- a/hil-test/src/bin/spi_half_duplex_slave_qspi.rs +++ b/hil-test/src/bin/spi_half_duplex_slave_qspi.rs @@ -20,10 +20,7 @@ mod read { time::Rate, }; #[cfg(spi_master_supports_dma)] - use esp_hal::{ - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, - }; + use esp_hal::{dma_rx_buffer, dma_tx_buffer}; #[cfg(spi_master_supports_dma)] type DmaChannel<'a> = cfg_select! { @@ -127,8 +124,7 @@ mod read { fn test_spidma_reads_correctly_from_gpio_pin(mut ctx: Context) { const DMA_BUFFER_SIZE: usize = 4; - let (buffer, descriptors, _, _) = dma_buffers!(DMA_BUFFER_SIZE, 0); - let mut dma_rx_buf = DmaRxBuf::new(descriptors, buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); // SPI should read '0's from the MISO pin ctx.miso_mirror.set_low(); @@ -175,10 +171,8 @@ mod read { fn test_spidmabus_reads_correctly_from_gpio_pin(mut ctx: Context) { const DMA_BUFFER_SIZE: usize = 4; - // WAS THIS AN ERROR? - let (buffer, descriptors, tx, txd) = dma_buffers!(DMA_BUFFER_SIZE, 1); - let dma_rx_buf = DmaRxBuf::new(descriptors, buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(txd, tx).unwrap(); + let dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let dma_tx_buf = dma_tx_buffer!(1).unwrap(); let mut spi = ctx .spi @@ -220,9 +214,8 @@ mod read { fn data_mode_combinations_are_not_rejected(ctx: Context) { const DMA_BUFFER_SIZE: usize = 4; - let (buffer, descriptors, tx, txd) = dma_buffers!(DMA_BUFFER_SIZE, DMA_BUFFER_SIZE); - let dma_rx_buf = DmaRxBuf::new(descriptors, buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(txd, tx).unwrap(); + let dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); let mut buffer = [0xAA; DMA_BUFFER_SIZE]; let mut spi = ctx @@ -288,10 +281,7 @@ mod write { time::Rate, }; #[cfg(spi_master_supports_dma)] - use esp_hal::{ - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, - }; + use esp_hal::{dma_rx_buffer, dma_tx_buffer}; #[cfg(spi_master_supports_dma)] type DmaChannel<'a> = cfg_select! { @@ -393,8 +383,7 @@ mod write { fn perform_spidma_writes_are_correctly_by_pcnt(ctx: Context, mode: DataMode) { const DMA_BUFFER_SIZE: usize = 4; - let (_, _, buffer, descriptors) = dma_buffers!(0, DMA_BUFFER_SIZE); - let mut dma_tx_buf = DmaTxBuf::new(descriptors, buffer).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); let unit = ctx.pcnt_unit; let mut spi = ctx.spi.with_dma(ctx.dma_channel); @@ -442,9 +431,8 @@ mod write { fn perform_spidmabus_writes_are_correctly_by_pcnt(ctx: Context, mode: DataMode) { const DMA_BUFFER_SIZE: usize = 4; - let (rx, rxd, buffer, descriptors) = dma_buffers!(4, DMA_BUFFER_SIZE); - let dma_rx_buf = DmaRxBuf::new(rxd, rx).unwrap(); - let dma_tx_buf = DmaTxBuf::new(descriptors, buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); let unit = ctx.pcnt_unit; let mut spi = ctx @@ -531,10 +519,7 @@ mod spi_slave { spi::{Mode, slave::Spi}, }; #[cfg(spi_slave_supports_dma)] - use esp_hal::{ - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, - }; + use esp_hal::{dma_rx_buffer, dma_tx_buffer}; #[cfg(spi_slave_supports_dma)] type DmaChannel<'a> = cfg_select! { @@ -679,9 +664,9 @@ mod spi_slave { #[test] fn test_basic_dma(mut ctx: Context) { const DMA_SIZE: usize = 32; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(DMA_SIZE); - let mut slave_receive = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut slave_send = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + + let mut slave_receive = dma_rx_buffer!(DMA_SIZE).unwrap(); + let mut slave_send = dma_tx_buffer!(DMA_SIZE).unwrap(); let spi = ctx.spi.with_dma(ctx.dma_channel); @@ -719,7 +704,8 @@ mod qspi_dma { use esp_hal::{ Blocking, dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, gpio::{AnyPin, Input, InputConfig, Level, Output, OutputConfig, Pull}, spi::{ Mode, @@ -830,8 +816,7 @@ mod qspi_dma { fn execute_read(mut spi: SpiUnderTest, mut miso_mirror: Output<'static>, expected: u8) { const DMA_BUFFER_SIZE: usize = 4; - let (_, _, buffer, descriptors) = dma_buffers!(0, DMA_BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(descriptors, buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); miso_mirror.set_low(); (spi, dma_rx_buf) = transfer_read(spi, dma_rx_buf, Command::None); @@ -847,10 +832,8 @@ mod qspi_dma { fn execute_write_read(mut spi: SpiUnderTest, mut mosi_mirror: Output<'static>, expected: u8) { const DMA_BUFFER_SIZE: usize = 4; - let (rx_buffer, rx_descriptors, buffer, descriptors) = - dma_buffers!(DMA_BUFFER_SIZE, DMA_BUFFER_SIZE); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(descriptors, buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(DMA_BUFFER_SIZE).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); dma_tx_buf.fill(&[0x00; DMA_BUFFER_SIZE]); @@ -878,8 +861,7 @@ mod qspi_dma { ) { const DMA_BUFFER_SIZE: usize = 4; - let (_, _, buffer, descriptors) = dma_buffers!(0, DMA_BUFFER_SIZE); - let mut dma_tx_buf = DmaTxBuf::new(descriptors, buffer).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(DMA_BUFFER_SIZE).unwrap(); for command_data_mode in COMMAND_DATA_MODES { dma_tx_buf.fill(&[write; DMA_BUFFER_SIZE]); @@ -1100,9 +1082,8 @@ mod qspi_dma { let [pin, pin_mirror, _] = ctx.gpios; let mut pin_mirror = Output::new(pin_mirror, Level::Low, OutputConfig::default()); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(BUFFER_SIZE); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(BUFFER_SIZE).unwrap(); + let dma_tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); let mut spi = ctx .spi .with_sio0(pin) @@ -1242,9 +1223,8 @@ mod qspi_dma { .channel0 .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(BUFFER_SIZE); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx_buf = dma_rx_buffer!(BUFFER_SIZE).unwrap(); + let dma_tx_buf = dma_tx_buffer!(BUFFER_SIZE).unwrap(); let spi = ctx .spi .with_sio0(mosi) diff --git a/hil-test/src/bin/spi_half_duplex_write_psram.rs b/hil-test/src/bin/spi_half_duplex_write_psram.rs index 5840f16e8fd..2c85a3b121b 100644 --- a/hil-test/src/bin/spi_half_duplex_write_psram.rs +++ b/hil-test/src/bin/spi_half_duplex_write_psram.rs @@ -11,9 +11,8 @@ use defmt::error; use esp_alloc as _; use esp_hal::{ Blocking, - dma::{DmaRxBuf, DmaTxBuf, ExternalBurstConfig}, - dma_buffers, - dma_descriptors_chunk_size, + dma::ExternalBurstConfig, + dma_rx_buffer, gpio::{Flex, interconnect::InputSignal}, pcnt::{Pcnt, channel::EdgeMode, unit::Unit}, spi::{ @@ -25,17 +24,25 @@ use esp_hal::{ use hil_test as _; extern crate alloc; -macro_rules! dma_alloc_buffer { +macro_rules! dma_alloc_tx_buffer { ($size:expr, $align:expr) => {{ - let layout = core::alloc::Layout::from_size_align($size, $align).unwrap(); - unsafe { + let layout = core::alloc::Layout::from_size_align($size, $align as usize).unwrap(); + let buffer = unsafe { let ptr = alloc::alloc::alloc(layout); if ptr.is_null() { error!("dma_alloc_buffer: alloc failed"); alloc::alloc::handle_alloc_error(layout); } core::slice::from_raw_parts_mut(ptr, $size) - } + }; + + const DMA_CHUNK_SIZE: usize = 4096 - $align as usize; + let descriptors = esp_hal::dma_descriptors_impl!($size, DMA_CHUNK_SIZE); + esp_hal::dma::DmaTxBuf::new_with_config( + descriptors, + unsafe { esp_hal::dma::aligned::DmaAlignedMut::new_unchecked(buffer) }, + $align, + ) }}; } @@ -65,6 +72,7 @@ mod tests { let dma_channel = cfg_select! { spi_master_dma_engine = "SPI_DMA" => peripherals.DMA_SPI2, spi_master_dma_engine = "AHB_GDMA" => peripherals.DMA_CH0, + spi_master_dma_engine = "AXI_GDMA" => peripherals.DMA_AXI_CH0, }; let mut mosi = Flex::new(mosi); @@ -94,11 +102,7 @@ mod tests { fn test_spi_writes_are_correctly_by_pcnt(ctx: Context) { const DMA_BUFFER_SIZE: usize = 4; const DMA_ALIGNMENT: ExternalBurstConfig = ExternalBurstConfig::Size32; - const DMA_CHUNK_SIZE: usize = 4096 - DMA_ALIGNMENT as usize; - - let (_, descriptors) = dma_descriptors_chunk_size!(0, DMA_BUFFER_SIZE, DMA_CHUNK_SIZE); - let buffer = dma_alloc_buffer!(DMA_BUFFER_SIZE, DMA_ALIGNMENT as usize); - let mut dma_tx_buf = DmaTxBuf::new_with_config(descriptors, buffer, DMA_ALIGNMENT).unwrap(); + let mut dma_tx_buf = dma_alloc_tx_buffer!(DMA_BUFFER_SIZE, DMA_ALIGNMENT).unwrap(); let unit = ctx.pcnt_unit; let mut spi = ctx.spi; @@ -144,14 +148,9 @@ mod tests { fn test_spidmabus_writes_are_correctly_by_pcnt(ctx: Context) { const DMA_BUFFER_SIZE: usize = 4; const DMA_ALIGNMENT: ExternalBurstConfig = ExternalBurstConfig::Size32; // matches dcache line size - const DMA_CHUNK_SIZE: usize = 4096 - DMA_ALIGNMENT as usize; // 64 byte aligned - - let (_, descriptors) = dma_descriptors_chunk_size!(0, DMA_BUFFER_SIZE, DMA_CHUNK_SIZE); - let buffer = dma_alloc_buffer!(DMA_BUFFER_SIZE, DMA_ALIGNMENT as usize); - let dma_tx_buf = DmaTxBuf::new_with_config(descriptors, buffer, DMA_ALIGNMENT).unwrap(); - let (rx, rxd, _, _) = dma_buffers!(1, 0); - let dma_rx_buf = DmaRxBuf::new(rxd, rx).unwrap(); + let dma_tx_buf = dma_alloc_tx_buffer!(DMA_BUFFER_SIZE, DMA_ALIGNMENT).unwrap(); + let dma_rx_buf = dma_rx_buffer!(1).unwrap(); let unit = ctx.pcnt_unit; let mut spi = ctx.spi.with_buffers(dma_rx_buf, dma_tx_buf); diff --git a/hil-test/src/bin/uart.rs b/hil-test/src/bin/uart.rs index 81bd6dd37a8..b81de611d33 100644 --- a/hil-test/src/bin/uart.rs +++ b/hil-test/src/bin/uart.rs @@ -914,7 +914,8 @@ mod async_tx_rx_split { mod uhci { use esp_hal::{ dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, interrupt::software::SoftwareInterruptControl, peripherals::Peripherals, timer::timg::TimerGroup, @@ -942,10 +943,8 @@ mod uhci { async fn init() -> Context { let peripherals = esp_hal::init(esp_hal::Config::default()); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = - dma_buffers!(DMA_BUFFER_SIZE as usize); - let dma_rx = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let dma_rx = dma_rx_buffer!(DMA_BUFFER_SIZE as usize).unwrap(); + let dma_tx = dma_tx_buffer!(DMA_BUFFER_SIZE as usize).unwrap(); Context { dma_rx, diff --git a/qa-test/src/bin/psram.rs b/qa-test/src/bin/psram.rs index dfd2b37b551..57f6e13aa92 100644 --- a/qa-test/src/bin/psram.rs +++ b/qa-test/src/bin/psram.rs @@ -14,7 +14,7 @@ use alloc::{string::String, vec::Vec}; use esp_alloc as _; use esp_backtrace as _; -use esp_hal::{main, psram}; +use esp_hal::main; use esp_println::println; esp_bootloader_esp_idf::esp_app_desc!(); diff --git a/qa-test/src/bin/qspi_flash.rs b/qa-test/src/bin/qspi_flash.rs index 88b5fa72a8a..9394043d974 100644 --- a/qa-test/src/bin/qspi_flash.rs +++ b/qa-test/src/bin/qspi_flash.rs @@ -31,8 +31,8 @@ use esp_backtrace as _; use esp_hal::{ delay::Delay, - dma::{DmaRxBuf, DmaTxBuf}, - dma_buffers, + dma_rx_buffer, + dma_tx_buffer, main, spi::{ Mode, @@ -72,9 +72,8 @@ fn main() -> ! { _ => peripherals.DMA_CH0, }; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(320, 256); - let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let mut dma_rx_buf = dma_rx_buffer!(320).unwrap(); + let mut dma_tx_buf = dma_tx_buffer!(256).unwrap(); let mut spi = Spi::new( peripherals.SPI2,