Create and use a new cacheline-aligned reference type

esp-hal/src/aes/mod.rs

@@ -382,8 +382,8 @@ pub mod dma {
             DmaError,
             DmaRxBuffer,
             DmaTxBuffer,
-            InternalMemoryCachelineAligned,
             NoBuffer,
+            aligned::InternalMemory,
             prepare_for_rx,
             prepare_for_tx,
         },
@@ -761,7 +761,7 @@ pub mod dma {
 
         #[cfg(dma_can_access_psram)]
         unaligned_data_buffers: [Option<ManualWritebackBuffer>; 2],
-        descriptors: InternalMemoryCachelineAligned<[DmaDescriptor; OUT_DESCR_COUNT + 1]>,
+        descriptors: InternalMemory<[DmaDescriptor; OUT_DESCR_COUNT + 1]>,
     }
 
     // The DMA descriptors prevent auto-implementing Sync and Send, but they can be treated as Send
@@ -801,9 +801,7 @@ pub mod dma {
 
                 #[cfg(dma_can_access_psram)]
                 unaligned_data_buffers: [const { None }; 2],
-                descriptors: InternalMemoryCachelineAligned::new(
-                    [DmaDescriptor::EMPTY; OUT_DESCR_COUNT + 1],
-                ),
+                descriptors: InternalMemory::new([DmaDescriptor::EMPTY; OUT_DESCR_COUNT + 1]),
             }
         }
 
@@ -944,7 +942,7 @@ pub mod dma {
                     #[cfg(dma_can_access_psram)]
                     for buffer in self.unaligned_data_buffers.iter_mut() {
                         // Avoid copying the write_back buffer
-                        if let Some(buffer) = buffer.as_ref() {
+                        if let Some(buffer) = buffer.as_mut() {
                             buffer.write_back();
                         }
                         *buffer = None;
@@ -988,7 +986,7 @@ pub mod dma {
             work_item: &mut AesOperation,
         ) -> Result<(), AesDma<'d>> {
             let input_len = work_item.buffers.input.len();
-            let (input_dscr, output_dscr) = self.descriptors.get_mut().split_at_mut(1);
+            let (input_dscr, output_dscr) = self.descriptors.get_mut().into_inner().split_at_mut(1);
             let (input_buffer, data_len) = unsafe {
                 // This unwrap is infallible as AES-DMA devices don't have TX DMA alignment
                 // requirements.

esp-hal/src/dma/aligned.rs

@@ -0,0 +1,205 @@
+//! Helper types for DMA buffers.
+
+use core::ops::{Deref, DerefMut};
+
+use procmacros::doc_replace;
+
+#[cfg(dma_can_access_psram)]
+use crate::soc::is_valid_psram_address;
+use crate::{
+    dma::{DmaAlignmentError, DmaBufError},
+    soc::is_valid_ram_address,
+};
+
+/// DMA appropriate wrapper type for internal memory values.
+///
+/// The value wrapped in this type is guaranteed to be safely useable
+/// as a DMA buffer or descriptor array, meaning DMA or cache management
+/// operations will not corrupt surrounding data.
+///
+/// [`DmaAlignedMut`] is a reference type that carries this guarantee.
+// ESP32-P4 internal memory is cached, enforce alignment to avoid memory
+// corruption. Technically only needed for IN buffers and descriptor lists.
+#[derive(Debug)]
+#[cfg_attr(feature = "defmt", derive(defmt::Format))]
+#[cfg_attr(soc_internal_memory_cached, repr(C, align(64)))] // dcache cache line
+#[cfg_attr(not(soc_internal_memory_cached), repr(C, align(4)))] // Worst-case word alignment
+#[instability::unstable]
+pub struct InternalMemory<T>(T);
+
+impl<T> InternalMemory<T> {
+    /// Creates a new value aligned for DMA operations in internal memory.
+    #[instability::unstable]
+    pub const fn new(init: T) -> Self {
+        Self(init)
+    }
+
+    /// Returns a [`DmaAlignedMut`] to the underlying value.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the value is not located at a valid internal RAM address.
+    #[instability::unstable]
+    pub fn get_mut(&mut self) -> DmaAlignedMut<'_, T> {
+        assert!(is_valid_ram_address(&raw const self.0 as usize));
+        DmaAlignedMut(&mut self.0)
+    }
+}
+
+/// A mutable reference to an [`InternalMemory`] object.
+#[derive(Debug)]
+#[cfg_attr(feature = "defmt", derive(defmt::Format))]
+#[instability::unstable]
+pub struct DmaAlignedMut<'a, T>(&'a mut T)
+where
+    T: ?Sized;
+
+impl<'a, T: ?Sized> DmaAlignedMut<'a, T> {
+    #[doc_replace("align_req" => {
+        cfg(soc_internal_memory_cached) => "64",
+        _ => "4"
+    })]
+    /// Creates a new [`DmaAlignedMut`] from a mutable variable, if it's
+    /// provably compatible.
+    ///
+    /// In internal memory, the address and size of the variable
+    /// must be at least __align_req__ byte aligned.
+    #[instability::unstable]
+    pub fn new(ptr: &'a mut T) -> Result<Self, DmaBufError> {
+        if core::mem::size_of_val(ptr) == 0 {
+            return Ok(Self(ptr));
+        }
+
+        let addr = ptr as *mut T as *mut () as usize;
+
+        if is_valid_ram_address(addr) {
+            let alignment = core::mem::align_of::<InternalMemory<()>>();
+
+            if !core::mem::size_of_val(ptr).is_multiple_of(alignment) {
+                return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Size));
+            }
+            if !addr.is_multiple_of(alignment) {
+                return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Address));
+            }
+
+            return Ok(Self(ptr));
+        }
+
+        #[cfg(dma_can_access_psram)]
+        if is_valid_psram_address(addr) {
+            let alignment = cfg_select! {
+                // TODO(esp32p4): PSRAM is cached through the L2 cache, whose line size is
+                // configurable (64 or 128 bytes) and is not encoded anywhere yet. Assume the
+                // larger, always-safe value until the L2 line size is available.
+                soc_internal_memory_cached => 128,
+                any(esp32, esp32c5, esp32c61) => 32, // TODO: fixed 32-bytes, metadata-ify
+                _ => crate::soc::CONFIG_DATA_CACHE_LINE_SIZE,
+            };
+
+            if !core::mem::size_of_val(ptr).is_multiple_of(alignment) {
+                return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Size));
+            }
+            if !addr.is_multiple_of(alignment) {
+                return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Address));
+            }
+
+            return Ok(Self(ptr));
+        }
+
+        Err(DmaBufError::UnsupportedMemoryRegion)
+    }
+
+    /// Creates a new [`DmaAlignedMut`] from *any* mutable slice.
+    ///
+    /// # Safety
+    ///
+    /// The caller must ensure that the reference is properly aligned for the memory region it
+    /// occupies and the cachelines occupied by the reference do not overlap with any other data
+    /// that can be corrupted by a cacheline invalidation operation.
+    #[instability::unstable]
+    pub const unsafe fn new_unchecked(ptr: &'a mut T) -> Self {
+        Self(ptr)
+    }
+
+    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
+    fn do_cache_op(&self) -> bool {
+        // Do not do cache operations on zero-sized values.
+        if core::mem::size_of_val(self.0) == 0 {
+            return false;
+        }
+
+        let mut in_cached_region = false;
+        let address = self.0 as *const T as *const () as usize;
+
+        #[cfg(soc_internal_memory_cached)]
+        {
+            in_cached_region |= is_valid_ram_address(address);
+        }
+        #[cfg(dma_can_access_psram)]
+        {
+            in_cached_region |= is_valid_psram_address(address);
+        }
+
+        in_cached_region
+    }
+
+    /// Writes back the data from the cache to memory.
+    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
+    #[instability::unstable]
+    pub fn writeback(&mut self) {
+        if !self.do_cache_op() {
+            return;
+        }
+
+        // SAFETY: we own the cachelines and can't trash anything else
+        unsafe {
+            crate::soc::cache_writeback_addr(
+                self.0 as *const T as *const () as u32,
+                core::mem::size_of_val(self.0) as u32,
+            );
+        }
+    }
+
+    /// Invalidates the cache lines for this data.
+    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
+    #[instability::unstable]
+    pub fn invalidate(&mut self) {
+        if !self.do_cache_op() {
+            return;
+        }
+
+        // SAFETY: we own the cachelines and can't trash anything else
+        unsafe {
+            crate::soc::cache_invalidate_addr(
+                self.0 as *const T as *const () as u32,
+                core::mem::size_of_val(self.0) as u32,
+            );
+        }
+    }
+
+    /// Converts this object into a mutable reference.
+    pub fn into_inner(self) -> &'a mut T {
+        self.0
+    }
+}
+
+impl<'a, T, const N: usize> DmaAlignedMut<'a, [T; N]> {
+    /// Converts this object into a slice reference.
+    pub fn unsize(self) -> DmaAlignedMut<'a, [T]> {
+        DmaAlignedMut(self.0)
+    }
+}
+
+impl<'a, T: ?Sized> Deref for DmaAlignedMut<'a, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.0
+    }
+}
+
+impl<'a, T: ?Sized> DerefMut for DmaAlignedMut<'a, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.0
+    }
+}