diff --git a/src/lib.rs b/src/lib.rs index 98b6f2f2..304af45e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -199,15 +199,17 @@ impl RawSmallVec { #[inline] const fn as_ptr_inline(&self) -> *const T { // SAFETY: it is safe because we aren't reading the value, just getting a - // reference to it. reading it would be UB potentially, but for that downstream - // unsafe is required - (unsafe {&raw const self.inline}) as *mut T + // pointer to it. reading it would be UB potentially, but for that downstream + // unsafe is required. This is no longer unsafe as of Rust 1.92.0. + #[allow(unused_unsafe, reason = "requires unsafe in MSRV 1.83.0")] + (unsafe { &raw const self.inline }).cast::() } #[inline] const fn as_mut_ptr_inline(&mut self) -> *mut T { // SAFETY: same as above - (unsafe {&raw mut self.inline}) as *mut T + #[allow(unused_unsafe, reason = "requires unsafe in MSRV 1.83.0")] + (unsafe { &raw mut self.inline }).cast::() } /// # Safety @@ -2036,314 +2038,7 @@ pub fn from_elem(elem: T, n: usize) -> SmallVec } #[cfg(feature = "specialization")] -mod spec_traits { - use super::*; - - /// A trait for specializing the implementation of [`from_elem`]. - /// - /// [`from_elem`]: crate::from_elem - pub(crate) trait SpecFromElem { - /// Creates a `Smallvec` value where `elem` is repeated `n` times. - /// This will use the inline storage, not the heap. - /// - /// # Safety - /// - /// The caller must ensure that `n <= Self::inline_size()`. - unsafe fn spec_from_elem(elem: T, n: usize) -> Self; - } - - impl SpecFromElem for SmallVec { - #[inline] - default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { - // SAFETY: Safety conditions are identical. - unsafe { SmallVec::from_elem_fallback(elem, n) } - } - } - - impl SpecFromElem for SmallVec { - unsafe fn spec_from_elem(elem: T, n: usize) -> Self { - let mut result = Self::new(); - - if n > 0 { - let ptr = result.raw.as_mut_ptr_inline(); - - // SAFETY: The caller ensures that the first `n` - // is smaller than the inline size. - unsafe { - for i in 0..n { - ptr.add(i).write(elem); - } - } - } - - // SAFETY: The first `n` elements of the vector - // have been initialized in the loop above. - unsafe { - result.set_len(n); - } - - result - } - } - - /// A trait for specializing the implementations of [`Extend`] and [`extend_from_slice`]. - /// - /// [`extend_from_slice`]: crate::SmallVec::extend_from_slice - pub(crate) trait SpecExtend { - fn spec_extend(&mut self, iter: I); - } - - impl SpecExtend for SmallVec - where - I: Iterator, - { - #[inline] - default fn spec_extend(&mut self, iter: I) { - self.extend_fallback(iter); - } - } - - impl SpecExtend for SmallVec - where - I: core::iter::TrustedLen, - { - fn spec_extend(&mut self, iter: I) { - let (_, Some(additional)) = iter.size_hint() else { - panic!("capacity overflow") - }; - self.reserve(additional); - - // SAFETY: A `TrustedLen` iterator provides accurate information - // about its size, which was used to reserve additional memory. - // This ensures that the access operations inside the loop always - // operate on valid memory. - unsafe { - let len = self.len(); - let ptr = self.as_mut_ptr().add(len); - let mut guard = DropGuard { ptr, len: 0 }; - - for x in iter { - ptr.add(guard.len).write(x); - guard.len += 1; - } - - // The elements have been initialized in the loop above. - self.set_len(len + guard.len); - core::mem::forget(guard); - } - } - } - - impl SpecExtend> for SmallVec { - fn spec_extend(&mut self, mut iter: IntoIter) { - let slice = iter.as_slice(); - let len = slice.len(); - let old_len = self.len(); - - self.reserve(len); - - // SAFETY: Additional memory has been reserved above. - // Therefore, the copy operates on valid memory. - unsafe { - let dst = self.as_mut_ptr().add(old_len); - let src = slice.as_ptr(); - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - self.set_len(old_len + len); - } - - // Mark the iterator as fully consumed. - iter.begin = iter.end.value(); - } - } - - impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec - where - I: Iterator, - T: Clone, - { - #[inline] - default fn spec_extend(&mut self, iterator: I) { - self.spec_extend(iterator.cloned()) - } - } - - impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec - where - T: Copy, - { - fn spec_extend(&mut self, iter: core::slice::Iter<'a, T>) { - let slice = iter.as_slice(); - let len = slice.len(); - let old_len = self.len(); - - self.reserve(len); - - // SAFETY: Additional memory has been reserved above. - // Therefore, the copy operates on valid memory. - unsafe { - let dst = self.as_mut_ptr().add(old_len); - let src = slice.as_ptr(); - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - self.set_len(old_len + len); - } - } - } - - /// A trait for specializing the implementation of [`extend_from_within`]. - /// - /// [`extend_from_within`]: crate::SmallVec::extend_from_within - pub(crate) trait SpecExtendFromWithin { - /// Main worker for [`extend_from_within`]. - /// - /// # Safety - /// - /// * The length of the vector is larger than or equal to `src.len()`. - /// * The spare capacity of the vector is larger than or equal to `src.len()`. - /// - /// [`extend_from_within`]: SmallVec::extend_from_within - unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); - } - - impl SpecExtendFromWithin for SmallVec { - default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { - // SAFETY: Safety conditions are identical. - unsafe { - self.extend_from_within_fallback(src); - } - } - } - - impl SpecExtendFromWithin for SmallVec { - unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { - let old_len = self.len(); - - let start = src.start; - let len = src.len(); - - // SAFETY: The caller ensures that the vector has spare capacity - // for at least `src.len()` elements. This is also the amount of memory - // accessed when the data is copied. - unsafe { - let ptr = self.as_mut_ptr(); - let dst = ptr.add(old_len); - let src = ptr.add(start); - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - self.set_len(old_len + len); - } - } - } - - /// A trait for specializing the implementation of [`FromIterator`]. - /// - /// [`clone_from`]: Clone::clone_from - pub(crate) trait SpecFromIterator { - fn spec_from_iter(iter: I) -> Self; - } - - impl SpecFromIterator for SmallVec - where - I: Iterator, - { - #[inline] - default fn spec_from_iter(iter: I) -> Self { - Self::from_iter_fallback(iter) - } - } - - impl SpecFromIterator for SmallVec - where - I: core::iter::TrustedLen, - { - fn spec_from_iter(iter: I) -> Self { - let mut v = match iter.size_hint() { - (_, Some(upper)) => SmallVec::with_capacity(upper), - // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there - // are more than `usize::MAX` elements. - // Since the previous branch would eagerly panic if the capacity is too large - // (via `with_capacity`) we do the same here. - _ => panic!("capacity overflow"), - }; - // Reuse the extend specialization for TrustedLen. - v.spec_extend(iter); - v - } - } - - /// A trait for specializing the implementation of [`clone_from`]. - /// - /// [`clone_from`]: Clone::clone_from - pub(crate) trait SpecCloneFrom { - fn spec_clone_from(&mut self, source: &[T]); - } - - impl SpecCloneFrom for SmallVec { - #[inline] - default fn spec_clone_from(&mut self, source: &[T]) { - self.clone_from_fallback(source); - } - } - - impl SpecCloneFrom for SmallVec { - fn spec_clone_from(&mut self, source: &[T]) { - self.clear(); - self.extend_from_slice(source); - } - } - - /// A trait for specializing the implementation of [`From`] - /// with the source type being slices. - pub(crate) trait SpecFromSlice { - /// Creates a `SmallVec` value based on the contents of `slice`. - /// This will use the inline storage, not the heap. - /// - /// # Safety - /// - /// The caller must ensure that `slice.len() <= Self::inline_size()`. - unsafe fn spec_from(slice: &[T]) -> Self; - } - - impl SpecFromSlice for SmallVec { - default unsafe fn spec_from(slice: &[T]) -> Self { - // SAFETY: Safety conditions are identical. - unsafe { Self::from_slice_fallback(slice) } - } - } - - impl SpecFromSlice for SmallVec { - unsafe fn spec_from(slice: &[T]) -> Self { - let mut v = Self::new(); - - let src = slice.as_ptr(); - let len = slice.len(); - let dst = v.as_mut_ptr(); - - // SAFETY: The caller ensures that the slice length is smaller - // than or equal to the inline length. - unsafe { - copy_nonoverlapping(src, dst, len); - } - - // SAFETY: The elements were initialized above. - unsafe { - v.set_len(len); - } - - v - } - } -} +mod spec_traits; /// Fallback functions for various specialized methods. These are kept in /// a separate implementation block for easy access whenever specialization is disabled. diff --git a/src/spec_traits.rs b/src/spec_traits.rs new file mode 100644 index 00000000..3f716eda --- /dev/null +++ b/src/spec_traits.rs @@ -0,0 +1,306 @@ +use super::*; + +/// A trait for specializing the implementation of [`from_elem`]. +/// +/// [`from_elem`]: crate::from_elem +pub(crate) trait SpecFromElem { + /// Creates a `Smallvec` value where `elem` is repeated `n` times. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `n <= Self::inline_size()`. + unsafe fn spec_from_elem(elem: T, n: usize) -> Self; +} + +impl SpecFromElem for SmallVec { + #[inline] + default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + // SAFETY: Safety conditions are identical. + unsafe { SmallVec::from_elem_fallback(elem, n) } + } +} + +impl SpecFromElem for SmallVec { + unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + let mut result = Self::new(); + + if n > 0 { + let ptr = result.raw.as_mut_ptr_inline(); + + // SAFETY: The caller ensures that the first `n` + // is smaller than the inline size. + unsafe { + for i in 0..n { + ptr.add(i).write(elem); + } + } + } + + // SAFETY: The first `n` elements of the vector + // have been initialized in the loop above. + unsafe { + result.set_len(n); + } + + result + } +} + +/// A trait for specializing the implementations of [`Extend`] and [`extend_from_slice`]. +/// +/// [`extend_from_slice`]: crate::SmallVec::extend_from_slice +pub(crate) trait SpecExtend { + fn spec_extend(&mut self, iter: I); +} + +impl SpecExtend for SmallVec +where + I: Iterator, +{ + #[inline] + default fn spec_extend(&mut self, iter: I) { + self.extend_fallback(iter); + } +} + +impl SpecExtend for SmallVec +where + I: core::iter::TrustedLen, +{ + fn spec_extend(&mut self, iter: I) { + let (_, Some(additional)) = iter.size_hint() else { + panic!("capacity overflow") + }; + self.reserve(additional); + + // SAFETY: A `TrustedLen` iterator provides accurate information + // about its size, which was used to reserve additional memory. + // This ensures that the access operations inside the loop always + // operate on valid memory. + unsafe { + let len = self.len(); + let ptr = self.as_mut_ptr().add(len); + let mut guard = DropGuard { ptr, len: 0 }; + + for x in iter { + ptr.add(guard.len).write(x); + guard.len += 1; + } + + // The elements have been initialized in the loop above. + self.set_len(len + guard.len); + core::mem::forget(guard); + } + } +} + +impl SpecExtend> for SmallVec { + fn spec_extend(&mut self, mut iter: IntoIter) { + let slice = iter.as_slice(); + let len = slice.len(); + let old_len = self.len(); + + self.reserve(len); + + // SAFETY: Additional memory has been reserved above. + // Therefore, the copy operates on valid memory. + unsafe { + let dst = self.as_mut_ptr().add(old_len); + let src = slice.as_ptr(); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + + // Mark the iterator as fully consumed. + iter.begin = iter.end.value(); + } +} + +impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec +where + I: Iterator, + T: Clone, +{ + #[inline] + default fn spec_extend(&mut self, iterator: I) { + self.spec_extend(iterator.cloned()) + } +} + +impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec +where + T: Copy, +{ + fn spec_extend(&mut self, iter: core::slice::Iter<'a, T>) { + let slice = iter.as_slice(); + let len = slice.len(); + let old_len = self.len(); + + self.reserve(len); + + // SAFETY: Additional memory has been reserved above. + // Therefore, the copy operates on valid memory. + unsafe { + let dst = self.as_mut_ptr().add(old_len); + let src = slice.as_ptr(); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + } +} + +/// A trait for specializing the implementation of [`extend_from_within`]. +/// +/// [`extend_from_within`]: crate::SmallVec::extend_from_within +pub(crate) trait SpecExtendFromWithin { + /// Main worker for [`extend_from_within`]. + /// + /// # Safety + /// + /// * The length of the vector is larger than or equal to `src.len()`. + /// * The spare capacity of the vector is larger than or equal to `src.len()`. + /// + /// [`extend_from_within`]: SmallVec::extend_from_within + unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); +} + +impl SpecExtendFromWithin for SmallVec { + default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { + // SAFETY: Safety conditions are identical. + unsafe { + self.extend_from_within_fallback(src); + } + } +} + +impl SpecExtendFromWithin for SmallVec { + unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { + let old_len = self.len(); + + let start = src.start; + let len = src.len(); + + // SAFETY: The caller ensures that the vector has spare capacity + // for at least `src.len()` elements. This is also the amount of memory + // accessed when the data is copied. + unsafe { + let ptr = self.as_mut_ptr(); + let dst = ptr.add(old_len); + let src = ptr.add(start); + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + self.set_len(old_len + len); + } + } +} + +/// A trait for specializing the implementation of [`FromIterator`]. +/// +/// [`clone_from`]: Clone::clone_from +pub(crate) trait SpecFromIterator { + fn spec_from_iter(iter: I) -> Self; +} + +impl SpecFromIterator for SmallVec +where + I: Iterator, +{ + #[inline] + default fn spec_from_iter(iter: I) -> Self { + Self::from_iter_fallback(iter) + } +} + +impl SpecFromIterator for SmallVec +where + I: core::iter::TrustedLen, +{ + fn spec_from_iter(iter: I) -> Self { + let mut v = match iter.size_hint() { + (_, Some(upper)) => SmallVec::with_capacity(upper), + // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there + // are more than `usize::MAX` elements. + // Since the previous branch would eagerly panic if the capacity is too large + // (via `with_capacity`) we do the same here. + _ => panic!("capacity overflow"), + }; + // Reuse the extend specialization for TrustedLen. + v.spec_extend(iter); + v + } +} + +/// A trait for specializing the implementation of [`clone_from`]. +/// +/// [`clone_from`]: Clone::clone_from +pub(crate) trait SpecCloneFrom { + fn spec_clone_from(&mut self, source: &[T]); +} + +impl SpecCloneFrom for SmallVec { + #[inline] + default fn spec_clone_from(&mut self, source: &[T]) { + self.clone_from_fallback(source); + } +} + +impl SpecCloneFrom for SmallVec { + fn spec_clone_from(&mut self, source: &[T]) { + self.clear(); + self.extend_from_slice(source); + } +} + +/// A trait for specializing the implementation of [`From`] +/// with the source type being slices. +pub(crate) trait SpecFromSlice { + /// Creates a `SmallVec` value based on the contents of `slice`. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `slice.len() <= Self::inline_size()`. + unsafe fn spec_from(slice: &[T]) -> Self; +} + +impl SpecFromSlice for SmallVec { + default unsafe fn spec_from(slice: &[T]) -> Self { + // SAFETY: Safety conditions are identical. + unsafe { Self::from_slice_fallback(slice) } + } +} + +impl SpecFromSlice for SmallVec { + unsafe fn spec_from(slice: &[T]) -> Self { + let mut v = Self::new(); + + let src = slice.as_ptr(); + let len = slice.len(); + let dst = v.as_mut_ptr(); + + // SAFETY: The caller ensures that the slice length is smaller + // than or equal to the inline length. + unsafe { + copy_nonoverlapping(src, dst, len); + } + + // SAFETY: The elements were initialized above. + unsafe { + v.set_len(len); + } + + v + } +}