1 //! Memory allocation APIs
3 #![stable(feature = "alloc_module", since = "1.28.0")]
7 use core::intrinsics::{min_align_of_val, size_of_val};
11 use core::ptr::{self, NonNull};
13 #[stable(feature = "alloc_module", since = "1.28.0")]
15 pub use core::alloc::*;
21 // These are the magic symbols to call the global allocator. rustc generates
22 // them to call `__rg_alloc` etc. if there is a `#[global_allocator]` attribute
23 // (the code expanding that attribute macro generates those functions), or to call
24 // the default implementations in libstd (`__rdl_alloc` etc. in `library/std/src/alloc.rs`)
26 // The rustc fork of LLVM also special-cases these function names to be able to optimize them
27 // like `malloc`, `realloc`, and `free`, respectively.
29 #[rustc_allocator_nounwind]
30 fn __rust_alloc(size: usize, align: usize) -> *mut u8;
31 #[rustc_allocator_nounwind]
32 fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize);
33 #[rustc_allocator_nounwind]
34 fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8;
35 #[rustc_allocator_nounwind]
36 fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8;
39 /// The global memory allocator.
41 /// This type implements the [`Allocator`] trait by forwarding calls
42 /// to the allocator registered with the `#[global_allocator]` attribute
43 /// if there is one, or the `std` crate’s default.
45 /// Note: while this type is unstable, the functionality it provides can be
46 /// accessed through the [free functions in `alloc`](self#functions).
47 #[unstable(feature = "allocator_api", issue = "32838")]
48 #[derive(Copy, Clone, Default, Debug)]
53 pub use std::alloc::Global;
55 /// Allocate memory with the global allocator.
57 /// This function forwards calls to the [`GlobalAlloc::alloc`] method
58 /// of the allocator registered with the `#[global_allocator]` attribute
59 /// if there is one, or the `std` crate’s default.
61 /// This function is expected to be deprecated in favor of the `alloc` method
62 /// of the [`Global`] type when it and the [`Allocator`] trait become stable.
66 /// See [`GlobalAlloc::alloc`].
71 /// use std::alloc::{alloc, dealloc, Layout};
74 /// let layout = Layout::new::<u16>();
75 /// let ptr = alloc(layout);
77 /// *(ptr as *mut u16) = 42;
78 /// assert_eq!(*(ptr as *mut u16), 42);
80 /// dealloc(ptr, layout);
83 #[stable(feature = "global_alloc", since = "1.28.0")]
85 pub unsafe fn alloc(layout: Layout) -> *mut u8 {
86 unsafe { __rust_alloc(layout.size(), layout.align()) }
89 /// Deallocate memory with the global allocator.
91 /// This function forwards calls to the [`GlobalAlloc::dealloc`] method
92 /// of the allocator registered with the `#[global_allocator]` attribute
93 /// if there is one, or the `std` crate’s default.
95 /// This function is expected to be deprecated in favor of the `dealloc` method
96 /// of the [`Global`] type when it and the [`Allocator`] trait become stable.
100 /// See [`GlobalAlloc::dealloc`].
101 #[stable(feature = "global_alloc", since = "1.28.0")]
103 pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) {
104 unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) }
107 /// Reallocate memory with the global allocator.
109 /// This function forwards calls to the [`GlobalAlloc::realloc`] method
110 /// of the allocator registered with the `#[global_allocator]` attribute
111 /// if there is one, or the `std` crate’s default.
113 /// This function is expected to be deprecated in favor of the `realloc` method
114 /// of the [`Global`] type when it and the [`Allocator`] trait become stable.
118 /// See [`GlobalAlloc::realloc`].
119 #[stable(feature = "global_alloc", since = "1.28.0")]
121 pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
122 unsafe { __rust_realloc(ptr, layout.size(), layout.align(), new_size) }
125 /// Allocate zero-initialized memory with the global allocator.
127 /// This function forwards calls to the [`GlobalAlloc::alloc_zeroed`] method
128 /// of the allocator registered with the `#[global_allocator]` attribute
129 /// if there is one, or the `std` crate’s default.
131 /// This function is expected to be deprecated in favor of the `alloc_zeroed` method
132 /// of the [`Global`] type when it and the [`Allocator`] trait become stable.
136 /// See [`GlobalAlloc::alloc_zeroed`].
141 /// use std::alloc::{alloc_zeroed, dealloc, Layout};
144 /// let layout = Layout::new::<u16>();
145 /// let ptr = alloc_zeroed(layout);
147 /// assert_eq!(*(ptr as *mut u16), 0);
149 /// dealloc(ptr, layout);
152 #[stable(feature = "global_alloc", since = "1.28.0")]
154 pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 {
155 unsafe { __rust_alloc_zeroed(layout.size(), layout.align()) }
161 fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result<NonNull<[u8]>, AllocError> {
162 match layout.size() {
163 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)),
164 // SAFETY: `layout` is non-zero in size,
166 let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) };
167 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
168 Ok(NonNull::slice_from_raw_parts(ptr, size))
173 // SAFETY: Same as `Allocator::grow`
181 ) -> Result<NonNull<[u8]>, AllocError> {
183 new_layout.size() >= old_layout.size(),
184 "`new_layout.size()` must be greater than or equal to `old_layout.size()`"
187 match old_layout.size() {
188 0 => self.alloc_impl(new_layout, zeroed),
190 // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size`
191 // as required by safety conditions. Other conditions must be upheld by the caller
192 old_size if old_layout.align() == new_layout.align() => unsafe {
193 let new_size = new_layout.size();
195 // `realloc` probably checks for `new_size >= old_layout.size()` or something similar.
196 intrinsics::assume(new_size >= old_layout.size());
198 let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size);
199 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
201 raw_ptr.add(old_size).write_bytes(0, new_size - old_size);
203 Ok(NonNull::slice_from_raw_parts(ptr, new_size))
206 // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`,
207 // both the old and new memory allocation are valid for reads and writes for `old_size`
208 // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap
209 // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract
210 // for `dealloc` must be upheld by the caller.
212 let new_ptr = self.alloc_impl(new_layout, zeroed)?;
213 ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size);
214 self.deallocate(ptr, old_layout);
221 #[unstable(feature = "allocator_api", issue = "32838")]
223 unsafe impl Allocator for Global {
225 fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
226 self.alloc_impl(layout, false)
230 fn allocate_zeroed(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
231 self.alloc_impl(layout, true)
235 unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
236 if layout.size() != 0 {
237 // SAFETY: `layout` is non-zero in size,
238 // other conditions must be upheld by the caller
239 unsafe { dealloc(ptr.as_ptr(), layout) }
249 ) -> Result<NonNull<[u8]>, AllocError> {
250 // SAFETY: all conditions must be upheld by the caller
251 unsafe { self.grow_impl(ptr, old_layout, new_layout, false) }
255 unsafe fn grow_zeroed(
260 ) -> Result<NonNull<[u8]>, AllocError> {
261 // SAFETY: all conditions must be upheld by the caller
262 unsafe { self.grow_impl(ptr, old_layout, new_layout, true) }
271 ) -> Result<NonNull<[u8]>, AllocError> {
273 new_layout.size() <= old_layout.size(),
274 "`new_layout.size()` must be smaller than or equal to `old_layout.size()`"
277 match new_layout.size() {
278 // SAFETY: conditions must be upheld by the caller
280 self.deallocate(ptr, old_layout);
281 Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0))
284 // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller
285 new_size if old_layout.align() == new_layout.align() => unsafe {
286 // `realloc` probably checks for `new_size <= old_layout.size()` or something similar.
287 intrinsics::assume(new_size <= old_layout.size());
289 let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size);
290 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
291 Ok(NonNull::slice_from_raw_parts(ptr, new_size))
294 // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`,
295 // both the old and new memory allocation are valid for reads and writes for `new_size`
296 // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap
297 // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract
298 // for `dealloc` must be upheld by the caller.
300 let new_ptr = self.allocate(new_layout)?;
301 ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size);
302 self.deallocate(ptr, old_layout);
309 /// The allocator for unique pointers.
310 // This function must not unwind. If it does, MIR codegen will fail.
311 #[cfg(all(not(no_global_oom_handling), not(test)))]
312 #[lang = "exchange_malloc"]
314 unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
315 let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
316 match Global.allocate(layout) {
317 Ok(ptr) => ptr.as_mut_ptr(),
318 Err(_) => handle_alloc_error(layout),
322 #[cfg_attr(not(test), lang = "box_free")]
324 // This signature has to be the same as `Box`, otherwise an ICE will happen.
325 // When an additional parameter to `Box` is added (like `A: Allocator`), this has to be added here as
327 // For example if `Box` is changed to `struct Box<T: ?Sized, A: Allocator>(Unique<T>, A)`,
328 // this function has to be changed to `fn box_free<T: ?Sized, A: Allocator>(Unique<T>, A)` as well.
329 pub(crate) unsafe fn box_free<T: ?Sized, A: Allocator>(ptr: Unique<T>, alloc: A) {
331 let size = size_of_val(ptr.as_ref());
332 let align = min_align_of_val(ptr.as_ref());
333 let layout = Layout::from_size_align_unchecked(size, align);
334 alloc.deallocate(ptr.cast().into(), layout)
338 // # Allocation error handler
340 #[cfg(not(no_global_oom_handling))]
342 // This is the magic symbol to call the global alloc error handler. rustc generates
343 // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the
344 // default implementations below (`__rdl_oom`) otherwise.
345 #[rustc_allocator_nounwind]
346 fn __rust_alloc_error_handler(size: usize, align: usize) -> !;
349 /// Abort on memory allocation error or failure.
351 /// Callers of memory allocation APIs wishing to abort computation
352 /// in response to an allocation error are encouraged to call this function,
353 /// rather than directly invoking `panic!` or similar.
355 /// The default behavior of this function is to print a message to standard error
356 /// and abort the process.
357 /// It can be replaced with [`set_alloc_error_hook`] and [`take_alloc_error_hook`].
359 /// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html
360 /// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html
361 #[stable(feature = "global_alloc", since = "1.28.0")]
362 #[cfg(all(not(no_global_oom_handling), not(test)))]
363 #[rustc_allocator_nounwind]
365 pub fn handle_alloc_error(layout: Layout) -> ! {
367 __rust_alloc_error_handler(layout.size(), layout.align());
371 // For alloc test `std::alloc::handle_alloc_error` can be used directly.
372 #[cfg(all(not(no_global_oom_handling), test))]
373 pub use std::alloc::handle_alloc_error;
375 #[cfg(all(not(no_global_oom_handling), not(any(target_os = "hermit", test))))]
377 #[allow(unused_attributes)]
378 #[unstable(feature = "alloc_internals", issue = "none")]
379 pub mod __alloc_error_handler {
380 use crate::alloc::Layout;
382 // called via generated `__rust_alloc_error_handler`
384 // if there is no `#[alloc_error_handler]`
385 #[rustc_std_internal_symbol]
386 pub unsafe extern "C" fn __rdl_oom(size: usize, _align: usize) -> ! {
387 panic!("memory allocation of {} bytes failed", size)
390 // if there is a `#[alloc_error_handler]`
391 #[rustc_std_internal_symbol]
392 pub unsafe extern "C" fn __rg_oom(size: usize, align: usize) -> ! {
393 let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
396 fn oom_impl(layout: Layout) -> !;
398 unsafe { oom_impl(layout) }
402 /// Specialize clones into pre-allocated, uninitialized memory.
403 /// Used by `Box::clone` and `Rc`/`Arc::make_mut`.
404 pub(crate) trait WriteCloneIntoRaw: Sized {
405 unsafe fn write_clone_into_raw(&self, target: *mut Self);
408 impl<T: Clone> WriteCloneIntoRaw for T {
410 default unsafe fn write_clone_into_raw(&self, target: *mut Self) {
411 // Having allocated *first* may allow the optimizer to create
412 // the cloned value in-place, skipping the local and move.
413 unsafe { target.write(self.clone()) };
417 impl<T: Copy> WriteCloneIntoRaw for T {
419 unsafe fn write_clone_into_raw(&self, target: *mut Self) {
420 // We can always copy in-place, without ever involving a local value.
421 unsafe { target.copy_from_nonoverlapping(self, 1) };