1 // Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 // FIXME: #13996: mark the `allocate` and `reallocate` return value as `noalias`
13 /// Return a pointer to `size` bytes of memory aligned to `align`.
15 /// On failure, return a null pointer.
17 /// Behavior is undefined if the requested size is 0 or the alignment is not a
18 /// power of 2. The alignment must be no larger than the largest supported page
19 /// size on the platform.
21 pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
22 imp::allocate(size, align)
25 /// Resize the allocation referenced by `ptr` to `size` bytes.
27 /// On failure, return a null pointer and leave the original allocation intact.
29 /// If the allocation was relocated, the memory at the passed-in pointer is
30 /// undefined after the call.
32 /// Behavior is undefined if the requested size is 0 or the alignment is not a
33 /// power of 2. The alignment must be no larger than the largest supported page
34 /// size on the platform.
36 /// The `old_size` and `align` parameters are the parameters that were used to
37 /// create the allocation referenced by `ptr`. The `old_size` parameter may be
38 /// any value in range_inclusive(requested_size, usable_size).
40 pub unsafe fn reallocate(ptr: *mut u8, old_size: usize, size: usize, align: usize) -> *mut u8 {
41 imp::reallocate(ptr, old_size, size, align)
44 /// Resize the allocation referenced by `ptr` to `size` bytes.
46 /// If the operation succeeds, it returns `usable_size(size, align)` and if it
47 /// fails (or is a no-op) it returns `usable_size(old_size, align)`.
49 /// Behavior is undefined if the requested size is 0 or the alignment is not a
50 /// power of 2. The alignment must be no larger than the largest supported page
51 /// size on the platform.
53 /// The `old_size` and `align` parameters are the parameters that were used to
54 /// create the allocation referenced by `ptr`. The `old_size` parameter may be
55 /// any value in range_inclusive(requested_size, usable_size).
57 pub unsafe fn reallocate_inplace(ptr: *mut u8, old_size: usize, size: usize,
58 align: usize) -> usize {
59 imp::reallocate_inplace(ptr, old_size, size, align)
62 /// Deallocates the memory referenced by `ptr`.
64 /// The `ptr` parameter must not be null.
66 /// The `old_size` and `align` parameters are the parameters that were used to
67 /// create the allocation referenced by `ptr`. The `old_size` parameter may be
68 /// any value in range_inclusive(requested_size, usable_size).
70 pub unsafe fn deallocate(ptr: *mut u8, old_size: usize, align: usize) {
71 imp::deallocate(ptr, old_size, align)
74 /// Returns the usable size of an allocation created with the specified the
75 /// `size` and `align`.
77 pub fn usable_size(size: usize, align: usize) -> usize {
78 imp::usable_size(size, align)
81 /// Prints implementation-defined allocator statistics.
83 /// These statistics may be inconsistent if other threads use the allocator
85 #[unstable(feature = "alloc")]
86 pub fn stats_print() {
90 /// An arbitrary non-null address to represent zero-size allocations.
92 /// This preserves the non-null invariant for types like `Box<T>`. The address may overlap with
93 /// non-zero-size memory allocations.
94 pub const EMPTY: *mut () = 0x1 as *mut ();
96 /// The allocator for unique pointers.
98 #[lang="exchange_malloc"]
100 unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
104 let ptr = allocate(size, align);
105 if ptr.is_null() { ::oom() }
111 #[lang="exchange_free"]
113 unsafe fn exchange_free(ptr: *mut u8, old_size: usize, align: usize) {
114 deallocate(ptr, old_size, align);
117 // The minimum alignment guaranteed by the architecture. This value is used to
118 // add fast paths for low alignment values. In practice, the alignment is a
119 // constant at the call site and the branch will be optimized out.
120 #[cfg(all(not(feature = "external_funcs"),
121 not(feature = "external_crate"),
122 any(target_arch = "arm",
123 target_arch = "mips",
124 target_arch = "mipsel",
125 target_arch = "powerpc")))]
126 const MIN_ALIGN: usize = 8;
127 #[cfg(all(not(feature = "external_funcs"),
128 not(feature = "external_crate"),
129 any(target_arch = "x86",
130 target_arch = "x86_64",
131 target_arch = "aarch64")))]
132 const MIN_ALIGN: usize = 16;
134 #[cfg(feature = "external_funcs")]
137 fn rust_allocate(size: usize, align: usize) -> *mut u8;
138 fn rust_deallocate(ptr: *mut u8, old_size: usize, align: usize);
139 fn rust_reallocate(ptr: *mut u8, old_size: usize, size: usize, align: usize) -> *mut u8;
140 fn rust_reallocate_inplace(ptr: *mut u8, old_size: usize, size: usize,
141 align: usize) -> usize;
142 fn rust_usable_size(size: usize, align: usize) -> usize;
143 fn rust_stats_print();
147 pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
148 rust_allocate(size, align)
152 pub unsafe fn deallocate(ptr: *mut u8, old_size: usize, align: usize) {
153 rust_deallocate(ptr, old_size, align)
157 pub unsafe fn reallocate(ptr: *mut u8, old_size: usize, size: usize, align: usize) -> *mut u8 {
158 rust_reallocate(ptr, old_size, size, align)
162 pub unsafe fn reallocate_inplace(ptr: *mut u8, old_size: usize, size: usize,
163 align: usize) -> usize {
164 rust_reallocate_inplace(ptr, old_size, size, align)
168 pub fn usable_size(size: usize, align: usize) -> usize {
169 unsafe { rust_usable_size(size, align) }
173 pub fn stats_print() {
174 unsafe { rust_stats_print() }
178 #[cfg(feature = "external_crate")]
180 extern crate external;
181 pub use self::external::{allocate, deallocate, reallocate_inplace, reallocate};
182 pub use self::external::{usable_size, stats_print};
185 #[cfg(all(not(feature = "external_funcs"),
186 not(feature = "external_crate"),
189 use core::option::Option;
190 use core::option::Option::None;
191 use core::ptr::{null_mut, null};
192 use libc::{c_char, c_int, c_void, size_t};
193 use super::MIN_ALIGN;
195 #[link(name = "jemalloc", kind = "static")]
201 fn je_mallocx(size: size_t, flags: c_int) -> *mut c_void;
202 fn je_rallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void;
203 fn je_xallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t;
204 fn je_sdallocx(ptr: *mut c_void, size: size_t, flags: c_int);
205 fn je_nallocx(size: size_t, flags: c_int) -> size_t;
206 fn je_malloc_stats_print(write_cb: Option<extern "C" fn(cbopaque: *mut c_void,
208 cbopaque: *mut c_void,
209 opts: *const c_char);
212 // -lpthread needs to occur after -ljemalloc, the earlier argument isn't enough
213 #[cfg(all(not(windows), not(target_os = "android")))]
214 #[link(name = "pthread")]
217 // MALLOCX_ALIGN(a) macro
219 fn mallocx_align(a: usize) -> c_int { a.trailing_zeros() as c_int }
222 fn align_to_flags(align: usize) -> c_int {
223 if align <= MIN_ALIGN { 0 } else { mallocx_align(align) }
227 pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
228 let flags = align_to_flags(align);
229 je_mallocx(size as size_t, flags) as *mut u8
233 pub unsafe fn reallocate(ptr: *mut u8, _old_size: usize, size: usize, align: usize) -> *mut u8 {
234 let flags = align_to_flags(align);
235 je_rallocx(ptr as *mut c_void, size as size_t, flags) as *mut u8
239 pub unsafe fn reallocate_inplace(ptr: *mut u8, _old_size: usize, size: usize,
240 align: usize) -> usize {
241 let flags = align_to_flags(align);
242 je_xallocx(ptr as *mut c_void, size as size_t, 0, flags) as usize
246 pub unsafe fn deallocate(ptr: *mut u8, old_size: usize, align: usize) {
247 let flags = align_to_flags(align);
248 je_sdallocx(ptr as *mut c_void, old_size as size_t, flags)
252 pub fn usable_size(size: usize, align: usize) -> usize {
253 let flags = align_to_flags(align);
254 unsafe { je_nallocx(size as size_t, flags) as usize }
257 pub fn stats_print() {
259 je_malloc_stats_print(None, null_mut(), null())
264 #[cfg(all(not(feature = "external_funcs"),
265 not(feature = "external_crate"),
272 use super::MIN_ALIGN;
275 fn posix_memalign(memptr: *mut *mut libc::c_void,
277 size: libc::size_t) -> libc::c_int;
281 pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
282 if align <= MIN_ALIGN {
283 libc::malloc(size as libc::size_t) as *mut u8
285 let mut out = ptr::null_mut();
286 let ret = posix_memalign(&mut out,
287 align as libc::size_t,
288 size as libc::size_t);
298 pub unsafe fn reallocate(ptr: *mut u8, old_size: usize, size: usize, align: usize) -> *mut u8 {
299 if align <= MIN_ALIGN {
300 libc::realloc(ptr as *mut libc::c_void, size as libc::size_t) as *mut u8
302 let new_ptr = allocate(size, align);
303 ptr::copy(ptr, new_ptr, cmp::min(size, old_size));
304 deallocate(ptr, old_size, align);
310 pub unsafe fn reallocate_inplace(_ptr: *mut u8, old_size: usize, _size: usize,
311 _align: usize) -> usize {
316 pub unsafe fn deallocate(ptr: *mut u8, _old_size: usize, _align: usize) {
317 libc::free(ptr as *mut libc::c_void)
321 pub fn usable_size(size: usize, _align: usize) -> usize {
325 pub fn stats_print() {}
328 #[cfg(all(not(feature = "external_funcs"),
329 not(feature = "external_crate"),
333 use libc::{c_void, size_t};
335 use super::MIN_ALIGN;
338 fn _aligned_malloc(size: size_t, align: size_t) -> *mut c_void;
339 fn _aligned_realloc(block: *mut c_void, size: size_t,
340 align: size_t) -> *mut c_void;
341 fn _aligned_free(ptr: *mut c_void);
345 pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
346 if align <= MIN_ALIGN {
347 libc::malloc(size as size_t) as *mut u8
349 _aligned_malloc(size as size_t, align as size_t) as *mut u8
354 pub unsafe fn reallocate(ptr: *mut u8, _old_size: usize, size: usize, align: usize) -> *mut u8 {
355 if align <= MIN_ALIGN {
356 libc::realloc(ptr as *mut c_void, size as size_t) as *mut u8
358 _aligned_realloc(ptr as *mut c_void, size as size_t, align as size_t) as *mut u8
363 pub unsafe fn reallocate_inplace(_ptr: *mut u8, old_size: usize, _size: usize,
364 _align: usize) -> usize {
369 pub unsafe fn deallocate(ptr: *mut u8, _old_size: usize, align: usize) {
370 if align <= MIN_ALIGN {
371 libc::free(ptr as *mut libc::c_void)
373 _aligned_free(ptr as *mut c_void)
378 pub fn usable_size(size: usize, _align: usize) -> usize {
382 pub fn stats_print() {}
388 use self::test::Bencher;
393 fn basic_reallocate_inplace_noop() {
396 let ptr = heap::allocate(size, 8);
397 if ptr.is_null() { ::oom() }
398 let ret = heap::reallocate_inplace(ptr, size, size, 8);
399 heap::deallocate(ptr, size, 8);
400 assert_eq!(ret, heap::usable_size(size, 8));
405 fn alloc_owned_small(b: &mut Bencher) {
407 let _: Box<_> = box 10;