unsafe impl<T> Sync for Lazy<T> {}
impl<T> Lazy<T> {
+ #[unstable(feature = "sys_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> Lazy<T> {
Lazy {
lock: Mutex::new(),
pub mod thread;
pub mod tls;
#[macro_use]
-mod usercalls;
+pub mod usercalls;
global_asm!(concat!(usercalls_asm!(), include_str!("entry.S")));
// (main function exists). If this is a library, the crate author should be
// able to specify this
#[no_mangle]
-#[allow(unreachable_code)]
extern "C" fn entry(p1: u64, p2: u64, p3: u64, secondary: bool, p4: u64, p5: u64) -> (u64, u64) {
// FIXME: how to support TLS in library mode?
let tls = Box::new(tls::Tls::new());
let _tls_guard = unsafe { tls.activate() };
if secondary {
- unimplemented!("thread entrypoint");
+ super::thread::Thread::entry();
(0, 0)
} else {
#[macro_use]
mod raw;
+pub fn launch_thread() -> IoResult<()> {
+ unsafe { raw::launch_thread().from_sgx_result() }
+}
+
pub fn exit(panic: bool) -> ! {
unsafe { raw::exit(panic) }
}
+pub fn wait(event_mask: u64, timeout: u64) -> IoResult<u64> {
+ unsafe { raw::wait(event_mask, timeout).from_sgx_result() }
+}
+
+pub fn send(event_set: u64, tcs: Option<Tcs>) -> IoResult<()> {
+ unsafe { raw::send(event_set, tcs).from_sgx_result() }
+}
+
pub fn alloc(size: usize, alignment: usize) -> IoResult<*mut u8> {
unsafe { raw::alloc(size, alignment).from_sgx_result() }
}
use alloc::{GlobalAlloc, Layout, System};
-// FIXME: protect this value for concurrent access
-static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT;
+use super::waitqueue::SpinMutex;
+
+// Using a SpinMutex because we never want to exit the enclave waiting for the
+// allocator.
+static DLMALLOC: SpinMutex<dlmalloc::Dlmalloc> = SpinMutex::new(dlmalloc::DLMALLOC_INIT);
#[stable(feature = "alloc_system_type", since = "1.28.0")]
unsafe impl GlobalAlloc for System {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
- DLMALLOC.malloc(layout.size(), layout.align())
+ DLMALLOC.lock().malloc(layout.size(), layout.align())
}
#[inline]
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
- DLMALLOC.calloc(layout.size(), layout.align())
+ DLMALLOC.lock().calloc(layout.size(), layout.align())
}
#[inline]
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
- DLMALLOC.free(ptr, layout.size(), layout.align())
+ DLMALLOC.lock().free(ptr, layout.size(), layout.align())
}
#[inline]
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
- DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size)
+ DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size)
}
}
use sys::mutex::Mutex;
use time::Duration;
-pub struct Condvar { }
+use super::waitqueue::{WaitVariable, WaitQueue, SpinMutex};
+
+pub struct Condvar {
+ inner: SpinMutex<WaitVariable<()>>,
+}
impl Condvar {
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> Condvar {
- Condvar { }
+ Condvar { inner: SpinMutex::new(WaitVariable::new(())) }
}
#[inline]
#[inline]
pub unsafe fn notify_one(&self) {
+ let _ = WaitQueue::notify_one(self.inner.lock());
}
#[inline]
pub unsafe fn notify_all(&self) {
+ let _ = WaitQueue::notify_all(self.inner.lock());
}
- pub unsafe fn wait(&self, _mutex: &Mutex) {
- panic!("can't block with web assembly")
+ pub unsafe fn wait(&self, mutex: &Mutex) {
+ let guard = self.inner.lock();
+ mutex.unlock();
+ WaitQueue::wait(guard);
+ mutex.lock()
}
pub unsafe fn wait_timeout(&self, _mutex: &Mutex, _dur: Duration) -> bool {
- panic!("can't block with web assembly");
+ panic!("timeout not supported in SGX");
}
#[inline]
- pub unsafe fn destroy(&self) {
- }
+ pub unsafe fn destroy(&self) {}
}
use sync::atomic::{AtomicBool, Ordering};
pub mod abi;
+mod waitqueue;
pub mod alloc;
pub mod args;
// option. This file may not be copied, modified, or distributed
// except according to those terms.
-use cell::UnsafeCell;
+use fortanix_sgx_abi::Tcs;
+
+use super::abi::thread;
+
+use super::waitqueue::{WaitVariable, WaitQueue, SpinMutex, NotifiedTcs, try_lock_or_false};
pub struct Mutex {
- locked: UnsafeCell<bool>,
+ inner: SpinMutex<WaitVariable<bool>>,
}
-unsafe impl Send for Mutex {}
-unsafe impl Sync for Mutex {} // FIXME
-
+// Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28
impl Mutex {
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> Mutex {
- Mutex { locked: UnsafeCell::new(false) }
+ Mutex { inner: SpinMutex::new(WaitVariable::new(false)) }
}
#[inline]
- pub unsafe fn init(&mut self) {
- }
+ pub unsafe fn init(&mut self) {}
#[inline]
pub unsafe fn lock(&self) {
- let locked = self.locked.get();
- assert!(!*locked, "cannot recursively acquire mutex");
- *locked = true;
+ let mut guard = self.inner.lock();
+ if *guard.lock_var() {
+ // Another thread has the lock, wait
+ WaitQueue::wait(guard)
+ // Another thread has passed the lock to us
+ } else {
+ // We are just now obtaining the lock
+ *guard.lock_var_mut() = true;
+ }
}
#[inline]
pub unsafe fn unlock(&self) {
- *self.locked.get() = false;
+ let guard = self.inner.lock();
+ if let Err(mut guard) = WaitQueue::notify_one(guard) {
+ // No other waiters, unlock
+ *guard.lock_var_mut() = false;
+ } else {
+ // There was a thread waiting, just pass the lock
+ }
}
#[inline]
pub unsafe fn try_lock(&self) -> bool {
- let locked = self.locked.get();
- if *locked {
+ let mut guard = try_lock_or_false!(self.inner);
+ if *guard.lock_var() {
+ // Another thread has the lock
false
} else {
- *locked = true;
+ // We are just now obtaining the lock
+ *guard.lock_var_mut() = true;
true
}
}
#[inline]
- pub unsafe fn destroy(&self) {
- }
+ pub unsafe fn destroy(&self) {}
+}
+
+struct ReentrantLock {
+ owner: Option<Tcs>,
+ count: usize
}
-// FIXME
pub struct ReentrantMutex {
+ inner: SpinMutex<WaitVariable<ReentrantLock>>,
}
impl ReentrantMutex {
- pub unsafe fn uninitialized() -> ReentrantMutex {
- ReentrantMutex { }
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
+ pub const fn uninitialized() -> ReentrantMutex {
+ ReentrantMutex {
+ inner: SpinMutex::new(WaitVariable::new(ReentrantLock { owner: None, count: 0 }))
+ }
}
+ #[inline]
pub unsafe fn init(&mut self) {}
- pub unsafe fn lock(&self) {}
+ #[inline]
+ pub unsafe fn lock(&self) {
+ let mut guard = self.inner.lock();
+ match guard.lock_var().owner {
+ Some(tcs) if tcs != thread::current() => {
+ // Another thread has the lock, wait
+ WaitQueue::wait(guard);
+ // Another thread has passed the lock to us
+ },
+ _ => {
+ // We are just now obtaining the lock
+ guard.lock_var_mut().owner = Some(thread::current());
+ guard.lock_var_mut().count += 1;
+ },
+ }
+ }
#[inline]
- pub unsafe fn try_lock(&self) -> bool {
- true
+ pub unsafe fn unlock(&self) {
+ let mut guard = self.inner.lock();
+ if guard.lock_var().count > 1 {
+ guard.lock_var_mut().count -= 1;
+ } else {
+ match WaitQueue::notify_one(guard) {
+ Err(mut guard) => {
+ // No other waiters, unlock
+ guard.lock_var_mut().count = 0;
+ guard.lock_var_mut().owner = None;
+ },
+ Ok(mut guard) => {
+ // There was a thread waiting, just pass the lock
+ if let NotifiedTcs::Single(tcs) = guard.notified_tcs() {
+ guard.lock_var_mut().owner = Some(tcs)
+ } else {
+ unreachable!() // called notify_one
+ }
+ }
+ }
+ }
}
- pub unsafe fn unlock(&self) {}
+ #[inline]
+ pub unsafe fn try_lock(&self) -> bool {
+ let mut guard = try_lock_or_false!(self.inner);
+ match guard.lock_var().owner {
+ Some(tcs) if tcs != thread::current() => {
+ // Another thread has the lock
+ false
+ },
+ _ => {
+ // We are just now obtaining the lock
+ guard.lock_var_mut().owner = Some(thread::current());
+ guard.lock_var_mut().count += 1;
+ true
+ },
+ }
+ }
+ #[inline]
pub unsafe fn destroy(&self) {}
}
}
pub fn getenv(_k: &OsStr) -> io::Result<Option<OsString>> {
- unsupported()
+ Ok(None)
}
pub fn setenv(_k: &OsStr, _v: &OsStr) -> io::Result<()> {
// option. This file may not be copied, modified, or distributed
// except according to those terms.
-use cell::UnsafeCell;
+use num::NonZeroUsize;
+
+use super::waitqueue::{WaitVariable, WaitQueue, SpinMutex, NotifiedTcs, try_lock_or_false};
pub struct RWLock {
- mode: UnsafeCell<isize>,
+ readers: SpinMutex<WaitVariable<Option<NonZeroUsize>>>,
+ writer: SpinMutex<WaitVariable<bool>>,
}
-unsafe impl Send for RWLock {}
-unsafe impl Sync for RWLock {} // FIXME
+//unsafe impl Send for RWLock {}
+//unsafe impl Sync for RWLock {} // FIXME
impl RWLock {
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> RWLock {
RWLock {
- mode: UnsafeCell::new(0),
+ readers: SpinMutex::new(WaitVariable::new(None)),
+ writer: SpinMutex::new(WaitVariable::new(false))
}
}
#[inline]
pub unsafe fn read(&self) {
- let mode = self.mode.get();
- if *mode >= 0 {
- *mode += 1;
+ let mut rguard = self.readers.lock();
+ let wguard = self.writer.lock();
+ if *wguard.lock_var() || !wguard.queue_empty() {
+ // Another thread has or is waiting for the write lock, wait
+ drop(wguard);
+ WaitQueue::wait(rguard);
+ // Another thread has passed the lock to us
} else {
- rtabort!("rwlock locked for writing");
+ // No waiting writers, acquire the read lock
+ *rguard.lock_var_mut() =
+ NonZeroUsize::new(rguard.lock_var().map_or(0, |n| n.get()) + 1);
}
}
#[inline]
pub unsafe fn try_read(&self) -> bool {
- let mode = self.mode.get();
- if *mode >= 0 {
- *mode += 1;
- true
- } else {
+ let mut rguard = try_lock_or_false!(self.readers);
+ let wguard = try_lock_or_false!(self.writer);
+ if *wguard.lock_var() || !wguard.queue_empty() {
+ // Another thread has or is waiting for the write lock
false
+ } else {
+ // No waiting writers, acquire the read lock
+ *rguard.lock_var_mut() =
+ NonZeroUsize::new(rguard.lock_var().map_or(0, |n| n.get()) + 1);
+ true
}
}
#[inline]
pub unsafe fn write(&self) {
- let mode = self.mode.get();
- if *mode == 0 {
- *mode = -1;
+ let rguard = self.readers.lock();
+ let mut wguard = self.writer.lock();
+ if *wguard.lock_var() || rguard.lock_var().is_some() {
+ // Another thread has the lock, wait
+ drop(rguard);
+ WaitQueue::wait(wguard);
+ // Another thread has passed the lock to us
} else {
- rtabort!("rwlock locked for reading")
+ // We are just now obtaining the lock
+ *wguard.lock_var_mut() = true;
}
}
#[inline]
pub unsafe fn try_write(&self) -> bool {
- let mode = self.mode.get();
- if *mode == 0 {
- *mode = -1;
- true
- } else {
+ let rguard = try_lock_or_false!(self.readers);
+ let mut wguard = try_lock_or_false!(self.writer);
+ if *wguard.lock_var() || rguard.lock_var().is_some() {
+ // Another thread has the lock
false
+ } else {
+ // We are just now obtaining the lock
+ *wguard.lock_var_mut() = true;
+ true
}
}
#[inline]
pub unsafe fn read_unlock(&self) {
- *self.mode.get() -= 1;
+ let mut rguard = self.readers.lock();
+ let wguard = self.writer.lock();
+ *rguard.lock_var_mut() = NonZeroUsize::new(rguard.lock_var().unwrap().get() - 1);
+ if rguard.lock_var().is_some() {
+ // There are other active readers
+ } else {
+ if let Ok(mut wguard) = WaitQueue::notify_one(wguard) {
+ // A writer was waiting, pass the lock
+ *wguard.lock_var_mut() = true;
+ } else {
+ // No writers were waiting, the lock is released
+ assert!(rguard.queue_empty());
+ }
+ }
}
#[inline]
pub unsafe fn write_unlock(&self) {
- *self.mode.get() += 1;
+ let rguard = self.readers.lock();
+ let wguard = self.writer.lock();
+ if let Err(mut wguard) = WaitQueue::notify_one(wguard) {
+ // No writers waiting, release the write lock
+ *wguard.lock_var_mut() = false;
+ if let Ok(mut rguard) = WaitQueue::notify_all(rguard) {
+ // One or more readers were waiting, pass the lock to them
+ if let NotifiedTcs::All { count } = rguard.notified_tcs() {
+ *rguard.lock_var_mut() = Some(count)
+ } else {
+ unreachable!() // called notify_all
+ }
+ } else {
+ // No readers waiting, the lock is released
+ }
+ } else {
+ // There was a thread waiting for write, just pass the lock
+ }
}
#[inline]
- pub unsafe fn destroy(&self) {
- }
+ pub unsafe fn destroy(&self) {}
}
use boxed::FnBox;
use ffi::CStr;
use io;
-use sys::{unsupported, Void};
use time::Duration;
-pub struct Thread(Void);
+use super::abi::usercalls;
+
+pub struct Thread(task_queue::JoinHandle);
pub const DEFAULT_MIN_STACK_SIZE: usize = 4096;
+mod task_queue {
+ use sync::{Mutex, MutexGuard, Once};
+ use sync::mpsc;
+ use boxed::FnBox;
+
+ pub type JoinHandle = mpsc::Receiver<()>;
+
+ pub(super) struct Task {
+ p: Box<dyn FnBox()>,
+ done: mpsc::Sender<()>,
+ }
+
+ impl Task {
+ pub(super) fn new(p: Box<dyn FnBox()>) -> (Task, JoinHandle) {
+ let (done, recv) = mpsc::channel();
+ (Task { p, done }, recv)
+ }
+
+ pub(super) fn run(self) {
+ (self.p)();
+ let _ = self.done.send(());
+ }
+ }
+
+ static TASK_QUEUE_INIT: Once = Once::new();
+ static mut TASK_QUEUE: Option<Mutex<Vec<Task>>> = None;
+
+ pub(super) fn lock() -> MutexGuard<'static, Vec<Task>> {
+ unsafe {
+ TASK_QUEUE_INIT.call_once(|| TASK_QUEUE = Some(Default::default()) );
+ TASK_QUEUE.as_ref().unwrap().lock().unwrap()
+ }
+ }
+}
+
impl Thread {
// unsafe: see thread::Builder::spawn_unchecked for safety requirements
- pub unsafe fn new(_stack: usize, _p: Box<dyn FnBox()>)
+ pub unsafe fn new(_stack: usize, p: Box<dyn FnBox()>)
-> io::Result<Thread>
{
- unsupported()
+ let mut queue_lock = task_queue::lock();
+ usercalls::launch_thread()?;
+ let (task, handle) = task_queue::Task::new(p);
+ queue_lock.push(task);
+ Ok(Thread(handle))
+ }
+
+ pub(super) fn entry() {
+ let mut guard = task_queue::lock();
+ let task = guard.pop().expect("Thread started but no tasks pending");
+ drop(guard); // make sure to not hold the task queue lock longer than necessary
+ task.run()
}
pub fn yield_now() {
- // do nothing
+ assert_eq!(
+ usercalls::wait(0, usercalls::WAIT_NO).unwrap_err().kind(),
+ io::ErrorKind::WouldBlock
+ );
}
pub fn set_name(_name: &CStr) {
- // nope
+ // FIXME: could store this pointer in TLS somewhere
}
pub fn sleep(_dur: Duration) {
- panic!("can't sleep");
+ panic!("can't sleep"); // FIXME
}
pub fn join(self) {
- match self.0 {}
+ let _ = self.0.recv();
}
}
--- /dev/null
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+/// A simple queue implementation for synchronization primitives.
+///
+/// This queue is used to implement condition variable and mutexes.
+///
+/// Users of this API are expected to use the `WaitVariable<T>` type. Since
+/// that type is not `Sync`, it needs to be protected by e.g. a `SpinMutex` to
+/// allow shared access.
+///
+/// Since userspace may send spurious wake-ups, the wakeup event state is
+/// recorded in the enclave. The wakeup event state is protected by a spinlock.
+/// The queue and associated wait state are stored in a `WaitVariable`.
+
+use ops::{Deref, DerefMut};
+use num::NonZeroUsize;
+
+use fortanix_sgx_abi::{Tcs, EV_UNPARK, WAIT_INDEFINITE};
+use super::abi::usercalls;
+use super::abi::thread;
+
+use self::unsafe_list::{UnsafeList, UnsafeListEntry};
+pub use self::spin_mutex::{SpinMutex, SpinMutexGuard, try_lock_or_false};
+
+/// An queue entry in a `WaitQueue`.
+struct WaitEntry {
+ /// TCS address of the thread that is waiting
+ tcs: Tcs,
+ /// Whether this thread has been notified to be awoken
+ wake: bool
+}
+
+/// Data stored with a `WaitQueue` alongside it. This ensures accesses to the
+/// queue and the data are synchronized, since the type itself is not `Sync`.
+///
+/// Consumers of this API should use a synchronization primitive for shared
+/// access, such as `SpinMutex`.
+#[derive(Default)]
+pub struct WaitVariable<T> {
+ queue: WaitQueue,
+ lock: T
+}
+
+impl<T> WaitVariable<T> {
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
+ pub const fn new(var: T) -> Self {
+ WaitVariable {
+ queue: WaitQueue::new(),
+ lock: var
+ }
+ }
+
+ pub fn queue_empty(&self) -> bool {
+ self.queue.is_empty()
+ }
+
+ pub fn lock_var(&self) -> &T {
+ &self.lock
+ }
+
+ pub fn lock_var_mut(&mut self) -> &mut T {
+ &mut self.lock
+ }
+}
+
+#[derive(Copy, Clone)]
+pub enum NotifiedTcs {
+ Single(Tcs),
+ All { count: NonZeroUsize }
+}
+
+/// An RAII guard that will notify a set of target threads as well as unlock
+/// a mutex on drop.
+pub struct WaitGuard<'a, T: 'a> {
+ mutex_guard: Option<SpinMutexGuard<'a, WaitVariable<T>>>,
+ notified_tcs: NotifiedTcs
+}
+
+/// A queue of threads that are waiting on some synchronization primitive.
+///
+/// `UnsafeList` entries are allocated on the waiting thread's stack. This
+/// avoids any global locking that might happen in the heap allocator. This is
+/// safe because the waiting thread will not return from that stack frame until
+/// after it is notified. The notifying thread ensures to clean up any
+/// references to the list entries before sending the wakeup event.
+pub struct WaitQueue {
+ // We use an inner Mutex here to protect the data in the face of spurious
+ // wakeups.
+ inner: UnsafeList<SpinMutex<WaitEntry>>,
+}
+unsafe impl Send for WaitQueue {}
+
+impl Default for WaitQueue {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<'a, T> WaitGuard<'a, T> {
+ /// Returns which TCSes will be notified when this guard drops.
+ pub fn notified_tcs(&self) -> NotifiedTcs {
+ self.notified_tcs
+ }
+}
+
+impl<'a, T> Deref for WaitGuard<'a, T> {
+ type Target = SpinMutexGuard<'a, WaitVariable<T>>;
+
+ fn deref(&self) -> &Self::Target {
+ self.mutex_guard.as_ref().unwrap()
+ }
+}
+
+impl<'a, T> DerefMut for WaitGuard<'a, T> {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ self.mutex_guard.as_mut().unwrap()
+ }
+}
+
+impl<'a, T> Drop for WaitGuard<'a, T> {
+ fn drop(&mut self) {
+ drop(self.mutex_guard.take());
+ let target_tcs = match self.notified_tcs {
+ NotifiedTcs::Single(tcs) => Some(tcs),
+ NotifiedTcs::All { .. } => None
+ };
+ usercalls::send(EV_UNPARK, target_tcs).unwrap();
+ }
+}
+
+impl WaitQueue {
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
+ pub const fn new() -> Self {
+ WaitQueue {
+ inner: UnsafeList::new()
+ }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.inner.is_empty()
+ }
+
+ /// Add the calling thread to the WaitVariable's wait queue, then wait
+ /// until a wakeup event.
+ ///
+ /// This function does not return until this thread has been awoken.
+ pub fn wait<T>(mut guard: SpinMutexGuard<WaitVariable<T>>) {
+ unsafe {
+ let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry {
+ tcs: thread::current(),
+ wake: false
+ }));
+ let entry = guard.queue.inner.push(&mut entry);
+ drop(guard);
+ while !entry.lock().wake {
+ assert_eq!(
+ usercalls::wait(EV_UNPARK, WAIT_INDEFINITE).unwrap() & EV_UNPARK,
+ EV_UNPARK
+ );
+ }
+ }
+ }
+
+ /// Either find the next waiter on the wait queue, or return the mutex
+ /// guard unchanged.
+ ///
+ /// If a waiter is found, a `WaitGuard` is returned which will notify the
+ /// waiter when it is dropped.
+ pub fn notify_one<T>(mut guard: SpinMutexGuard<WaitVariable<T>>)
+ -> Result<WaitGuard<T>, SpinMutexGuard<WaitVariable<T>>>
+ {
+ unsafe {
+ if let Some(entry) = guard.queue.inner.pop() {
+ let mut entry_guard = entry.lock();
+ let tcs = entry_guard.tcs;
+ entry_guard.wake = true;
+ drop(entry);
+ Ok(WaitGuard {
+ mutex_guard: Some(guard),
+ notified_tcs: NotifiedTcs::Single(tcs)
+ })
+ } else {
+ Err(guard)
+ }
+ }
+ }
+
+ /// Either find any and all waiters on the wait queue, or return the mutex
+ /// guard unchanged.
+ ///
+ /// If at least one waiter is found, a `WaitGuard` is returned which will
+ /// notify all waiters when it is dropped.
+ pub fn notify_all<T>(mut guard: SpinMutexGuard<WaitVariable<T>>)
+ -> Result<WaitGuard<T>, SpinMutexGuard<WaitVariable<T>>>
+ {
+ unsafe {
+ let mut count = 0;
+ while let Some(entry) = guard.queue.inner.pop() {
+ count += 1;
+ let mut entry_guard = entry.lock();
+ entry_guard.wake = true;
+ }
+ if let Some(count) = NonZeroUsize::new(count) {
+ Ok(WaitGuard {
+ mutex_guard: Some(guard),
+ notified_tcs: NotifiedTcs::All { count }
+ })
+ } else {
+ Err(guard)
+ }
+ }
+ }
+}
+
+/// A doubly-linked list where callers are in charge of memory allocation
+/// of the nodes in the list.
+mod unsafe_list {
+ use ptr::NonNull;
+ use mem;
+
+ pub struct UnsafeListEntry<T> {
+ next: NonNull<UnsafeListEntry<T>>,
+ prev: NonNull<UnsafeListEntry<T>>,
+ value: Option<T>
+ }
+
+ impl<T> UnsafeListEntry<T> {
+ fn dummy() -> Self {
+ UnsafeListEntry {
+ next: NonNull::dangling(),
+ prev: NonNull::dangling(),
+ value: None
+ }
+ }
+
+ pub fn new(value: T) -> Self {
+ UnsafeListEntry {
+ value: Some(value),
+ ..Self::dummy()
+ }
+ }
+ }
+
+ pub struct UnsafeList<T> {
+ head_tail: NonNull<UnsafeListEntry<T>>,
+ head_tail_entry: Option<UnsafeListEntry<T>>,
+ }
+
+ impl<T> UnsafeList<T> {
+ #[unstable(feature = "sgx_internals", issue = "0")] // FIXME: min_const_fn
+ pub const fn new() -> Self {
+ unsafe {
+ UnsafeList {
+ head_tail: NonNull::new_unchecked(1 as _),
+ head_tail_entry: None
+ }
+ }
+ }
+
+ unsafe fn init(&mut self) {
+ if self.head_tail_entry.is_none() {
+ self.head_tail_entry = Some(UnsafeListEntry::dummy());
+ self.head_tail = NonNull::new_unchecked(self.head_tail_entry.as_mut().unwrap());
+ self.head_tail.as_mut().next = self.head_tail;
+ self.head_tail.as_mut().prev = self.head_tail;
+ }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ unsafe {
+ if self.head_tail_entry.is_some() {
+ let first = self.head_tail.as_ref().next;
+ if first == self.head_tail {
+ // ,-------> /---------\ next ---,
+ // | |head_tail| |
+ // `--- prev \---------/ <-------`
+ assert_eq!(self.head_tail.as_ref().prev, first);
+ true
+ } else {
+ false
+ }
+ } else {
+ true
+ }
+ }
+ }
+
+ /// Pushes an entry onto the back of the list.
+ ///
+ /// # Safety
+ ///
+ /// The entry must remain allocated until the entry is removed from the
+ /// list AND the caller who popped is done using the entry.
+ pub unsafe fn push<'a>(&mut self, entry: &'a mut UnsafeListEntry<T>) -> &'a T {
+ self.init();
+
+ // BEFORE:
+ // /---------\ next ---> /---------\
+ // ... |prev_tail| |head_tail| ...
+ // \---------/ <--- prev \---------/
+ //
+ // AFTER:
+ // /---------\ next ---> /-----\ next ---> /---------\
+ // ... |prev_tail| |entry| |head_tail| ...
+ // \---------/ <--- prev \-----/ <--- prev \---------/
+ let mut entry = NonNull::new_unchecked(entry);
+ let mut prev_tail = mem::replace(&mut self.head_tail.as_mut().prev, entry);
+ entry.as_mut().prev = prev_tail;
+ entry.as_mut().next = self.head_tail;
+ prev_tail.as_mut().next = entry;
+ (*entry.as_ptr()).value.as_ref().unwrap()
+ }
+
+ /// Pops an entry from the front of the list.
+ ///
+ /// # Safety
+ ///
+ /// The caller must make sure to synchronize ending the borrow of the
+ /// return value and deallocation of the containing entry.
+ pub unsafe fn pop<'a>(&mut self) -> Option<&'a T> {
+ self.init();
+
+ if self.is_empty() {
+ None
+ } else {
+ // BEFORE:
+ // /---------\ next ---> /-----\ next ---> /------\
+ // ... |head_tail| |first| |second| ...
+ // \---------/ <--- prev \-----/ <--- prev \------/
+ //
+ // AFTER:
+ // /---------\ next ---> /------\
+ // ... |head_tail| |second| ...
+ // \---------/ <--- prev \------/
+ let mut first = self.head_tail.as_mut().next;
+ let mut second = first.as_mut().next;
+ self.head_tail.as_mut().next = second;
+ second.as_mut().prev = self.head_tail;
+ first.as_mut().next = NonNull::dangling();
+ first.as_mut().prev = NonNull::dangling();
+ Some((*first.as_ptr()).value.as_ref().unwrap())
+ }
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ use super::*;
+ use cell::Cell;
+
+ unsafe fn assert_empty<T>(list: &mut UnsafeList<T>) {
+ assert!(list.pop().is_none(), "assertion failed: list is not empty");
+ }
+
+ #[test]
+ fn init_empty() {
+ unsafe {
+ assert_empty(&mut UnsafeList::<i32>::new());
+ }
+ }
+
+ #[test]
+ fn push_pop() {
+ unsafe {
+ let mut node = UnsafeListEntry::new(1234);
+ let mut list = UnsafeList::new();
+ assert_eq!(list.push(&mut node), &1234);
+ assert_eq!(list.pop().unwrap(), &1234);
+ assert_empty(&mut list);
+ }
+ }
+
+ #[test]
+ fn complex_pushes_pops() {
+ unsafe {
+ let mut node1 = UnsafeListEntry::new(1234);
+ let mut node2 = UnsafeListEntry::new(4567);
+ let mut node3 = UnsafeListEntry::new(9999);
+ let mut node4 = UnsafeListEntry::new(8642);
+ let mut list = UnsafeList::new();
+ list.push(&mut node1);
+ list.push(&mut node2);
+ assert_eq!(list.pop().unwrap(), &1234);
+ list.push(&mut node3);
+ assert_eq!(list.pop().unwrap(), &4567);
+ assert_eq!(list.pop().unwrap(), &9999);
+ assert_empty(&mut list);
+ list.push(&mut node4);
+ assert_eq!(list.pop().unwrap(), &8642);
+ assert_empty(&mut list);
+ }
+ }
+
+ #[test]
+ fn cell() {
+ unsafe {
+ let mut node = UnsafeListEntry::new(Cell::new(0));
+ let mut list = UnsafeList::new();
+ let noderef = list.push(&mut node);
+ assert_eq!(noderef.get(), 0);
+ list.pop().unwrap().set(1);
+ assert_empty(&mut list);
+ assert_eq!(noderef.get(), 1);
+ }
+ }
+ }
+}
+
+/// Trivial spinlock-based implementation of `sync::Mutex`.
+// FIXME: Perhaps use Intel TSX to avoid locking?
+mod spin_mutex {
+ use cell::UnsafeCell;
+ use sync::atomic::{AtomicBool, Ordering, spin_loop_hint};
+ use ops::{Deref, DerefMut};
+
+ #[derive(Default)]
+ pub struct SpinMutex<T> {
+ value: UnsafeCell<T>,
+ lock: AtomicBool,
+ }
+
+ unsafe impl<T: Send> Send for SpinMutex<T> {}
+ unsafe impl<T: Send> Sync for SpinMutex<T> {}
+
+ pub struct SpinMutexGuard<'a, T: 'a> {
+ mutex: &'a SpinMutex<T>,
+ }
+
+ impl<'a, T> !Send for SpinMutexGuard<'a, T> {}
+ unsafe impl<'a, T: Sync> Sync for SpinMutexGuard<'a, T> {}
+
+ impl<T> SpinMutex<T> {
+ pub const fn new(value: T) -> Self {
+ SpinMutex {
+ value: UnsafeCell::new(value),
+ lock: AtomicBool::new(false)
+ }
+ }
+
+ #[inline(always)]
+ pub fn lock(&self) -> SpinMutexGuard<T> {
+ loop {
+ match self.try_lock() {
+ None => while self.lock.load(Ordering::Relaxed) {
+ spin_loop_hint()
+ },
+ Some(guard) => return guard
+ }
+ }
+ }
+
+ #[inline(always)]
+ pub fn try_lock(&self) -> Option<SpinMutexGuard<T>> {
+ if !self.lock.compare_and_swap(false, true, Ordering::Acquire) {
+ Some(SpinMutexGuard {
+ mutex: self,
+ })
+ } else {
+ None
+ }
+ }
+ }
+
+ pub macro try_lock_or_false {
+ ($e:expr) => {
+ if let Some(v) = $e.try_lock() {
+ v
+ } else {
+ return false
+ }
+ }
+ }
+
+ impl<'a, T> Deref for SpinMutexGuard<'a, T> {
+ type Target = T;
+
+ fn deref(&self) -> &T {
+ unsafe {
+ &*self.mutex.value.get()
+ }
+ }
+ }
+
+ impl<'a, T> DerefMut for SpinMutexGuard<'a, T> {
+ fn deref_mut(&mut self) -> &mut T {
+ unsafe {
+ &mut*self.mutex.value.get()
+ }
+ }
+ }
+
+ impl<'a, T> Drop for SpinMutexGuard<'a, T> {
+ fn drop(&mut self) {
+ self.mutex.lock.store(false, Ordering::Release)
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ #![allow(deprecated)]
+
+ use super::*;
+ use sync::Arc;
+ use thread;
+
+ #[test]
+ fn sleep() {
+ let mutex = Arc::new(SpinMutex::<i32>::default());
+ let mutex2 = mutex.clone();
+ let guard = mutex.lock();
+ let t1 = thread::spawn(move || {
+ *mutex2.lock() = 1;
+ });
+ thread::sleep_ms(50);
+ assert_eq!(*guard, 0);
+ drop(guard);
+ t1.join().unwrap();
+ assert_eq!(*mutex.lock(), 1);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use sync::Arc;
+ use thread;
+
+ #[test]
+ fn queue() {
+ let wq = Arc::new(SpinMutex::<WaitVariable<()>>::default());
+ let wq2 = wq.clone();
+
+ let locked = wq.lock();
+
+ let t1 = thread::spawn(move || {
+ assert!(WaitQueue::notify_one(wq2.lock()).is_none())
+ });
+
+ WaitQueue::wait(locked);
+
+ t1.join().unwrap();
+ }
+}
///
/// Behavior is undefined if the condition variable is moved after it is
/// first used with any of the functions below.
+ #[unstable(feature = "sys_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> Condvar { Condvar(imp::Condvar::new()) }
/// Prepares the condition variable for use.
/// Also, until `init` is called, behavior is undefined if this
/// mutex is ever used reentrantly, i.e., `raw_lock` or `try_lock`
/// are called by the thread currently holding the lock.
+ #[unstable(feature = "sys_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> Mutex { Mutex(imp::Mutex::new()) }
/// Prepare the mutex for use.
///
/// Behavior is undefined if the reader-writer lock is moved after it is
/// first used with any of the functions below.
+ #[unstable(feature = "sys_internals", issue = "0")] // FIXME: min_const_fn
pub const fn new() -> RWLock { RWLock(imp::RWLock::new()) }
/// Acquires shared access to the underlying lock, blocking the current
projects / rust.git / commitdiff