1 //! Implement thread-local storage.
3 use std::collections::BTreeMap;
4 use std::collections::btree_map::Entry as BTreeEntry;
5 use std::collections::hash_map::Entry as HashMapEntry;
9 use rustc_data_structures::fx::FxHashMap;
11 use rustc_target::abi::{Size, HasDataLayout};
14 HelpersEvalContextExt, InterpResult, MPlaceTy, Scalar, StackPopCleanup, Tag, ThreadId,
15 ThreadsEvalContextExt,
18 pub type TlsKey = u128;
20 #[derive(Clone, Debug)]
21 pub struct TlsEntry<'tcx> {
22 /// The data for this key. None is used to represent NULL.
23 /// (We normalize this early to avoid having to do a NULL-ptr-test each time we access the data.)
24 data: BTreeMap<ThreadId, Scalar<Tag>>,
25 dtor: Option<ty::Instance<'tcx>>,
28 #[derive(Clone, Debug)]
29 struct RunningDtorsState {
30 /// The last TlsKey used to retrieve a TLS destructor. `None` means that we
31 /// have not tried to retrieve a TLS destructor yet or that we already tried
33 last_dtor_key: Option<TlsKey>,
37 pub struct TlsData<'tcx> {
38 /// The Key to use for the next thread-local allocation.
41 /// pthreads-style thread-local storage.
42 keys: BTreeMap<TlsKey, TlsEntry<'tcx>>,
44 /// A single per thread destructor of the thread local storage (that's how
45 /// things work on macOS) with a data argument.
46 macos_thread_dtors: BTreeMap<ThreadId, (ty::Instance<'tcx>, Scalar<Tag>)>,
48 /// State for currently running TLS dtors. If this map contains a key for a
49 /// specific thread, it means that we are in the "destruct" phase, during
50 /// which some operations are UB.
51 dtors_running: FxHashMap<ThreadId, RunningDtorsState>,
54 impl<'tcx> Default for TlsData<'tcx> {
55 fn default() -> Self {
57 next_key: 1, // start with 1 as we must not use 0 on Windows
58 keys: Default::default(),
59 macos_thread_dtors: Default::default(),
60 dtors_running: Default::default(),
65 impl<'tcx> TlsData<'tcx> {
66 /// Generate a new TLS key with the given destructor.
67 /// `max_size` determines the integer size the key has to fit in.
68 pub fn create_tls_key(&mut self, dtor: Option<ty::Instance<'tcx>>, max_size: Size) -> InterpResult<'tcx, TlsKey> {
69 let new_key = self.next_key;
71 self.keys.insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap_none();
72 trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor);
74 if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits() as u128) {
75 throw_unsup_format!("we ran out of TLS key space");
80 pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> {
81 match self.keys.remove(&key) {
83 trace!("TLS key {} removed", key);
86 None => throw_ub_format!("removing a non-existig TLS key: {}", key),
94 cx: &impl HasDataLayout,
95 ) -> InterpResult<'tcx, Scalar<Tag>> {
96 match self.keys.get(&key) {
97 Some(TlsEntry { data, .. }) => {
98 let value = data.get(&thread_id).copied();
99 trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value);
100 Ok(value.unwrap_or_else(|| Scalar::null_ptr(cx).into()))
102 None => throw_ub_format!("loading from a non-existing TLS key: {}", key),
110 new_data: Option<Scalar<Tag>>
111 ) -> InterpResult<'tcx> {
112 match self.keys.get_mut(&key) {
113 Some(TlsEntry { data, .. }) => {
116 trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, scalar);
117 data.insert(thread_id, scalar);
120 trace!("TLS key {} for thread {:?} removed", key, thread_id);
121 data.remove(&thread_id);
126 None => throw_ub_format!("storing to a non-existing TLS key: {}", key),
130 /// Set the thread wide destructor of the thread local storage for the given
131 /// thread. This function is used to implement `_tlv_atexit` shim on MacOS.
133 /// Thread wide dtors are available only on MacOS. There is one destructor
134 /// per thread as can be guessed from the following comment in the
136 /// implementation](https://github.com/opensource-apple/dyld/blob/195030646877261f0c8c7ad8b001f52d6a26f514/src/threadLocalVariables.c#L389):
138 /// // NOTE: this does not need locks because it only operates on current thread data
139 pub fn set_macos_thread_dtor(
142 dtor: ty::Instance<'tcx>,
144 ) -> InterpResult<'tcx> {
145 if self.dtors_running.contains_key(&thread) {
146 // UB, according to libstd docs.
147 throw_ub_format!("setting thread's local storage destructor while destructors are already running");
149 if self.macos_thread_dtors.insert(thread, (dtor, data)).is_some() {
150 throw_unsup_format!("setting more than one thread local storage destructor for the same thread is not supported");
155 /// Returns a dtor, its argument and its index, if one is supposed to run.
156 /// `key` is the last dtors that was run; we return the *next* one after that.
158 /// An optional destructor function may be associated with each key value.
159 /// At thread exit, if a key value has a non-NULL destructor pointer,
160 /// and the thread has a non-NULL value associated with that key,
161 /// the value of the key is set to NULL, and then the function pointed
162 /// to is called with the previously associated value as its sole argument.
163 /// The order of destructor calls is unspecified if more than one destructor
164 /// exists for a thread when it exits.
166 /// If, after all the destructors have been called for all non-NULL values
167 /// with associated destructors, there are still some non-NULL values with
168 /// associated destructors, then the process is repeated.
169 /// If, after at least {PTHREAD_DESTRUCTOR_ITERATIONS} iterations of destructor
170 /// calls for outstanding non-NULL values, there are still some non-NULL values
171 /// with associated destructors, implementations may stop calling destructors,
172 /// or they may continue calling destructors until no non-NULL values with
173 /// associated destructors exist, even though this might result in an infinite loop.
178 ) -> Option<(ty::Instance<'tcx>, Scalar<Tag>, TlsKey)> {
179 use std::collections::Bound::*;
181 let thread_local = &mut self.keys;
182 let start = match key {
183 Some(key) => Excluded(key),
186 for (&key, TlsEntry { data, dtor }) in
187 thread_local.range_mut((start, Unbounded))
189 match data.entry(thread_id) {
190 BTreeEntry::Occupied(entry) => {
191 if let Some(dtor) = dtor {
192 // Set TLS data to NULL, and call dtor with old value.
193 let data_scalar = entry.remove();
194 let ret = Some((*dtor, data_scalar, key));
198 BTreeEntry::Vacant(_) => {}
204 /// Set that dtors are running for `thread`. It is guaranteed not to change
205 /// the existing values stored in `dtors_running` for this thread. Returns
206 /// `true` if dtors for `thread` are already running.
207 fn set_dtors_running_for_thread(&mut self, thread: ThreadId) -> bool {
208 match self.dtors_running.entry(thread) {
209 HashMapEntry::Occupied(_) => true,
210 HashMapEntry::Vacant(entry) => {
211 // We cannot just do `self.dtors_running.insert` because that
212 // would overwrite `last_dtor_key` with `None`.
213 entry.insert(RunningDtorsState { last_dtor_key: None });
219 /// Delete all TLS entries for the given thread. This function should be
220 /// called after all TLS destructors have already finished.
221 fn delete_all_thread_tls(&mut self, thread_id: ThreadId) {
222 for TlsEntry { data, .. } in self.keys.values_mut() {
223 data.remove(&thread_id);
228 impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
229 trait EvalContextPrivExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
230 /// Schedule TLS destructors for the main thread on Windows. The
231 /// implementation assumes that we do not support concurrency on Windows
233 fn schedule_windows_tls_dtors(&mut self) -> InterpResult<'tcx> {
234 let this = self.eval_context_mut();
235 let active_thread = this.get_active_thread();
236 assert_eq!(this.get_total_thread_count(), 1, "concurrency on Windows not supported");
237 // Windows has a special magic linker section that is run on certain events.
238 // Instead of searching for that section and supporting arbitrary hooks in there
239 // (that would be basically https://github.com/rust-lang/miri/issues/450),
240 // we specifically look up the static in libstd that we know is placed
242 let thread_callback = this.eval_path_scalar(&["std", "sys", "windows", "thread_local", "p_thread_callback"])?;
243 let thread_callback = this.memory.get_fn(thread_callback.not_undef()?)?.as_instance()?;
245 // The signature of this function is `unsafe extern "system" fn(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID)`.
246 let reason = this.eval_path_scalar(&["std", "sys", "windows", "c", "DLL_THREAD_DETACH"])?;
247 let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
250 &[Scalar::null_ptr(this).into(), reason.into(), Scalar::null_ptr(this).into()],
252 StackPopCleanup::None { cleanup: true },
255 this.enable_thread(active_thread);
259 /// Schedule the MacOS thread destructor of the thread local storage to be
260 /// executed. Returns `true` if scheduled.
262 /// Note: It is safe to call this function also on other Unixes.
263 fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx, bool> {
264 let this = self.eval_context_mut();
265 let thread_id = this.get_active_thread();
266 if let Some((instance, data)) = this.machine.tls.macos_thread_dtors.remove(&thread_id) {
267 trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id);
269 let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
274 StackPopCleanup::None { cleanup: true },
277 // Enable the thread so that it steps through the destructor which
278 // we just scheduled. Since we deleted the destructor, it is
279 // guaranteed that we will schedule it again. The `dtors_running`
280 // flag will prevent the code from adding the destructor again.
281 this.enable_thread(thread_id);
288 /// Schedule a pthread TLS destructor. Returns `true` if found
289 /// a destructor to schedule, and `false` otherwise.
290 fn schedule_next_pthread_tls_dtor(&mut self) -> InterpResult<'tcx, bool> {
291 let this = self.eval_context_mut();
292 let active_thread = this.get_active_thread();
294 assert!(this.has_terminated(active_thread), "running TLS dtors for non-terminated thread");
295 // Fetch next dtor after `key`.
296 let last_key = this.machine.tls.dtors_running[&active_thread].last_dtor_key.clone();
297 let dtor = match this.machine.tls.fetch_tls_dtor(last_key, active_thread) {
298 dtor @ Some(_) => dtor,
299 // We ran each dtor once, start over from the beginning.
301 this.machine.tls.fetch_tls_dtor(None, active_thread)
304 if let Some((instance, ptr, key)) = dtor {
305 this.machine.tls.dtors_running.get_mut(&active_thread).unwrap().last_dtor_key = Some(key);
306 trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread);
307 assert!(!this.is_null(ptr).unwrap(), "data can't be NULL when dtor is called!");
309 let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
314 StackPopCleanup::None { cleanup: true },
317 this.enable_thread(active_thread);
320 this.machine.tls.dtors_running.get_mut(&active_thread).unwrap().last_dtor_key = None;
326 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
327 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
329 /// Schedule an active thread's TLS destructor to run on the active thread.
330 /// Note that this function does not run the destructors itself, it just
331 /// schedules them one by one each time it is called and reenables the
332 /// thread so that it can be executed normally by the main execution loop.
334 /// FIXME: we do not support yet deallocation of thread local statics.
335 /// Issue: https://github.com/rust-lang/miri/issues/1369
337 /// Note: we consistently run TLS destructors for all threads, including the
338 /// main thread. However, it is not clear that we should run the TLS
339 /// destructors for the main thread. See issue:
340 /// https://github.com/rust-lang/rust/issues/28129.
341 fn schedule_next_tls_dtor_for_active_thread(&mut self) -> InterpResult<'tcx> {
342 let this = self.eval_context_mut();
343 let active_thread = this.get_active_thread();
345 if !this.machine.tls.set_dtors_running_for_thread(active_thread) {
346 // This is the first time we got asked to schedule a destructor. The
347 // Windows schedule destructor function must be called exactly once,
348 // this is why it is in this block.
349 if this.tcx.sess.target.target.target_os == "windows" {
350 // On Windows, we signal that the thread quit by starting the
351 // relevant function, reenabling the thread, and going back to
353 this.schedule_windows_tls_dtors()?;
357 // The macOS thread wide destructor runs "before any TLS slots get
358 // freed", so do that first.
359 if this.schedule_macos_tls_dtor()? {
360 // We have scheduled a MacOS dtor to run on the thread. Execute it
361 // to completion and come back here. Scheduling a destructor
362 // destroys it, so we will not enter this branch again.
365 if this.schedule_next_pthread_tls_dtor()? {
366 // We have scheduled a pthread destructor and removed it from the
367 // destructors list. Run it to completion and come back here.
372 this.machine.tls.delete_all_thread_tls(active_thread);