1 //! Implement thread-local storage.
3 use std::collections::btree_map::Entry as BTreeEntry;
4 use std::collections::hash_map::Entry as HashMapEntry;
5 use std::collections::BTreeMap;
9 use rustc_data_structures::fx::FxHashMap;
11 use rustc_target::abi::{HasDataLayout, Size};
12 use rustc_target::spec::abi::Abi;
16 pub type TlsKey = u128;
18 #[derive(Clone, Debug)]
19 pub struct TlsEntry<'tcx> {
20 /// The data for this key. None is used to represent NULL.
21 /// (We normalize this early to avoid having to do a NULL-ptr-test each time we access the data.)
22 data: BTreeMap<ThreadId, Scalar<Provenance>>,
23 dtor: Option<ty::Instance<'tcx>>,
26 #[derive(Clone, Debug)]
27 struct RunningDtorsState {
28 /// The last TlsKey used to retrieve a TLS destructor. `None` means that we
29 /// have not tried to retrieve a TLS destructor yet or that we already tried
31 last_dtor_key: Option<TlsKey>,
35 pub struct TlsData<'tcx> {
36 /// The Key to use for the next thread-local allocation.
39 /// pthreads-style thread-local storage.
40 keys: BTreeMap<TlsKey, TlsEntry<'tcx>>,
42 /// A single per thread destructor of the thread local storage (that's how
43 /// things work on macOS) with a data argument.
44 macos_thread_dtors: BTreeMap<ThreadId, (ty::Instance<'tcx>, Scalar<Provenance>)>,
46 /// State for currently running TLS dtors. If this map contains a key for a
47 /// specific thread, it means that we are in the "destruct" phase, during
48 /// which some operations are UB.
49 dtors_running: FxHashMap<ThreadId, RunningDtorsState>,
52 impl<'tcx> Default for TlsData<'tcx> {
53 fn default() -> Self {
55 next_key: 1, // start with 1 as we must not use 0 on Windows
56 keys: Default::default(),
57 macos_thread_dtors: Default::default(),
58 dtors_running: Default::default(),
63 impl<'tcx> TlsData<'tcx> {
64 /// Generate a new TLS key with the given destructor.
65 /// `max_size` determines the integer size the key has to fit in.
66 #[allow(clippy::integer_arithmetic)]
67 pub fn create_tls_key(
69 dtor: Option<ty::Instance<'tcx>>,
71 ) -> InterpResult<'tcx, TlsKey> {
72 let new_key = self.next_key;
74 self.keys.try_insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap();
75 trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor);
77 if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits()) {
78 throw_unsup_format!("we ran out of TLS key space");
83 pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> {
84 match self.keys.remove(&key) {
86 trace!("TLS key {} removed", key);
89 None => throw_ub_format!("removing a non-existig TLS key: {}", key),
97 cx: &impl HasDataLayout,
98 ) -> InterpResult<'tcx, Scalar<Provenance>> {
99 match self.keys.get(&key) {
100 Some(TlsEntry { data, .. }) => {
101 let value = data.get(&thread_id).copied();
102 trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value);
103 Ok(value.unwrap_or_else(|| Scalar::null_ptr(cx)))
105 None => throw_ub_format!("loading from a non-existing TLS key: {}", key),
113 new_data: Scalar<Provenance>,
114 cx: &impl HasDataLayout,
115 ) -> InterpResult<'tcx> {
116 match self.keys.get_mut(&key) {
117 Some(TlsEntry { data, .. }) => {
118 if new_data.to_machine_usize(cx)? != 0 {
119 trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, new_data);
120 data.insert(thread_id, new_data);
122 trace!("TLS key {} for thread {:?} removed", key, thread_id);
123 data.remove(&thread_id);
127 None => throw_ub_format!("storing to a non-existing TLS key: {}", key),
131 /// Set the thread wide destructor of the thread local storage for the given
132 /// thread. This function is used to implement `_tlv_atexit` shim on MacOS.
134 /// Thread wide dtors are available only on MacOS. There is one destructor
135 /// per thread as can be guessed from the following comment in the
137 /// implementation](https://github.com/opensource-apple/dyld/blob/195030646877261f0c8c7ad8b001f52d6a26f514/src/threadLocalVariables.c#L389):
139 /// NOTE: this does not need locks because it only operates on current thread data
140 pub fn set_macos_thread_dtor(
143 dtor: ty::Instance<'tcx>,
144 data: Scalar<Provenance>,
145 ) -> InterpResult<'tcx> {
146 if self.dtors_running.contains_key(&thread) {
147 // UB, according to libstd docs.
149 "setting thread's local storage destructor while destructors are already running"
152 if self.macos_thread_dtors.insert(thread, (dtor, data)).is_some() {
154 "setting more than one thread local storage destructor for the same thread is not supported"
160 /// Returns a dtor, its argument and its index, if one is supposed to run.
161 /// `key` is the last dtors that was run; we return the *next* one after that.
163 /// An optional destructor function may be associated with each key value.
164 /// At thread exit, if a key value has a non-NULL destructor pointer,
165 /// and the thread has a non-NULL value associated with that key,
166 /// the value of the key is set to NULL, and then the function pointed
167 /// to is called with the previously associated value as its sole argument.
168 /// **The order of destructor calls is unspecified if more than one destructor
169 /// exists for a thread when it exits.**
171 /// If, after all the destructors have been called for all non-NULL values
172 /// with associated destructors, there are still some non-NULL values with
173 /// associated destructors, then the process is repeated.
174 /// If, after at least {PTHREAD_DESTRUCTOR_ITERATIONS} iterations of destructor
175 /// calls for outstanding non-NULL values, there are still some non-NULL values
176 /// with associated destructors, implementations may stop calling destructors,
177 /// or they may continue calling destructors until no non-NULL values with
178 /// associated destructors exist, even though this might result in an infinite loop.
183 ) -> Option<(ty::Instance<'tcx>, Scalar<Provenance>, TlsKey)> {
184 use std::ops::Bound::*;
186 let thread_local = &mut self.keys;
187 let start = match key {
188 Some(key) => Excluded(key),
191 // We interpret the documentaion above (taken from POSIX) as saying that we need to iterate
192 // over all keys and run each destructor at least once before running any destructor a 2nd
193 // time. That's why we have `key` to indicate how far we got in the current iteration. If we
194 // return `None`, `schedule_next_pthread_tls_dtor` will re-try with `ket` set to `None` to
195 // start the next round.
196 // TODO: In the future, we might consider randomizing destructor order, but we still have to
197 // uphold this requirement.
198 for (&key, TlsEntry { data, dtor }) in thread_local.range_mut((start, Unbounded)) {
199 match data.entry(thread_id) {
200 BTreeEntry::Occupied(entry) => {
201 if let Some(dtor) = dtor {
202 // Set TLS data to NULL, and call dtor with old value.
203 let data_scalar = entry.remove();
204 let ret = Some((*dtor, data_scalar, key));
208 BTreeEntry::Vacant(_) => {}
214 /// Set that dtors are running for `thread`. It is guaranteed not to change
215 /// the existing values stored in `dtors_running` for this thread. Returns
216 /// `true` if dtors for `thread` are already running.
217 fn set_dtors_running_for_thread(&mut self, thread: ThreadId) -> bool {
218 match self.dtors_running.entry(thread) {
219 HashMapEntry::Occupied(_) => true,
220 HashMapEntry::Vacant(entry) => {
221 // We cannot just do `self.dtors_running.insert` because that
222 // would overwrite `last_dtor_key` with `None`.
223 entry.insert(RunningDtorsState { last_dtor_key: None });
229 /// Delete all TLS entries for the given thread. This function should be
230 /// called after all TLS destructors have already finished.
231 fn delete_all_thread_tls(&mut self, thread_id: ThreadId) {
232 for TlsEntry { data, .. } in self.keys.values_mut() {
233 data.remove(&thread_id);
238 impl VisitTags for TlsData<'_> {
239 fn visit_tags(&self, visit: &mut dyn FnMut(SbTag)) {
240 let TlsData { keys, macos_thread_dtors, next_key: _, dtors_running: _ } = self;
242 for scalar in keys.values().flat_map(|v| v.data.values()) {
243 scalar.visit_tags(visit);
245 for (_, scalar) in macos_thread_dtors.values() {
246 scalar.visit_tags(visit);
251 impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {}
252 trait EvalContextPrivExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
253 /// Schedule TLS destructors for Windows.
254 /// On windows, TLS destructors are managed by std.
255 fn schedule_windows_tls_dtors(&mut self) -> InterpResult<'tcx> {
256 let this = self.eval_context_mut();
257 let active_thread = this.get_active_thread();
259 // Windows has a special magic linker section that is run on certain events.
260 // Instead of searching for that section and supporting arbitrary hooks in there
261 // (that would be basically https://github.com/rust-lang/miri/issues/450),
262 // we specifically look up the static in libstd that we know is placed
264 let thread_callback =
265 this.eval_windows("thread_local_key", "p_thread_callback")?.to_pointer(this)?;
266 let thread_callback = this.get_ptr_fn(thread_callback)?.as_instance()?;
268 // FIXME: Technically, the reason should be `DLL_PROCESS_DETACH` when the main thread exits
269 // but std treats both the same.
270 let reason = this.eval_windows("c", "DLL_THREAD_DETACH")?;
272 // The signature of this function is `unsafe extern "system" fn(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID)`.
273 // FIXME: `h` should be a handle to the current module and what `pv` should be is unknown
274 // but both are ignored by std
277 Abi::System { unwind: false },
278 &[Scalar::null_ptr(this).into(), reason.into(), Scalar::null_ptr(this).into()],
280 StackPopCleanup::Root { cleanup: true },
283 this.enable_thread(active_thread);
287 /// Schedule the MacOS thread destructor of the thread local storage to be
288 /// executed. Returns `true` if scheduled.
290 /// Note: It is safe to call this function also on other Unixes.
291 fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx, bool> {
292 let this = self.eval_context_mut();
293 let thread_id = this.get_active_thread();
294 if let Some((instance, data)) = this.machine.tls.macos_thread_dtors.remove(&thread_id) {
295 trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id);
299 Abi::C { unwind: false },
302 StackPopCleanup::Root { cleanup: true },
305 // Enable the thread so that it steps through the destructor which
306 // we just scheduled. Since we deleted the destructor, it is
307 // guaranteed that we will schedule it again. The `dtors_running`
308 // flag will prevent the code from adding the destructor again.
309 this.enable_thread(thread_id);
316 /// Schedule a pthread TLS destructor. Returns `true` if found
317 /// a destructor to schedule, and `false` otherwise.
318 fn schedule_next_pthread_tls_dtor(&mut self) -> InterpResult<'tcx, bool> {
319 let this = self.eval_context_mut();
320 let active_thread = this.get_active_thread();
322 assert!(this.has_terminated(active_thread), "running TLS dtors for non-terminated thread");
323 // Fetch next dtor after `key`.
324 let last_key = this.machine.tls.dtors_running[&active_thread].last_dtor_key;
325 let dtor = match this.machine.tls.fetch_tls_dtor(last_key, active_thread) {
326 dtor @ Some(_) => dtor,
327 // We ran each dtor once, start over from the beginning.
328 None => this.machine.tls.fetch_tls_dtor(None, active_thread),
330 if let Some((instance, ptr, key)) = dtor {
331 this.machine.tls.dtors_running.get_mut(&active_thread).unwrap().last_dtor_key =
333 trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread);
335 !ptr.to_machine_usize(this).unwrap() != 0,
336 "data can't be NULL when dtor is called!"
341 Abi::C { unwind: false },
344 StackPopCleanup::Root { cleanup: true },
347 this.enable_thread(active_thread);
350 this.machine.tls.dtors_running.get_mut(&active_thread).unwrap().last_dtor_key = None;
356 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {}
357 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
358 /// Schedule an active thread's TLS destructor to run on the active thread.
359 /// Note that this function does not run the destructors itself, it just
360 /// schedules them one by one each time it is called and reenables the
361 /// thread so that it can be executed normally by the main execution loop.
363 /// Note: we consistently run TLS destructors for all threads, including the
364 /// main thread. However, it is not clear that we should run the TLS
365 /// destructors for the main thread. See issue:
366 /// <https://github.com/rust-lang/rust/issues/28129>.
367 fn schedule_next_tls_dtor_for_active_thread(&mut self) -> InterpResult<'tcx> {
368 let this = self.eval_context_mut();
369 let active_thread = this.get_active_thread();
370 trace!("schedule_next_tls_dtor_for_active_thread on thread {:?}", active_thread);
372 if !this.machine.tls.set_dtors_running_for_thread(active_thread) {
373 // This is the first time we got asked to schedule a destructor. The
374 // Windows schedule destructor function must be called exactly once,
375 // this is why it is in this block.
376 if this.tcx.sess.target.os == "windows" {
377 // On Windows, we signal that the thread quit by starting the
378 // relevant function, reenabling the thread, and going back to
380 this.schedule_windows_tls_dtors()?;
384 // The remaining dtors make some progress each time around the scheduler loop,
385 // until they return `false` to indicate that they are done.
387 // The macOS thread wide destructor runs "before any TLS slots get
388 // freed", so do that first.
389 if this.schedule_macos_tls_dtor()? {
390 // We have scheduled a MacOS dtor to run on the thread. Execute it
391 // to completion and come back here. Scheduling a destructor
392 // destroys it, so we will not enter this branch again.
395 if this.schedule_next_pthread_tls_dtor()? {
396 // We have scheduled a pthread destructor and removed it from the
397 // destructors list. Run it to completion and come back here.
402 this.machine.tls.delete_all_thread_tls(active_thread);
403 this.thread_terminated()?;