1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
12 The rust task is a cooperatively-scheduled green thread that executes
13 Rust code on a segmented stack.
15 This class has too many responsibilities:
17 * Working with the scheduler loop to signal and respond to state changes,
18 and dealing with all the thread synchronization issues involved
20 * Managing the dynamically resizing list of Rust stack segments
22 * Switching between running Rust code on the Rust segmented stack and
23 foreign C code on large stacks owned by the scheduler
27 The lifetime of a rust_task object closely mirrors that of a running Rust
28 task object, but they are not identical. In particular, the rust_task is an
29 atomically reference counted object that might be accessed from arbitrary
30 threads at any time. This may keep the task from being destroyed even after
31 the task is dead from a Rust task lifecycle perspective. The rust_tasks are
32 reference counted in the following places:
34 * By the task's lifetime (i.e., running tasks hold a reference to themself)
36 * In the rust_task_kill_all -> rust_kernel::fail ->
37 rust_sched_loop::kill_all_tasks path. When a task brings down the whole
38 runtime, each sched_loop must use refcounts to take a 'snapshot' of all
39 existing tasks so it can be sure to kill all of them.
41 * In core::pipes, tasks that use select() use reference counts to avoid
42 use-after-free races with multiple different signallers.
46 All task death goes through a single central path: The task invokes
47 rust_task::die(), which invokes transition(task_state_dead), which pumps
48 the scheduler loop, which switches to rust_sched_loop::run_single_turn(),
49 which calls reap_dead_tasks(), which cleans up the task's stack segments
50 and drops the reference count.
52 When a task's reference count hits zero, rust_sched_loop::release_task()
53 is called. This frees the memory and deregisters the task from the kernel,
54 which may trigger the sched_loop, the scheduler, and/or the kernel to exit
55 completely in the case it was the last task alive.
57 die() is called from two places: the successful exit path, in cleanup_task,
58 and on failure (on linux, this is also in cleanup_task, after unwinding
59 completes; on windows, it is in begin_failure).
61 Tasks do not force-quit other tasks; a task die()s only itself. However...
65 Tasks may kill each other. This happens when propagating failure between
66 tasks (see the task::spawn options interface). The code path for this is
67 rust_task_kill_other() -> rust_task::kill().
69 It also happens when the main ("root") task (or any task in that task's
70 linked-failure-group) fails: this brings down the whole runtime, and kills
71 all tasks in all groups. The code path for this is rust_task_kill_all() ->
72 rust_kernel::fail() -> rust_scheduler::kill_all_tasks() ->
73 rust_sched_loop::kill_all_tasks() -> rust_task::kill().
75 In either case, killing a task involves, under the protection of its
76 lifecycle_lock, (a) setting the 'killed' flag, and (b) checking if it is
77 'blocked'* and if so punting it awake.
78 (* and also isn't unkillable, which may happen via task::unkillable()
79 or via calling an extern rust function from C.)
81 The killed task will then (wake up if it was asleep, and) eventually call
82 yield() (or wait_event()), which will check the killed flag, see that it is
83 true, and then invoke 'fail', which begins the death process described
86 Three things guarantee concurrency safety in this whole affair:
88 * The lifecycle_lock protects tasks accessing each other's state: it makes
89 killing-and-waking up atomic with respect to a task in block() deciding
90 whether it's allowed to go to sleep, so tasks can't 'escape' being woken.
92 * In the case of linked failure propagation, we ensure (in task.rs) that
93 tasks can only see another task's rust_task pointer if that task is
94 already alive. Even before entering the runtime failure path, a task will
95 access (locked) the linked-failure data structures to remove its task
96 pointer so that no subsequently-failing tasks will do a use-after-free.
98 * In the case of bringing down the whole runtime, each sched_loop takes an
99 "atomic snapshot" of all its tasks, protected by the sched_loop's lock,
100 and also sets a 'failing' flag so that any subsequently-failing task will
101 know that it must fail immediately upon creation (which is also checked
102 under the same lock). A similar process exists at the one-step-higher
103 level of the kernel killing all the schedulers (the kernel snapshots all
104 the schedulers and sets a 'failing' flag in the scheduler table).
112 #include "rust_globals.h"
113 #include "util/array_list.h"
115 #include "rust_debug.h"
116 #include "rust_kernel.h"
117 #include "boxed_region.h"
118 #include "rust_stack.h"
119 #include "rust_type.h"
120 #include "rust_sched_loop.h"
122 // The amount of extra space at the end of each stack segment, available
123 // to the rt, compiler and dynamic linker for running small functions
124 // FIXME (#1509): We want this to be 128 but need to slim the red zone calls
125 // down, disable lazy symbol relocation, and other things we haven't
127 #define RZ_LINUX_32 (1024*2)
128 #define RZ_LINUX_64 (1024*2)
129 #define RZ_MAC_32 (1024*20)
130 #define RZ_MAC_64 (1024*20)
131 #define RZ_WIN_32 (1024*20)
132 #define RZ_BSD_32 (1024*20)
133 #define RZ_BSD_64 (1024*20)
137 #define RED_ZONE_SIZE RZ_LINUX_32
140 #define RED_ZONE_SIZE RZ_LINUX_64
145 #define RED_ZONE_SIZE RZ_MAC_32
148 #define RED_ZONE_SIZE RZ_MAC_64
153 #define RED_ZONE_SIZE RZ_WIN_32
156 #define RED_ZONE_SIZE RZ_WIN_64
161 #define RED_ZONE_SIZE RZ_BSD_32
164 #define RED_ZONE_SIZE RZ_BSD_64
168 #define RED_ZONE_SIZE RZ_MAC_32
171 struct frame_glue_fns {
172 uintptr_t mark_glue_off;
173 uintptr_t drop_glue_off;
174 uintptr_t reloc_glue_off;
177 // std::lib::task::task_result
178 typedef unsigned long task_result;
185 struct new_stack_args;
187 // std::lib::task::task_notification
189 // since it's currently a unary tag, we only add the fields.
190 struct task_notification {
192 task_result result; // task_result
196 rust_task_fail(rust_task *task,
202 rust_task : public kernel_owned<rust_task>
204 RUST_ATOMIC_REFCOUNT();
210 uintptr_t runtime_sp; // Runtime sp while task running.
211 rust_scheduler *sched;
212 rust_sched_loop *sched_loop;
214 // Fields known only to the runtime.
216 const char *const name;
220 memory_region local_region;
222 // Indicates that fail() has been called and we are cleaning up.
223 // We use this to suppress the "killed" flag during calls to yield.
226 bool propagate_failure;
228 debug::task_debug_info debug;
230 // The amount of stack we're using, excluding red zones
231 size_t total_stack_sz;
233 // Used by rust task management routines in libcore/task.rs.
234 void *task_local_data;
235 void (*task_local_data_cleanup)(void *data);
239 // Protects state, cond, cond_name
240 // Protects the killed flag, disallow_kill flag, reentered_rust_stack
241 lock_and_signal lifecycle_lock;
242 rust_task_state state;
244 const char *cond_name;
247 rust_cond event_cond;
250 // Indicates that the task was killed and needs to unwind
252 // Indicates that we've called back into Rust from C
253 bool reentered_rust_stack;
254 unsigned long disallow_kill;
255 unsigned long disallow_yield;
257 // The stack used for running C code, borrowed from the scheduler thread
260 uintptr_t next_rust_sp;
262 // Called when the atomic refcount reaches zero
265 void new_stack_fast(size_t requested_sz);
266 void new_stack(size_t requested_sz);
267 void free_stack(stk_seg *stk);
268 size_t get_next_stack_size(size_t min, size_t current, size_t requested);
270 void return_c_stack();
272 void transition(rust_task_state src, rust_task_state dst,
273 rust_cond *cond, const char* cond_name);
274 void transition_inner(rust_task_state src, rust_task_state dst,
275 rust_cond *cond, const char* cond_name);
277 bool must_fail_from_being_killed_inner();
278 // Called by rust_task_fail to unwind on failure
279 void begin_failure(char const *expr,
283 friend void task_start_wrapper(spawn_args *a);
284 friend void cleanup_task(cleanup_args *a);
285 friend void reset_stack_limit_on_c_stack(reset_args *a);
286 friend void new_stack_slow(new_stack_args *a);
287 friend void rust_task_fail(rust_task *task,
292 bool block_inner(rust_cond *on, const char* name);
293 void wakeup_inner(rust_cond *from);
294 bool blocked_on(rust_cond *cond);
297 // private and undefined to disable copying
298 rust_task(const rust_task& rhs);
299 rust_task& operator=(const rust_task& rhs);
303 // Only a pointer to 'name' is kept, so it must live as long as this task.
304 rust_task(rust_sched_loop *sched_loop,
305 rust_task_state state,
307 size_t init_stack_sz);
309 void start(spawn_fn spawnee_fn,
310 rust_opaque_box *env,
313 void assert_is_running();
315 void *malloc(size_t sz, const char *tag, type_desc *td=0);
316 void *realloc(void *data, size_t sz);
319 void set_state(rust_task_state state,
320 rust_cond *cond, const char* cond_name);
322 bool block(rust_cond *on, const char* name);
323 void wakeup(rust_cond *from);
326 // Print a backtrace, if the "bt" logging option is on.
329 // Yields control to the scheduler. Called from the Rust stack
330 // Returns TRUE if the task was killed and needs to fail.
331 MUST_CHECK bool yield();
333 // Fail this task (assuming caller-on-stack is different task).
337 // Indicates that we've been killed and now is an apropriate
338 // time to fail as a result
339 bool must_fail_from_being_killed();
341 // Fail self, assuming caller-on-stack is this task.
343 void fail(char const *expr, char const *file, size_t line);
345 // Propagate failure to the entire rust runtime.
346 void fail_sched_loop();
348 void *calloc(size_t size, const char *tag);
350 // Use this function sparingly. Depending on the ref count is generally
352 intptr_t get_ref_count() const { return ref_count; }
354 void *next_stack(size_t stk_sz, void *args_addr, size_t args_sz);
356 void record_stack_limit();
357 void reset_stack_limit();
359 bool on_rust_stack();
360 void check_stack_canary();
361 void delete_all_stacks();
363 void call_on_c_stack(void *args, void *fn_ptr);
364 void call_on_rust_stack(void *args, void *fn_ptr);
365 bool have_c_stack() { return c_stack != NULL; }
367 rust_task_state get_state() { return state; }
368 rust_cond *get_cond() { return cond; }
369 const char *get_cond_name() { return cond_name; }
371 void clear_event_reject() {
372 this->event_reject = false;
375 // Returns TRUE if the task was killed and needs to fail.
376 MUST_CHECK bool wait_event(void **result);
377 void signal_event(void *event);
379 void cleanup_after_turn();
383 void inhibit_yield();
387 // FIXME (#2697): It would be really nice to be able to get rid of this.
388 inline void *operator new[](size_t size, rust_task *task, const char *tag) {
389 return task->malloc(size, tag);
393 template <typename T> struct task_owned {
394 inline void *operator new(size_t size, rust_task *task,
396 return task->malloc(size, tag);
399 inline void *operator new[](size_t size, rust_task *task,
401 return task->malloc(size, tag);
404 inline void *operator new(size_t size, rust_task &task,
406 return task.malloc(size, tag);
409 inline void *operator new[](size_t size, rust_task &task,
411 return task.malloc(size, tag);
414 void operator delete(void *ptr) {
415 ((T *)ptr)->task->free(ptr);
419 // This stuff is on the stack-switching fast path
421 // Records the pointer to the end of the Rust stack in a platform-
422 // specific location in the thread control block
423 extern "C" CDECL void record_sp_limit(void *limit);
424 extern "C" CDECL uintptr_t get_sp_limit();
425 // Gets a pointer to the vicinity of the current stack pointer
426 extern "C" uintptr_t get_sp();
428 // This is the function that switches between the C and the Rust stack by
429 // calling another function with a single void* argument while changing the
430 // stack pointer. It has a funny name because gdb doesn't normally like to
431 // backtrace through split stacks (thinks it indicates a bug), but has a
432 // special case to allow functions named __morestack to move the stack pointer
434 extern "C" void __morestack(void *args, void *fn_ptr, uintptr_t stack_ptr);
436 inline static uintptr_t
437 sanitize_next_sp(uintptr_t next_sp) {
439 // Since I'm not precisely sure where the next stack pointer sits in
440 // relation to where the context switch actually happened, nor in relation
441 // to the amount of stack needed for calling __morestack I've added some
444 // FIXME (#2698): On the rust stack this potentially puts is quite far
445 // into the red zone. Might want to just allocate a new rust stack every
446 // time we switch back to rust.
447 const uintptr_t padding = 16;
449 return align_down(next_sp - padding);
453 rust_task::call_on_c_stack(void *args, void *fn_ptr) {
454 // Too expensive to check
455 // assert(on_rust_stack());
457 // The shim functions generated by rustc contain the morestack prologue,
458 // so we need to let them know they have enough stack.
461 uintptr_t prev_rust_sp = next_rust_sp;
462 next_rust_sp = get_sp();
464 bool borrowed_a_c_stack = false;
466 if (c_stack == NULL) {
467 c_stack = sched_loop->borrow_c_stack();
468 next_c_sp = align_down(c_stack->end);
470 borrowed_a_c_stack = true;
472 sp = sanitize_next_sp(next_c_sp);
475 __morestack(args, fn_ptr, sp);
477 // Note that we may not actually get here if we threw an exception,
478 // in which case we will return the c stack when the exception is caught.
479 if (borrowed_a_c_stack) {
483 next_rust_sp = prev_rust_sp;
485 record_stack_limit();
489 rust_task::call_on_rust_stack(void *args, void *fn_ptr) {
490 // Too expensive to check
491 // assert(!on_rust_stack());
493 // Because of the hack in the other function that disables the stack limit
494 // when entering the C stack, here we restore the stack limit again.
495 record_stack_limit();
497 assert(get_sp_limit() != 0 && "Stack must be configured");
498 assert(next_rust_sp);
500 // Unlocked access. Might "race" a killer, but harmlessly. This code is
501 // only run by the task itself, so cannot race itself. See the comment
502 // above inhibit_kill (or #3213) in rust_task.cpp for justification.
503 bool had_reentered_rust_stack = reentered_rust_stack;
504 reentered_rust_stack = true;
506 uintptr_t prev_c_sp = next_c_sp;
507 next_c_sp = get_sp();
509 uintptr_t sp = sanitize_next_sp(next_rust_sp);
511 // FIXME (#2047): There are times when this is called and needs
512 // to be able to throw, and we don't account for that.
513 __morestack(args, fn_ptr, sp);
515 next_c_sp = prev_c_sp;
516 reentered_rust_stack = had_reentered_rust_stack;
522 rust_task::return_c_stack() {
523 // Too expensive to check
524 // assert(on_rust_stack());
525 assert(c_stack != NULL);
526 sched_loop->return_c_stack(c_stack);
531 // NB: This runs on the Rust stack
533 rust_task::next_stack(size_t stk_sz, void *args_addr, size_t args_sz) {
534 new_stack_fast(stk_sz + args_sz);
535 assert(stk->end - (uintptr_t)stk->data >= stk_sz + args_sz
536 && "Did not receive enough stack");
537 uint8_t *new_sp = (uint8_t*)stk->end;
538 // Push the function arguments to the new stack
539 new_sp = align_down(new_sp - args_sz);
541 // I don't know exactly where the region ends that valgrind needs us
542 // to mark accessible. On x86_64 these extra bytes aren't needed, but
543 // on i386 we get errors without.
544 const int fudge_bytes = 16;
545 reuse_valgrind_stack(stk, new_sp - fudge_bytes);
547 memcpy(new_sp, args_addr, args_sz);
548 record_stack_limit();
552 // The amount of stack in a segment available to Rust code
554 user_stack_size(stk_seg *stk) {
555 return (size_t)(stk->end
556 - (uintptr_t)&stk->data[0]
560 struct new_stack_args {
566 new_stack_slow(new_stack_args *args);
568 // NB: This runs on the Rust stack
569 // This is the new stack fast path, in which we
570 // reuse the next cached stack segment
572 rust_task::new_stack_fast(size_t requested_sz) {
573 // The minimum stack size, in bytes, of a Rust stack, excluding red zone
574 size_t min_sz = sched_loop->min_stack_size;
576 // Try to reuse an existing stack segment
577 if (stk != NULL && stk->next != NULL) {
578 size_t next_sz = user_stack_size(stk->next);
579 if (min_sz <= next_sz && requested_sz <= next_sz) {
585 new_stack_args args = {this, requested_sz};
586 call_on_c_stack(&args, (void*)new_stack_slow);
589 // NB: This runs on the Rust stack
591 rust_task::prev_stack() {
592 // We're not going to actually delete anything now because that would
593 // require switching to the C stack and be costly. Instead we'll just move
594 // up the link list and clean up later, either in new_stack or after our
595 // turn ends on the scheduler.
597 record_stack_limit();
600 extern "C" CDECL void
601 record_sp_limit(void *limit);
603 // The LLVM-generated segmented-stack function prolog compares the amount of
604 // stack needed for each frame to the end-of-stack pointer stored in the
605 // TCB. As an optimization, when the frame size is less than 256 bytes, it
606 // will simply compare %esp to to the stack limit instead of subtracting the
607 // frame size. As a result we need our stack limit to account for those 256
609 const unsigned LIMIT_OFFSET = 256;
612 rust_task::record_stack_limit() {
614 assert((uintptr_t)stk->end - RED_ZONE_SIZE
615 - (uintptr_t)stk->data >= LIMIT_OFFSET
616 && "Stack size must be greater than LIMIT_OFFSET");
617 record_sp_limit(stk->data + LIMIT_OFFSET + RED_ZONE_SIZE);
620 inline rust_task* rust_try_get_current_task() {
621 uintptr_t sp_limit = get_sp_limit();
623 // FIXME (#1226) - Because of a hack in upcall_call_shim_on_c_stack this
624 // value is sometimes inconveniently set to 0, so we can't use this
625 // method of retreiving the task pointer and need to fall back to TLS.
627 return rust_sched_loop::try_get_task_tls();
629 // The stack pointer boundary is stored in a quickly-accessible location
630 // in the TCB. From that we can calculate the address of the stack segment
631 // structure it belongs to, and in that structure is a pointer to the task
634 sp_limit - RED_ZONE_SIZE - LIMIT_OFFSET - sizeof(stk_seg);
635 stk_seg *stk = (stk_seg*) seg_addr;
637 // Make sure we've calculated the right address
638 ::check_stack_canary(stk);
639 assert(stk->task != NULL && "task pointer not in stack structure");
643 inline rust_task* rust_get_current_task() {
644 rust_task* task = rust_try_get_current_task();
645 assert(task != NULL && "no current task");
653 // indent-tabs-mode: nil
655 // buffer-file-coding-system: utf-8-unix
659 #endif /* RUST_TASK_H */