rust/src/rt/rust_task.cpp

#ifndef __WIN32__
#include <execinfo.h>
#endif
#include <iostream>
#include <algorithm>

#include "rust_task.h"
#include "rust_cc.h"
#include "rust_env.h"
#include "rust_port.h"

// TODO(bblum): get rid of supervisors

// Tasks
rust_task::rust_task(rust_sched_loop *sched_loop, rust_task_state state,
                     rust_task *spawner, const char *name,
                     size_t init_stack_sz) :
    ref_count(1),
    id(0),
    notify_enabled(false),
    stk(NULL),
    runtime_sp(0),
    sched(sched_loop->sched),
    sched_loop(sched_loop),
    kernel(sched_loop->kernel),
    name(name),
    list_index(-1),
    rendezvous_ptr(0),
    local_region(&sched_loop->local_region),
    boxed(sched_loop->kernel->env, &local_region),
    unwinding(false),
    propagate_failure(true),
    cc_counter(0),
    total_stack_sz(0),
    task_local_data(NULL),
    task_local_data_cleanup(NULL),
    state(state),
    cond(NULL),
    cond_name("none"),
    event_reject(false),
    event(NULL),
    killed(false),
    reentered_rust_stack(false),
    disallow_kill(0),
    c_stack(NULL),
    next_c_sp(0),
    next_rust_sp(0),
    supervisor(spawner)
{
    LOGPTR(sched_loop, "new task", (uintptr_t)this);
    DLOG(sched_loop, task, "sizeof(task) = %d (0x%x)",
         sizeof *this, sizeof *this);

    new_stack(init_stack_sz);
    if (supervisor) {
        supervisor->ref();
    }
}

// NB: This does not always run on the task's scheduler thread
void
rust_task::delete_this()
{
    DLOG(sched_loop, task, "~rust_task %s @0x%" PRIxPTR ", refcnt=%d",
         name, (uintptr_t)this, ref_count);

    // FIXME (#2677): We should do this when the task exits, not in the
    // destructor
    {
        scoped_lock with(supervisor_lock);
        if (supervisor) {
            supervisor->deref();
        }
    }

    /* FIXME (#2677): tighten this up, there are some more
       assertions that hold at task-lifecycle events. */
    assert(ref_count == 0); // ||
    //   (ref_count == 1 && this == sched->root_task));

    sched_loop->release_task(this);
}

// All failure goes through me. Put your breakpoints here!
extern "C" void
rust_task_fail(rust_task *task,
               char const *expr,
               char const *file,
               size_t line) {
    assert(task != NULL);
    task->begin_failure(expr, file, line);
}

struct spawn_args {
    rust_task *task;
    spawn_fn f;
    rust_opaque_box *envptr;
    void *argptr;
};

struct cleanup_args {
    spawn_args *spargs;
    bool threw_exception;
};

void
annihilate_boxes(rust_task *task);

void
cleanup_task(cleanup_args *args) {
    spawn_args *a = args->spargs;
    bool threw_exception = args->threw_exception;
    rust_task *task = a->task;

    {
        scoped_lock with(task->lifecycle_lock);
        if (task->killed && !threw_exception) {
            LOG(task, task, "Task killed during termination");
            threw_exception = true;
        }
    }

    // Clean up TLS. This will only be set if TLS was used to begin with.
    // Because this is a crust function, it must be called from the C stack.
    if (task->task_local_data_cleanup != NULL) {
        // This assert should hold but it's not our job to ensure it (and
        // the condition might change). Handled in libcore/task.rs.
        // assert(task->task_local_data != NULL);
        task->task_local_data_cleanup(task->task_local_data);
        task->task_local_data = NULL;
    }

    // FIXME (#2676): For performance we should do the annihilator
    // instead of the cycle collector even under normal termination, but
    // since that would hide memory management errors (like not derefing
    // boxes), it needs to be disableable in debug builds.
    if (threw_exception) {
        // FIXME (#2676): When the annihilator is more powerful and
        // successfully runs resource destructors, etc. we can get rid
        // of this cc
        cc::do_cc(task);
        annihilate_boxes(task);
    }
    cc::do_final_cc(task);

    task->die();

    task->notify(!threw_exception);

#ifdef __WIN32__
    assert(!threw_exception && "No exception-handling yet on windows builds");
#endif
}

extern "C" CDECL void upcall_exchange_free(void *ptr);

// This runs on the Rust stack
void task_start_wrapper(spawn_args *a)
{
    rust_task *task = a->task;

    bool threw_exception = false;
    try {
        // The first argument is the return pointer; as the task fn
        // must have void return type, we can safely pass 0.
        a->f(0, a->envptr, a->argptr);
    } catch (rust_task *ex) {
        assert(ex == task && "Expected this task to be thrown for unwinding");
        threw_exception = true;

        if (task->c_stack) {
            task->return_c_stack();
        }

        // Since we call glue code below we need to make sure we
        // have the stack limit set up correctly
        task->reset_stack_limit();
    }

    // We should have returned any C stack by now
    assert(task->c_stack == NULL);

    rust_opaque_box* env = a->envptr;
    if(env) {
        // free the environment (which should be a unique closure).
        const type_desc *td = env->td;
        td->drop_glue(NULL, NULL, NULL, box_body(env));
        upcall_exchange_free(env);
    }

    // The cleanup work needs lots of stack
    cleanup_args ca = {a, threw_exception};
    task->call_on_c_stack(&ca, (void*)cleanup_task);

    task->ctx.next->swap(task->ctx);
}

void
rust_task::start(spawn_fn spawnee_fn,
                 rust_opaque_box *envptr,
                 void *argptr)
{
    LOG(this, task, "starting task from fn 0x%" PRIxPTR
        " with env 0x%" PRIxPTR " and arg 0x%" PRIxPTR,
        spawnee_fn, envptr, argptr);

    assert(stk->data != NULL);

    char *sp = (char *)stk->end;

    sp -= sizeof(spawn_args);

    spawn_args *a = (spawn_args *)sp;

    a->task = this;
    a->envptr = envptr;
    a->argptr = argptr;
    a->f = spawnee_fn;

    ctx.call((void *)task_start_wrapper, a, sp);

    this->start();
}

void rust_task::start()
{
    transition(task_state_newborn, task_state_running, NULL, "none");
}

bool
rust_task::must_fail_from_being_killed() {
    scoped_lock with(lifecycle_lock);
    return must_fail_from_being_killed_inner();
}

bool
rust_task::must_fail_from_being_killed_inner() {
    lifecycle_lock.must_have_lock();
    return killed && !reentered_rust_stack && disallow_kill == 0;
}

// Only run this on the rust stack
void
rust_task::yield(bool *killed) {
    // FIXME (#2787): clean this up
    if (must_fail_from_being_killed()) {
        {
            scoped_lock with(lifecycle_lock);
            assert(!(state == task_state_blocked));
        }
        *killed = true;
    }

    // Return to the scheduler.
    ctx.next->swap(ctx);

    if (must_fail_from_being_killed()) {
        *killed = true;
    }
}

void
rust_task::kill() {
    scoped_lock with(lifecycle_lock);

    // XXX: bblum: kill/kill race

    // Note the distinction here: kill() is when you're in an upcall
    // from task A and want to force-fail task B, you do B->kill().
    // If you want to fail yourself you do self->fail().
    LOG(this, task, "killing task %s @0x%" PRIxPTR, name, this);
    // When the task next goes to yield or resume it will fail
    killed = true;
    // Unblock the task so it can unwind.

    if (state == task_state_blocked &&
        must_fail_from_being_killed_inner()) {
        wakeup_inner(cond);
    }

    LOG(this, task, "preparing to unwind task: 0x%" PRIxPTR, this);
}

extern "C" CDECL
bool rust_task_is_unwinding(rust_task *rt) {
    return rt->unwinding;
}

void
rust_task::fail() {
    // See note in ::kill() regarding who should call this.
    fail(NULL, NULL, 0);
}

void
rust_task::fail(char const *expr, char const *file, size_t line) {
    rust_task_fail(this, expr, file, line);
}

// Called only by rust_task_fail
void
rust_task::begin_failure(char const *expr, char const *file, size_t line) {

    if (expr) {
        LOG_ERR(this, task, "task failed at '%s', %s:%" PRIdPTR,
                expr, file, line);
    }

    DLOG(sched_loop, task, "task %s @0x%" PRIxPTR " failing", name, this);
    backtrace();
    unwinding = true;
#ifndef __WIN32__
    throw this;
#else
    die();
    // FIXME (#908): Need unwinding on windows. This will end up aborting
    sched_loop->fail();
#endif
}

void
rust_task::unsupervise()
{
    scoped_lock with(supervisor_lock);
    if (supervisor) {
        DLOG(sched_loop, task,
             "task %s @0x%" PRIxPTR
             " disconnecting from supervisor %s @0x%" PRIxPTR,
             name, this, supervisor->name, supervisor);
        supervisor->deref();
    }
    supervisor = NULL;
    propagate_failure = false;
}

frame_glue_fns*
rust_task::get_frame_glue_fns(uintptr_t fp) {
    fp -= sizeof(uintptr_t);
    return *((frame_glue_fns**) fp);
}

void rust_task::assert_is_running()
{
    scoped_lock with(lifecycle_lock);
    assert(state == task_state_running);
}

// FIXME (#2851, #2787): This is only used by rust_port/rust_port selector,
// and is inherently racy. Get rid of it.
bool
rust_task::blocked_on(rust_cond *on)
{
    scoped_lock with(lifecycle_lock);
    return cond == on;
}

void *
rust_task::malloc(size_t sz, const char *tag, type_desc *td)
{
    return local_region.malloc(sz, tag);
}

void *
rust_task::realloc(void *data, size_t sz)
{
    return local_region.realloc(data, sz);
}

void
rust_task::free(void *p)
{
    local_region.free(p);
}

void
rust_task::transition(rust_task_state src, rust_task_state dst,
                      rust_cond *cond, const char* cond_name) {
    scoped_lock with(lifecycle_lock);
    transition_inner(src, dst, cond, cond_name);
}

void rust_task::transition_inner(rust_task_state src, rust_task_state dst,
                                  rust_cond *cond, const char* cond_name) {
    lifecycle_lock.must_have_lock();
    sched_loop->transition(this, src, dst, cond, cond_name);
}

void
rust_task::set_state(rust_task_state state,
                     rust_cond *cond, const char* cond_name) {
    lifecycle_lock.must_have_lock();
    this->state = state;
    this->cond = cond;
    this->cond_name = cond_name;
}

bool
rust_task::block(rust_cond *on, const char* name) {
    scoped_lock with(lifecycle_lock);
    return block_inner(on, name);
}

bool
rust_task::block_inner(rust_cond *on, const char* name) {
    if (must_fail_from_being_killed_inner()) {
        // We're already going to die. Don't block. Tell the task to fail
        return false;
    }

    LOG(this, task, "Blocking on 0x%" PRIxPTR ", cond: 0x%" PRIxPTR,
                         (uintptr_t) on, (uintptr_t) cond);
    assert(cond == NULL && "Cannot block an already blocked task.");
    assert(on != NULL && "Cannot block on a NULL object.");

    transition_inner(task_state_running, task_state_blocked, on, name);

    return true;
}

void
rust_task::wakeup(rust_cond *from) {
    scoped_lock with(lifecycle_lock);
    wakeup_inner(from);
}

void
rust_task::wakeup_inner(rust_cond *from) {
    assert(cond != NULL && "Cannot wake up unblocked task.");
    LOG(this, task, "Blocked on 0x%" PRIxPTR " woken up on 0x%" PRIxPTR,
                        (uintptr_t) cond, (uintptr_t) from);
    assert(cond == from && "Cannot wake up blocked task on wrong condition.");

    transition_inner(task_state_blocked, task_state_running, NULL, "none");
}

void
rust_task::die() {
    transition(task_state_running, task_state_dead, NULL, "none");
}

void
rust_task::backtrace() {
    if (!log_rt_backtrace) return;
#ifndef __WIN32__
    void *call_stack[256];
    int nframes = ::backtrace(call_stack, 256);
    backtrace_symbols_fd(call_stack + 1, nframes - 1, 2);
#endif
}

void *
rust_task::calloc(size_t size, const char *tag) {
    return local_region.calloc(size, tag);
}

void
rust_task::notify(bool success) {
    // FIXME (#1078) Do this in rust code
    if(notify_enabled) {
        rust_port *target_port =
            kernel->get_port_by_id(notify_port);
        if(target_port) {
            task_notification msg;
            msg.id = id;
            msg.result = !success ? tr_failure : tr_success;

            target_port->send(&msg);
            target_port->deref();
        }
    }
}

size_t
rust_task::get_next_stack_size(size_t min, size_t current, size_t requested) {
    LOG(this, mem, "calculating new stack size for 0x%" PRIxPTR, this);
    LOG(this, mem,
        "min: %" PRIdPTR " current: %" PRIdPTR " requested: %" PRIdPTR,
        min, current, requested);

    // Allocate at least enough to accomodate the next frame
    size_t sz = std::max(min, requested);

    // And double the stack size each allocation
    const size_t max = 1024 * 1024;
    size_t next = std::min(max, current * 2);

    sz = std::max(sz, next);

    LOG(this, mem, "next stack size: %" PRIdPTR, sz);
    assert(requested <= sz);
    return sz;
}

void
rust_task::free_stack(stk_seg *stk) {
    LOGPTR(sched_loop, "freeing stk segment", (uintptr_t)stk);
    total_stack_sz -= user_stack_size(stk);
    destroy_stack(&local_region, stk);
}

void
new_stack_slow(new_stack_args *args) {
    args->task->new_stack(args->requested_sz);
}

void
rust_task::new_stack(size_t requested_sz) {
    LOG(this, mem, "creating new stack for task %" PRIxPTR, this);
    if (stk) {
        ::check_stack_canary(stk);
    }

    // The minimum stack size, in bytes, of a Rust stack, excluding red zone
    size_t min_sz = sched_loop->min_stack_size;

    // Try to reuse an existing stack segment
    while (stk != NULL && stk->next != NULL) {
        size_t next_sz = user_stack_size(stk->next);
        if (min_sz <= next_sz && requested_sz <= next_sz) {
            LOG(this, mem, "reusing existing stack");
            stk = stk->next;
            return;
        } else {
            LOG(this, mem, "existing stack is not big enough");
            stk_seg *new_next = stk->next->next;
            free_stack(stk->next);
            stk->next = new_next;
            if (new_next) {
                new_next->prev = stk;
            }
        }
    }

    // The size of the current stack segment, excluding red zone
    size_t current_sz = 0;
    if (stk != NULL) {
        current_sz = user_stack_size(stk);
    }
    // The calculated size of the new stack, excluding red zone
    size_t rust_stk_sz = get_next_stack_size(min_sz,
                                             current_sz, requested_sz);

    size_t max_stack = kernel->env->max_stack_size;
    size_t used_stack = total_stack_sz + rust_stk_sz;

    // Don't allow stacks to grow forever. During unwinding we have to allow
    // for more stack than normal in order to allow destructors room to run,
    // arbitrarily selected as 2x the maximum stack size.
    if (!unwinding && used_stack > max_stack) {
        LOG_ERR(this, task, "task %" PRIxPTR " ran out of stack", this);
        fail();
    } else if (unwinding && used_stack > max_stack * 2) {
        LOG_ERR(this, task,
                "task %" PRIxPTR " ran out of stack during unwinding", this);
        fail();
    }

    size_t sz = rust_stk_sz + RED_ZONE_SIZE;
    stk_seg *new_stk = create_stack(&local_region, sz);
    LOGPTR(sched_loop, "new stk", (uintptr_t)new_stk);
    new_stk->task = this;
    new_stk->next = NULL;
    new_stk->prev = stk;
    if (stk) {
        stk->next = new_stk;
    }
    LOGPTR(sched_loop, "stk end", new_stk->end);

    stk = new_stk;
    total_stack_sz += user_stack_size(new_stk);
}

void
rust_task::cleanup_after_turn() {
    // Delete any spare stack segments that were left
    // behind by calls to prev_stack
    assert(stk);
    while (stk->next) {
        stk_seg *new_next = stk->next->next;
        free_stack(stk->next);
        stk->next = new_next;
    }
}

static bool
sp_in_stk_seg(uintptr_t sp, stk_seg *stk) {
    // Not positive these bounds for sp are correct.  I think that the first
    // possible value for esp on a new stack is stk->end, which points to the
    // address before the first value to be pushed onto a new stack. The last
    // possible address we can push data to is stk->data.  Regardless, there's
    // so much slop at either end that we should never hit one of these
    // boundaries.
    return (uintptr_t)stk->data <= sp && sp <= stk->end;
}

struct reset_args {
    rust_task *task;
    uintptr_t sp;
};

void
reset_stack_limit_on_c_stack(reset_args *args) {
}

/*
Called by landing pads during unwinding to figure out which stack segment we
are currently running on and record the stack limit (which was not restored
when unwinding through __morestack).
 */
void
rust_task::reset_stack_limit() {
    uintptr_t sp = get_sp();
    while (!sp_in_stk_seg(sp, stk)) {
        stk = stk->prev;
        assert(stk != NULL && "Failed to find the current stack");
    }
    record_stack_limit();
}

void
rust_task::check_stack_canary() {
    ::check_stack_canary(stk);
}

void
rust_task::delete_all_stacks() {
    assert(!on_rust_stack());
    // Delete all the stacks. There may be more than one if the task failed
    // and no landing pads stopped to clean up.
    assert(stk->next == NULL);
    while (stk != NULL) {
        stk_seg *prev = stk->prev;
        free_stack(stk);
        stk = prev;
    }
}

void
rust_task::config_notify(rust_port_id port) {
    notify_enabled = true;
    notify_port = port;
}

/*
Returns true if we're currently running on the Rust stack
 */
bool
rust_task::on_rust_stack() {
    if (stk == NULL) {
        // This only happens during construction
        return false;
    }

    uintptr_t sp = get_sp();
    bool in_first_segment = sp_in_stk_seg(sp, stk);
    if (in_first_segment) {
        return true;
    } else if (stk->prev != NULL) {
        // This happens only when calling the upcall to delete
        // a stack segment
        bool in_second_segment = sp_in_stk_seg(sp, stk->prev);
        return in_second_segment;
    } else {
        return false;
    }
}

void
rust_task::inhibit_kill() {
    scoped_lock with(lifecycle_lock);
    // FIXME (#1868) Check here if we have to die
    disallow_kill++;
}

void
rust_task::allow_kill() {
    scoped_lock with(lifecycle_lock);
    assert(disallow_kill > 0 && "Illegal allow_kill(): already killable!");
    disallow_kill--;
}

void *
rust_task::wait_event(bool *killed) {
    scoped_lock with(lifecycle_lock);

    if(!event_reject) {
        block_inner(&event_cond, "waiting on event");
        lifecycle_lock.unlock();
        yield(killed);
        lifecycle_lock.lock();
    }

    event_reject = false;
    return event;
}

void
rust_task::signal_event(void *event) {
    scoped_lock with(lifecycle_lock);

    this->event = event;
    event_reject = true;
    if(task_state_blocked == state) {
        wakeup_inner(&event_cond);
    }
}

//
// Local Variables:
// mode: C++
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// End:
//