2012-12-03 18:48:01 -06:00
|
|
|
// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
|
|
|
|
// file at the top-level directory of this distribution and at
|
|
|
|
// http://rust-lang.org/COPYRIGHT.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
|
|
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
|
|
// option. This file may not be copied, modified, or distributed
|
|
|
|
// except according to those terms.
|
|
|
|
|
2012-09-19 18:52:32 -05:00
|
|
|
/*! Precise garbage collector
|
2012-09-06 13:25:24 -05:00
|
|
|
|
|
|
|
The precise GC exposes two functions, gc and
|
|
|
|
cleanup_stack_for_failure. The gc function is the entry point to the
|
|
|
|
garbage collector itself. The cleanup_stack_for_failure is the entry
|
|
|
|
point for GC-based cleanup.
|
|
|
|
|
|
|
|
Precise GC depends on changes to LLVM's GC which add support for
|
|
|
|
automatic rooting and addrspace-based metadata marking. Rather than
|
|
|
|
explicitly rooting pointers with LLVM's gcroot intrinsic, the GC
|
|
|
|
merely creates allocas for pointers, and allows an LLVM pass to
|
|
|
|
automatically infer roots based on the allocas present in a function
|
|
|
|
(and live at a given location). The compiler communicates the type of
|
|
|
|
the pointer to LLVM by setting the addrspace of the pointer type. The
|
|
|
|
compiler then emits a map from addrspace to tydesc, which LLVM then
|
|
|
|
uses to match pointers with their tydesc. The GC reads the metadata
|
|
|
|
table produced by LLVM, and uses it to determine which glue functions
|
|
|
|
to call to free objects on their respective heaps.
|
|
|
|
|
|
|
|
GC-based cleanup is a replacement for landing pads which relies on the
|
|
|
|
GC infrastructure to find pointers on the stack to cleanup. Whereas
|
|
|
|
the normal GC needs to walk task-local heap allocations, the cleanup
|
|
|
|
code needs to walk exchange heap allocations and stack-allocations
|
|
|
|
with destructors.
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
2012-12-23 16:41:37 -06:00
|
|
|
use cast;
|
2013-01-25 18:57:39 -06:00
|
|
|
use container::{Container, Mutable, Map, Set};
|
2012-12-23 16:41:37 -06:00
|
|
|
use io;
|
|
|
|
use libc::{size_t, uintptr_t};
|
2013-01-08 21:37:25 -06:00
|
|
|
use option::{None, Option, Some};
|
2012-12-23 16:41:37 -06:00
|
|
|
use ptr;
|
2013-01-23 09:25:27 -06:00
|
|
|
use hashmap::linear::LinearSet;
|
2012-12-23 16:41:37 -06:00
|
|
|
use stackwalk;
|
|
|
|
use sys;
|
|
|
|
|
|
|
|
pub use stackwalk::Word;
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2012-08-24 12:43:57 -05:00
|
|
|
// Mirrors rust_stack.h stk_seg
|
2013-03-05 13:57:50 -06:00
|
|
|
pub struct StackSegment {
|
2012-09-06 21:40:15 -05:00
|
|
|
prev: *StackSegment,
|
|
|
|
next: *StackSegment,
|
|
|
|
end: uintptr_t,
|
2012-08-24 12:43:57 -05:00
|
|
|
// And other fields which we don't care about...
|
|
|
|
}
|
|
|
|
|
2013-03-05 13:57:50 -06:00
|
|
|
pub mod rustrt {
|
|
|
|
use libc::size_t;
|
|
|
|
use stackwalk::Word;
|
|
|
|
use super::StackSegment;
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2013-03-05 13:57:50 -06:00
|
|
|
#[link_name = "rustrt"]
|
|
|
|
pub extern {
|
|
|
|
#[rust_stack]
|
|
|
|
pub unsafe fn rust_call_tydesc_glue(root: *Word,
|
|
|
|
tydesc: *Word,
|
|
|
|
field: size_t);
|
2012-08-24 12:43:57 -05:00
|
|
|
|
2013-03-05 13:57:50 -06:00
|
|
|
#[rust_stack]
|
|
|
|
pub unsafe fn rust_gc_metadata() -> *Word;
|
|
|
|
|
|
|
|
pub unsafe fn rust_get_stack_segment() -> *StackSegment;
|
|
|
|
}
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
unsafe fn bump<T, U>(ptr: *T, count: uint) -> *U {
|
2012-09-18 19:34:08 -05:00
|
|
|
return cast::reinterpret_cast(&ptr::offset(ptr, count));
|
2012-09-06 15:51:02 -05:00
|
|
|
}
|
2012-08-24 12:43:57 -05:00
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
unsafe fn align_to_pointer<T>(ptr: *T) -> *T {
|
|
|
|
let align = sys::min_align_of::<*T>();
|
2012-09-18 19:34:08 -05:00
|
|
|
let ptr: uint = cast::reinterpret_cast(&ptr);
|
2012-09-06 15:51:02 -05:00
|
|
|
let ptr = (ptr + (align - 1)) & -align;
|
2012-09-18 19:34:08 -05:00
|
|
|
return cast::reinterpret_cast(&ptr);
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
|
|
|
|
2012-09-06 23:22:50 -05:00
|
|
|
unsafe fn get_safe_point_count() -> uint {
|
|
|
|
let module_meta = rustrt::rust_gc_metadata();
|
|
|
|
return *module_meta;
|
|
|
|
}
|
|
|
|
|
2013-01-22 10:12:52 -06:00
|
|
|
struct SafePoint {
|
|
|
|
sp_meta: *Word,
|
|
|
|
fn_meta: *Word,
|
|
|
|
}
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2012-09-06 13:25:24 -05:00
|
|
|
// Returns the safe point metadata for the given program counter, if
|
|
|
|
// any.
|
2012-07-16 14:28:15 -05:00
|
|
|
unsafe fn is_safe_point(pc: *Word) -> Option<SafePoint> {
|
|
|
|
let module_meta = rustrt::rust_gc_metadata();
|
2012-09-06 15:51:02 -05:00
|
|
|
let num_safe_points = *module_meta;
|
|
|
|
let safe_points: *Word = bump(module_meta, 1);
|
2012-07-16 14:28:15 -05:00
|
|
|
|
|
|
|
if ptr::is_null(pc) {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2012-09-06 13:25:24 -05:00
|
|
|
// FIXME (#2997): Use binary rather than linear search.
|
2012-09-06 15:51:02 -05:00
|
|
|
let mut spi = 0;
|
|
|
|
while spi < num_safe_points {
|
|
|
|
let sp: **Word = bump(safe_points, spi*3);
|
|
|
|
let sp_loc = *sp;
|
2012-07-16 14:28:15 -05:00
|
|
|
if sp_loc == pc {
|
2013-01-22 10:12:52 -06:00
|
|
|
return Some(SafePoint {
|
|
|
|
sp_meta: *bump(sp, 1),
|
|
|
|
fn_meta: *bump(sp, 2),
|
|
|
|
});
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
2012-09-06 15:51:02 -05:00
|
|
|
spi += 1;
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2013-03-14 13:22:51 -05:00
|
|
|
type Visitor = &'self fn(root: **Word, tydesc: *Word) -> bool;
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2012-09-06 13:25:24 -05:00
|
|
|
// Walks the list of roots for the given safe point, and calls visitor
|
|
|
|
// on each root.
|
2012-07-16 14:28:15 -05:00
|
|
|
unsafe fn walk_safe_point(fp: *Word, sp: SafePoint, visitor: Visitor) {
|
2012-09-18 19:34:08 -05:00
|
|
|
let fp_bytes: *u8 = cast::reinterpret_cast(&fp);
|
|
|
|
let sp_meta: *u32 = cast::reinterpret_cast(&sp.sp_meta);
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
let num_stack_roots = *sp_meta as uint;
|
|
|
|
let num_reg_roots = *ptr::offset(sp_meta, 1) as uint;
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
let stack_roots: *u32 = bump(sp_meta, 2);
|
|
|
|
let reg_roots: *u8 = bump(stack_roots, num_stack_roots);
|
|
|
|
let addrspaces: *Word = align_to_pointer(bump(reg_roots, num_reg_roots));
|
|
|
|
let tydescs: ***Word = bump(addrspaces, num_stack_roots);
|
2012-07-16 14:28:15 -05:00
|
|
|
|
|
|
|
// Stack roots
|
|
|
|
let mut sri = 0;
|
|
|
|
while sri < num_stack_roots {
|
|
|
|
if *ptr::offset(addrspaces, sri) >= 1 {
|
|
|
|
let root =
|
|
|
|
ptr::offset(fp_bytes, *ptr::offset(stack_roots, sri) as Word)
|
|
|
|
as **Word;
|
|
|
|
let tydescpp = ptr::offset(tydescs, sri);
|
|
|
|
let tydesc = if ptr::is_not_null(tydescpp) &&
|
|
|
|
ptr::is_not_null(*tydescpp) {
|
|
|
|
**tydescpp
|
|
|
|
} else {
|
|
|
|
ptr::null()
|
|
|
|
};
|
2012-08-17 18:17:18 -05:00
|
|
|
if !visitor(root, tydesc) { return; }
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
|
|
|
sri += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Register roots
|
|
|
|
let mut rri = 0;
|
|
|
|
while rri < num_reg_roots {
|
|
|
|
if *ptr::offset(addrspaces, num_stack_roots + rri) == 1 {
|
|
|
|
// FIXME(#2997): Need to find callee saved registers on the stack.
|
|
|
|
}
|
|
|
|
rri += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
// Is fp contained in segment?
|
|
|
|
unsafe fn is_frame_in_segment(fp: *Word, segment: *StackSegment) -> bool {
|
2012-09-18 19:34:08 -05:00
|
|
|
let begin: Word = cast::reinterpret_cast(&segment);
|
|
|
|
let end: Word = cast::reinterpret_cast(&(*segment).end);
|
|
|
|
let frame: Word = cast::reinterpret_cast(&fp);
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
return begin <= frame && frame <= end;
|
|
|
|
}
|
2012-07-16 14:28:15 -05:00
|
|
|
|
2013-02-07 20:23:47 -06:00
|
|
|
struct Segment { segment: *StackSegment, boundary: bool }
|
|
|
|
|
2012-09-06 13:25:24 -05:00
|
|
|
// Find and return the segment containing the given frame pointer. At
|
|
|
|
// stack segment boundaries, returns true for boundary, so that the
|
|
|
|
// caller can do any special handling to identify where the correct
|
|
|
|
// return address is in the stack frame.
|
2012-08-24 12:43:57 -05:00
|
|
|
unsafe fn find_segment_for_frame(fp: *Word, segment: *StackSegment)
|
2013-02-07 20:23:47 -06:00
|
|
|
-> Segment {
|
2012-08-24 12:43:57 -05:00
|
|
|
// Check if frame is in either current frame or previous frame.
|
|
|
|
let in_segment = is_frame_in_segment(fp, segment);
|
|
|
|
let in_prev_segment = ptr::is_not_null((*segment).prev) &&
|
|
|
|
is_frame_in_segment(fp, (*segment).prev);
|
|
|
|
|
|
|
|
// If frame is not in either segment, walk down segment list until
|
|
|
|
// we find the segment containing this frame.
|
|
|
|
if !in_segment && !in_prev_segment {
|
|
|
|
let mut segment = segment;
|
|
|
|
while ptr::is_not_null((*segment).next) &&
|
|
|
|
is_frame_in_segment(fp, (*segment).next) {
|
|
|
|
segment = (*segment).next;
|
|
|
|
}
|
2013-02-07 20:23:47 -06:00
|
|
|
return Segment {segment: segment, boundary: false};
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// If frame is in previous frame, then we're at a boundary.
|
|
|
|
if !in_segment && in_prev_segment {
|
2013-02-07 20:23:47 -06:00
|
|
|
return Segment {segment: (*segment).prev, boundary: true};
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, we're somewhere on the inside of the frame.
|
2013-02-07 20:23:47 -06:00
|
|
|
return Segment {segment: segment, boundary: false};
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
|
|
|
|
2012-09-06 15:51:02 -05:00
|
|
|
type Memory = uint;
|
|
|
|
|
|
|
|
const task_local_heap: Memory = 1;
|
|
|
|
const exchange_heap: Memory = 2;
|
|
|
|
const stack: Memory = 4;
|
|
|
|
|
|
|
|
const need_cleanup: Memory = exchange_heap | stack;
|
|
|
|
|
2012-09-06 13:25:24 -05:00
|
|
|
// Walks stack, searching for roots of the requested type, and passes
|
|
|
|
// each root to the visitor.
|
2012-08-23 14:14:47 -05:00
|
|
|
unsafe fn walk_gc_roots(mem: Memory, sentinel: **Word, visitor: Visitor) {
|
2012-08-24 12:43:57 -05:00
|
|
|
let mut segment = rustrt::rust_get_stack_segment();
|
2012-07-16 14:28:15 -05:00
|
|
|
let mut last_ret: *Word = ptr::null();
|
2012-08-23 14:14:47 -05:00
|
|
|
// To avoid collecting memory used by the GC itself, skip stack
|
|
|
|
// frames until past the root GC stack frame. The root GC stack
|
|
|
|
// frame is marked by a sentinel, which is a box pointer stored on
|
|
|
|
// the stack.
|
|
|
|
let mut reached_sentinel = ptr::is_null(sentinel);
|
2012-08-17 18:17:18 -05:00
|
|
|
for stackwalk::walk_stack |frame| {
|
2012-07-16 14:28:15 -05:00
|
|
|
unsafe {
|
2012-08-24 12:43:57 -05:00
|
|
|
let pc = last_ret;
|
2013-02-07 20:23:47 -06:00
|
|
|
let Segment {segment: next_segment, boundary: boundary} =
|
2012-08-24 12:43:57 -05:00
|
|
|
find_segment_for_frame(frame.fp, segment);
|
|
|
|
segment = next_segment;
|
2012-09-06 13:25:24 -05:00
|
|
|
// Each stack segment is bounded by a morestack frame. The
|
|
|
|
// morestack frame includes two return addresses, one for
|
|
|
|
// morestack itself, at the normal offset from the frame
|
|
|
|
// pointer, and then a second return address for the
|
|
|
|
// function prologue (which called morestack after
|
|
|
|
// determining that it had hit the end of the stack).
|
|
|
|
// Since morestack itself takes two parameters, the offset
|
|
|
|
// for this second return address is 3 greater than the
|
|
|
|
// return address for morestack.
|
2012-08-24 12:43:57 -05:00
|
|
|
let ret_offset = if boundary { 4 } else { 1 };
|
|
|
|
last_ret = *ptr::offset(frame.fp, ret_offset) as *Word;
|
|
|
|
|
|
|
|
if ptr::is_null(pc) {
|
2012-09-07 17:32:04 -05:00
|
|
|
loop;
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
|
|
|
|
2012-08-23 14:14:47 -05:00
|
|
|
let mut delay_reached_sentinel = reached_sentinel;
|
2012-08-24 12:43:57 -05:00
|
|
|
let sp = is_safe_point(pc);
|
|
|
|
match sp {
|
|
|
|
Some(sp_info) => {
|
|
|
|
for walk_safe_point(frame.fp, sp_info) |root, tydesc| {
|
|
|
|
// Skip roots until we see the sentinel.
|
|
|
|
if !reached_sentinel {
|
|
|
|
if root == sentinel {
|
|
|
|
delay_reached_sentinel = true;
|
2012-08-23 14:14:47 -05:00
|
|
|
}
|
2012-09-07 17:32:04 -05:00
|
|
|
loop;
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
2012-08-23 14:14:47 -05:00
|
|
|
|
2012-08-24 12:43:57 -05:00
|
|
|
// Skip null pointers, which can occur when a
|
|
|
|
// unique pointer has already been freed.
|
|
|
|
if ptr::is_null(*root) {
|
2012-09-07 17:32:04 -05:00
|
|
|
loop;
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
2012-08-20 12:09:22 -05:00
|
|
|
|
2012-08-24 12:43:57 -05:00
|
|
|
if ptr::is_null(tydesc) {
|
|
|
|
// Root is a generic box.
|
|
|
|
let refcount = **root;
|
|
|
|
if mem | task_local_heap != 0 && refcount != -1 {
|
|
|
|
if !visitor(root, tydesc) { return; }
|
|
|
|
} else if mem | exchange_heap != 0 && refcount == -1 {
|
|
|
|
if !visitor(root, tydesc) { return; }
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Root is a non-immediate.
|
|
|
|
if mem | stack != 0 {
|
|
|
|
if !visitor(root, tydesc) { return; }
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2012-08-24 12:43:57 -05:00
|
|
|
}
|
|
|
|
None => ()
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
2012-08-23 14:14:47 -05:00
|
|
|
reached_sentinel = delay_reached_sentinel;
|
2012-07-16 14:28:15 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-09-26 19:20:14 -05:00
|
|
|
pub fn gc() {
|
2012-07-16 14:28:15 -05:00
|
|
|
unsafe {
|
2012-09-06 23:22:50 -05:00
|
|
|
// Abort when GC is disabled.
|
|
|
|
if get_safe_point_count() == 0 {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2012-08-23 14:14:47 -05:00
|
|
|
for walk_gc_roots(task_local_heap, ptr::null()) |_root, _tydesc| {
|
2012-07-16 14:28:15 -05:00
|
|
|
// FIXME(#2997): Walk roots and mark them.
|
|
|
|
io::stdout().write([46]); // .
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-08-28 13:00:24 -05:00
|
|
|
#[cfg(gc)]
|
|
|
|
fn expect_sentinel() -> bool { true }
|
|
|
|
|
|
|
|
#[cfg(nogc)]
|
|
|
|
fn expect_sentinel() -> bool { false }
|
|
|
|
|
2012-09-06 13:25:24 -05:00
|
|
|
// Entry point for GC-based cleanup. Walks stack looking for exchange
|
|
|
|
// heap and stack allocations requiring drop, and runs all
|
|
|
|
// destructors.
|
|
|
|
//
|
2013-01-31 19:51:01 -06:00
|
|
|
// This should only be called from fail!, as it will drop the roots
|
2012-07-16 14:28:15 -05:00
|
|
|
// which are *live* on the stack, rather than dropping those that are
|
|
|
|
// dead.
|
2012-09-26 19:20:14 -05:00
|
|
|
pub fn cleanup_stack_for_failure() {
|
2012-07-16 14:28:15 -05:00
|
|
|
unsafe {
|
2012-09-06 23:22:50 -05:00
|
|
|
// Abort when GC is disabled.
|
|
|
|
if get_safe_point_count() == 0 {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2012-08-28 13:00:24 -05:00
|
|
|
// Leave a sentinel on the stack to mark the current frame. The
|
|
|
|
// stack walker will ignore any frames above the sentinel, thus
|
|
|
|
// avoiding collecting any memory being used by the stack walker
|
|
|
|
// itself.
|
|
|
|
//
|
|
|
|
// However, when core itself is not compiled with GC, then none of
|
|
|
|
// the functions in core will have GC metadata, which means we
|
|
|
|
// won't be able to find the sentinel root on the stack. In this
|
|
|
|
// case, we can safely skip the sentinel since we won't find our
|
|
|
|
// own stack roots on the stack anyway.
|
2012-08-23 14:14:47 -05:00
|
|
|
let sentinel_box = ~0;
|
2012-08-28 13:00:24 -05:00
|
|
|
let sentinel: **Word = if expect_sentinel() {
|
2012-10-01 14:47:02 -05:00
|
|
|
cast::reinterpret_cast(&ptr::addr_of(&sentinel_box))
|
2012-08-28 13:00:24 -05:00
|
|
|
} else {
|
|
|
|
ptr::null()
|
|
|
|
};
|
2012-08-23 14:14:47 -05:00
|
|
|
|
2013-01-22 20:13:06 -06:00
|
|
|
let mut roots = LinearSet::new();
|
2012-08-23 14:14:47 -05:00
|
|
|
for walk_gc_roots(need_cleanup, sentinel) |root, tydesc| {
|
2012-08-17 18:17:18 -05:00
|
|
|
// Track roots to avoid double frees.
|
2013-01-22 20:13:06 -06:00
|
|
|
if roots.contains(&*root) {
|
2012-09-07 17:32:04 -05:00
|
|
|
loop;
|
2012-08-17 18:17:18 -05:00
|
|
|
}
|
2013-01-22 20:13:06 -06:00
|
|
|
roots.insert(*root);
|
2012-08-17 18:17:18 -05:00
|
|
|
|
2012-07-16 14:28:15 -05:00
|
|
|
if ptr::is_null(tydesc) {
|
2013-01-10 17:16:15 -06:00
|
|
|
// FIXME #4420: Destroy this box
|
2013-01-04 16:48:19 -06:00
|
|
|
// FIXME #4330: Destroy this box
|
2012-07-16 14:28:15 -05:00
|
|
|
} else {
|
|
|
|
rustrt::rust_call_tydesc_glue(*root, tydesc, 3 as size_t);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|