From f6117173c9f26efdcf50cb664d006ea2bdf0d0cb Mon Sep 17 00:00:00 2001
From: Rob Arnold <robarnold@cs.cmu.edu>
Date: Tue, 5 Jul 2011 22:55:41 -0700
Subject: [PATCH] Allocate rust_ivec buffers out of the kernel pool
The duplication of upcalls is due to the fact that the runtime is
shared between stage0/rustc and stage1/rustc. Once snapshots are
updated, they should be de-duplicated.
---
src/comp/back/upcall.rs | 15 +++++++--
src/comp/middle/trans.rs | 32 ++++++++++++++----
src/lib/ivec.rs | 9 ++---
src/rt/rust_builtin.cpp | 60 +++++++++++++++++++++++++++++++++
src/rt/rust_upcall.cpp | 72 ++++++++++++++++++++++++++++++++++++++++
src/rt/rustrt.def.in | 6 ++++
6 files changed, 182 insertions(+), 12 deletions(-)
diff --git a/src/comp/back/upcall.rs b/src/comp/back/upcall.rs
index 5fad90ce031..db750fe7a6d 100644
--- a/src/comp/back/upcall.rs
+++ b/src/comp/back/upcall.rs
@@ -47,6 +47,8 @@ type upcalls =
ValueRef exit,
ValueRef malloc,
ValueRef free,
+ ValueRef shared_malloc,
+ ValueRef shared_free,
ValueRef mark,
ValueRef new_str,
ValueRef dup_str,
@@ -56,7 +58,9 @@ type upcalls =
ValueRef new_task,
ValueRef start_task,
ValueRef ivec_resize,
- ValueRef ivec_spill);
+ ValueRef ivec_spill,
+ ValueRef ivec_resize_shared,
+ ValueRef ivec_spill_shared);
fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
fn decl(type_names tn, ModuleRef llmod, str name, vec[TypeRef] tys,
@@ -97,6 +101,9 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
malloc=d("malloc", [T_size_t(), T_ptr(T_tydesc(tn))],
T_ptr(T_i8())),
free=dv("free", [T_ptr(T_i8()), T_int()]),
+ shared_malloc=d("shared_malloc",
+ [T_size_t(), T_ptr(T_tydesc(tn))], T_ptr(T_i8())),
+ shared_free=dv("shared_free", [T_ptr(T_i8())]),
mark=d("mark", [T_ptr(T_i8())], T_int()),
new_str=d("new_str", [T_ptr(T_i8()), T_size_t()],
T_ptr(T_str())),
@@ -119,7 +126,11 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
ivec_resize=d("ivec_resize", [T_ptr(T_opaque_ivec()), T_int()],
T_void()),
ivec_spill=d("ivec_spill", [T_ptr(T_opaque_ivec()), T_int()],
- T_void()));
+ T_void()),
+ ivec_resize_shared=d("ivec_resize_shared",
+ [T_ptr(T_opaque_ivec()), T_int()], T_void()),
+ ivec_spill_shared=d("ivec_spill_shared",
+ [T_ptr(T_opaque_ivec()), T_int()], T_void()));
}
//
// Local Variables:
diff --git a/src/comp/middle/trans.rs b/src/comp/middle/trans.rs
index e8139432ad2..6a7b5a412de 100644
--- a/src/comp/middle/trans.rs
+++ b/src/comp/middle/trans.rs
@@ -1150,6 +1150,12 @@ fn trans_non_gc_free(&@block_ctxt cx, ValueRef v) -> result {
ret rslt(cx, C_int(0));
}
+fn trans_shared_free(&@block_ctxt cx, ValueRef v) -> result {
+ cx.build.Call(cx.fcx.lcx.ccx.upcalls.shared_free,
+ [cx.fcx.lltaskptr, cx.build.PointerCast(v, T_ptr(T_i8()))]);
+ ret rslt(cx, C_int(0));
+}
+
fn find_scope_cx(&@block_ctxt cx) -> @block_ctxt {
if (cx.kind != NON_SCOPE_BLOCK) { ret cx; }
alt (cx.parent) {
@@ -1578,6 +1584,18 @@ fn trans_raw_malloc(&@block_ctxt cx, TypeRef llptr_ty, ValueRef llsize) ->
ret rslt(cx, cx.build.PointerCast(rval, llptr_ty));
}
+// trans_shared_malloc: expects a type indicating which pointer type we want
+// and a size indicating how much space we want malloc'd.
+fn trans_shared_malloc(&@block_ctxt cx, TypeRef llptr_ty, ValueRef llsize) ->
+ result {
+ // FIXME: need a table to collect tydesc globals.
+
+ auto tydesc = C_null(T_ptr(T_tydesc(cx.fcx.lcx.ccx.tn)));
+ auto rval =
+ cx.build.Call(cx.fcx.lcx.ccx.upcalls.shared_malloc,
+ [cx.fcx.lltaskptr, llsize, tydesc]);
+ ret rslt(cx, cx.build.PointerCast(rval, llptr_ty));
+}
// trans_malloc_boxed: expects an unboxed type and returns a pointer to enough
// space for something of that type, along with space for a reference count;
@@ -2097,7 +2115,7 @@ fn maybe_free_ivec_heap_part(&@block_ctxt cx, ValueRef v0, ty::t unit_ty) ->
auto m = maybe_on_heap_cx.build.InBoundsGEP(stub_ptr, v);
maybe_on_heap_cx.build.Load(m)
};
- auto after_free_cx = trans_non_gc_free(maybe_on_heap_cx, heap_ptr).bcx;
+ auto after_free_cx = trans_shared_free(maybe_on_heap_cx, heap_ptr).bcx;
after_free_cx.build.Br(next_cx.llbb);
ret rslt(next_cx, C_nil());
}
@@ -3617,7 +3635,8 @@ mod ivec {
{
auto p =
heap_resize_cx.build.PointerCast(v, T_ptr(T_opaque_ivec()));
- heap_resize_cx.build.Call(cx.fcx.lcx.ccx.upcalls.ivec_resize,
+ auto upcall = cx.fcx.lcx.ccx.upcalls.ivec_resize_shared;
+ heap_resize_cx.build.Call(upcall,
[cx.fcx.lltaskptr, p, new_heap_len]);
}
auto heap_ptr_resize =
@@ -3657,7 +3676,8 @@ mod ivec {
{
auto p =
stack_spill_cx.build.PointerCast(v, T_ptr(T_opaque_ivec()));
- stack_spill_cx.build.Call(cx.fcx.lcx.ccx.upcalls.ivec_spill,
+ auto upcall = cx.fcx.lcx.ccx.upcalls.ivec_spill_shared;
+ stack_spill_cx.build.Call(upcall,
[cx.fcx.lltaskptr, p, new_stack_len]);
}
auto spill_stub =
@@ -3908,7 +3928,7 @@ mod ivec {
heap_cx.build.InBoundsGEP(stub_ptr_heap,
stub_a));
auto heap_sz = heap_cx.build.Add(llsize_of(llheappartty), lllen);
- auto rs = trans_raw_malloc(heap_cx, T_ptr(llheappartty), heap_sz);
+ auto rs = trans_shared_malloc(heap_cx, T_ptr(llheappartty), heap_sz);
auto heap_part = rs.val;
heap_cx = rs.bcx;
heap_cx.build.Store(heap_part,
@@ -4045,7 +4065,7 @@ mod ivec {
auto heap_part_sz = on_heap_cx.build.Add(alen,
llsize_of(T_opaque_ivec_heap_part()));
- auto rs = trans_raw_malloc(on_heap_cx, T_ptr(llheappartty),
+ auto rs = trans_shared_malloc(on_heap_cx, T_ptr(llheappartty),
heap_part_sz);
on_heap_cx = rs.bcx;
auto new_heap_ptr = rs.val;
@@ -6003,7 +6023,7 @@ fn trans_ivec(@block_ctxt bcx, &vec[@ast::expr] args, ast::node_id id) ->
bcx.build.Store(lllen, bcx.build.InBoundsGEP(llstubptr, stub_a));
auto llheapsz = bcx.build.Add(llsize_of(llheapty), lllen);
- auto rslt = trans_raw_malloc(bcx, T_ptr(llheapty), llheapsz);
+ auto rslt = trans_shared_malloc(bcx, T_ptr(llheapty), llheapsz);
bcx = rslt.bcx;
auto llheapptr = rslt.val;
bcx.build.Store(llheapptr,
diff --git a/src/lib/ivec.rs b/src/lib/ivec.rs
index a9a5509dae9..d34b54d68e4 100644
--- a/src/lib/ivec.rs
+++ b/src/lib/ivec.rs
@@ -11,15 +11,16 @@ native "rust-intrinsic" mod rusti {
}
native "rust" mod rustrt {
- fn ivec_reserve[T](&mutable T[mutable?] v, uint n);
+ fn ivec_reserve_shared[T](&mutable T[mutable?] v, uint n);
fn ivec_on_heap[T](&T[] v) -> uint;
fn ivec_to_ptr[T](&T[] v) -> *T;
- fn ivec_copy_from_buf[T](&mutable T[mutable?] v, *T ptr, uint count);
+ fn ivec_copy_from_buf_shared[T](&mutable T[mutable?] v,
+ *T ptr, uint count);
}
/// Reserves space for `n` elements in the given vector.
fn reserve[T](&mutable T[mutable?] v, uint n) {
- rustrt::ivec_reserve(v, n);
+ rustrt::ivec_reserve_shared(v, n);
}
fn on_heap[T](&T[] v) -> bool {
@@ -204,7 +205,7 @@ fn all[T](fn(&T)->bool f, &T[] v) -> bool {
mod unsafe {
fn copy_from_buf[T](&mutable T[] v, *T ptr, uint count) {
- ret rustrt::ivec_copy_from_buf(v, ptr, count);
+ ret rustrt::ivec_copy_from_buf_shared(v, ptr, count);
}
}
diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp
index 6438c549c55..872a317ca38 100644
--- a/src/rt/rust_builtin.cpp
+++ b/src/rt/rust_builtin.cpp
@@ -648,6 +648,37 @@ ivec_reserve(rust_task *task, type_desc *ty, rust_ivec *v, size_t n_elems)
v->alloc = new_alloc;
}
+/**
+ * Preallocates the exact number of bytes in the given interior vector.
+ */
+extern "C" CDECL void
+ivec_reserve_shared(rust_task *task, type_desc *ty, rust_ivec *v,
+ size_t n_elems)
+{
+ size_t new_alloc = n_elems * ty->size;
+ if (new_alloc <= v->alloc)
+ return; // Already big enough.
+
+ rust_ivec_heap *heap_part;
+ if (v->fill || !v->payload.ptr) {
+ // On stack; spill to heap.
+ heap_part = (rust_ivec_heap *)task->kernel->malloc(new_alloc +
+ sizeof(size_t));
+ heap_part->fill = v->fill;
+ memcpy(&heap_part->data, v->payload.data, v->fill);
+
+ v->fill = 0;
+ v->payload.ptr = heap_part;
+ } else {
+ // On heap; resize.
+ heap_part = (rust_ivec_heap *)task->kernel->realloc(v->payload.ptr,
+ new_alloc + sizeof(size_t));
+ v->payload.ptr = heap_part;
+ }
+
+ v->alloc = new_alloc;
+}
+
/**
* Returns true if the given vector is on the heap and false if it's on the
* stack.
@@ -706,6 +737,35 @@ ivec_copy_from_buf(rust_task *task, type_desc *ty, rust_ivec *v, void *ptr,
v->payload.ptr->fill = new_size;
}
+/**
+ * Copies elements in an unsafe buffer to the given interior vector. The
+ * vector must have size zero.
+ */
+extern "C" CDECL void
+ivec_copy_from_buf_shared(rust_task *task, type_desc *ty, rust_ivec *v,
+ void *ptr, size_t count)
+{
+ size_t old_size = get_ivec_size(v);
+ if (old_size) {
+ task->fail(1);
+ return;
+ }
+
+ ivec_reserve_shared(task, ty, v, count);
+
+ size_t new_size = count * ty->size;
+ if (v->fill || !v->payload.ptr) {
+ // On stack.
+ memmove(v->payload.data, ptr, new_size);
+ v->fill = new_size;
+ return;
+ }
+
+ // On heap.
+ memmove(v->payload.ptr->data, ptr, new_size);
+ v->payload.ptr->fill = new_size;
+}
+
extern "C" CDECL void
pin_task(rust_task *task) {
task->pin();
diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp
index d8ff9236aa6..a89c01ebdfb 100644
--- a/src/rt/rust_upcall.cpp
+++ b/src/rt/rust_upcall.cpp
@@ -289,6 +289,36 @@ upcall_free(rust_task *task, void* ptr, uintptr_t is_gc) {
task->free(ptr, (bool) is_gc);
}
+extern "C" CDECL uintptr_t
+upcall_shared_malloc(rust_task *task, size_t nbytes, type_desc *td) {
+ LOG_UPCALL_ENTRY(task);
+ scoped_lock with(task->kernel->scheduler_lock);
+
+ LOG(task, mem,
+ "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR ")",
+ nbytes, td);
+ void *p = task->kernel->malloc(nbytes);
+ LOG(task, mem,
+ "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR
+ ") = 0x%" PRIxPTR,
+ nbytes, td, (uintptr_t)p);
+ return (uintptr_t) p;
+}
+
+/**
+ * Called whenever an object's ref count drops to zero.
+ */
+extern "C" CDECL void
+upcall_shared_free(rust_task *task, void* ptr) {
+ LOG_UPCALL_ENTRY(task);
+ scoped_lock with(task->kernel->scheduler_lock);
+ rust_scheduler *sched = task->sched;
+ DLOG(sched, mem,
+ "upcall shared_free(0x%" PRIxPTR")",
+ (uintptr_t)ptr);
+ task->kernel->free(ptr);
+}
+
extern "C" CDECL uintptr_t
upcall_mark(rust_task *task, void* ptr) {
LOG_UPCALL_ENTRY(task);
@@ -537,6 +567,7 @@ extern "C" CDECL void
upcall_ivec_resize(rust_task *task,
rust_ivec *v,
size_t newsz) {
+ LOG_UPCALL_ENTRY(task);
scoped_lock with(task->kernel->scheduler_lock);
I(task->sched, !v->fill);
@@ -556,6 +587,7 @@ extern "C" CDECL void
upcall_ivec_spill(rust_task *task,
rust_ivec *v,
size_t newsz) {
+ LOG_UPCALL_ENTRY(task);
scoped_lock with(task->kernel->scheduler_lock);
size_t new_alloc = next_power_of_two(newsz);
@@ -569,6 +601,46 @@ upcall_ivec_spill(rust_task *task,
v->payload.ptr = heap_part;
}
+/**
+ * Resizes an interior vector that has been spilled to the heap.
+ */
+extern "C" CDECL void
+upcall_ivec_resize_shared(rust_task *task,
+ rust_ivec *v,
+ size_t newsz) {
+ LOG_UPCALL_ENTRY(task);
+ scoped_lock with(task->kernel->scheduler_lock);
+ I(task->sched, !v->fill);
+
+ size_t new_alloc = next_power_of_two(newsz);
+ rust_ivec_heap *new_heap_part = (rust_ivec_heap *)
+ task->kernel->realloc(v->payload.ptr, new_alloc + sizeof(size_t));
+
+ new_heap_part->fill = newsz;
+ v->alloc = new_alloc;
+ v->payload.ptr = new_heap_part;
+}
+
+/**
+ * Spills an interior vector to the heap.
+ */
+extern "C" CDECL void
+upcall_ivec_spill_shared(rust_task *task,
+ rust_ivec *v,
+ size_t newsz) {
+ LOG_UPCALL_ENTRY(task);
+ scoped_lock with(task->kernel->scheduler_lock);
+ size_t new_alloc = next_power_of_two(newsz);
+
+ rust_ivec_heap *heap_part = (rust_ivec_heap *)
+ task->kernel->malloc(new_alloc + sizeof(size_t));
+ heap_part->fill = newsz;
+ memcpy(&heap_part->data, v->payload.data, v->fill);
+
+ v->fill = 0;
+ v->alloc = new_alloc;
+ v->payload.ptr = heap_part;
+}
//
// Local Variables:
// mode: C++
diff --git a/src/rt/rustrt.def.in b/src/rt/rustrt.def.in
index 444faccef26..481936eee69 100644
--- a/src/rt/rustrt.def.in
+++ b/src/rt/rustrt.def.in
@@ -11,8 +11,10 @@ debug_tydesc
do_gc
get_time
ivec_copy_from_buf
+ivec_copy_from_buf_shared
ivec_on_heap
ivec_reserve
+ivec_reserve_shared
ivec_to_ptr
last_os_error
nano_time
@@ -59,7 +61,9 @@ upcall_free
upcall_get_type_desc
upcall_grow_task
upcall_ivec_resize
+upcall_ivec_resize_shared
upcall_ivec_spill
+upcall_ivec_spill_shared
upcall_kill
upcall_log_double
upcall_log_float
@@ -74,6 +78,8 @@ upcall_new_task
upcall_new_vec
upcall_recv
upcall_send
+upcall_shared_malloc
+upcall_shared_free
upcall_sleep
upcall_start_task
upcall_trace_str