diff --git a/src/libcore/private.rs b/src/libcore/private.rs index a54db3fa759..6addb106de3 100644 --- a/src/libcore/private.rs +++ b/src/libcore/private.rs @@ -5,7 +5,6 @@ #[doc(hidden)]; -use compare_and_swap = rustrt::rust_compare_and_swap_ptr; use task::TaskBuilder; use task::atomically; @@ -14,14 +13,6 @@ extern mod rustrt { fn rust_task_weaken(ch: rust_port_id); fn rust_task_unweaken(ch: rust_port_id); - #[rust_stack] - fn rust_atomic_increment(p: &mut libc::intptr_t) - -> libc::intptr_t; - - #[rust_stack] - fn rust_atomic_decrement(p: &mut libc::intptr_t) - -> libc::intptr_t; - #[rust_stack] fn rust_compare_and_swap_ptr(address: &mut libc::uintptr_t, oldval: libc::uintptr_t, @@ -33,11 +24,36 @@ extern mod rustrt { fn rust_unlock_little_lock(lock: rust_little_lock); } +#[abi = "rust-intrinsic"] +extern mod rusti { + + #[cfg(stage1)] #[cfg(stage2)] #[cfg(stage3)] + fn atomic_cxchg(dst: &mut int, old: int, src: int) -> int; + fn atomic_xadd(dst: &mut int, src: int) -> int; + fn atomic_xsub(dst: &mut int, src: int) -> int; +} + #[allow(non_camel_case_types)] // runtime type type rust_port_id = uint; type GlobalPtr = *libc::uintptr_t; +// TODO: Remove once snapshots have atomic_cxchg +#[cfg(stage0)] +fn compare_and_swap(address: &mut libc::uintptr_t, + oldval: libc::uintptr_t, + newval: libc::uintptr_t) -> bool { + rustrt::rust_compare_and_swap_ptr(address, oldval, newval) +} + +#[cfg(stage1)] +#[cfg(stage2)] +#[cfg(stage3)] +fn compare_and_swap(address: &mut int, oldval: int, newval: int) -> bool { + let old = rusti::atomic_cxchg(address, oldval, newval); + old == oldval +} + /** * Atomically gets a channel from a pointer to a pointer-sized memory location * or, if no channel exists creates and installs a new channel and sets up a @@ -85,7 +101,7 @@ pub unsafe fn chan_from_global_ptr( log(debug,~"BEFORE COMPARE AND SWAP"); let swapped = compare_and_swap( cast::reinterpret_cast(&global), - 0u, cast::reinterpret_cast(&ch)); + 0, cast::reinterpret_cast(&ch)); log(debug,fmt!("AFTER .. swapped? %?", swapped)); if swapped { @@ -305,7 +321,7 @@ struct ArcDestruct { } do task::unkillable { let data: ~ArcData = cast::reinterpret_cast(&self.data); - let new_count = rustrt::rust_atomic_decrement(&mut data.count); + let new_count = rusti::atomic_xsub(&mut data.count, 1) - 1; assert new_count >= 0; if new_count == 0 { // Were we really last, or should we hand off to an unwrapper? @@ -373,8 +389,8 @@ pub unsafe fn unwrap_shared_mutable_state(rc: SharedMutableState) // Got in. Step 0: Tell destructor not to run. We are now it. rc.data = ptr::null(); // Step 1 - drop our own reference. - let new_count = rustrt::rust_atomic_decrement(&mut ptr.count); - // assert new_count >= 0; + let new_count = rusti::atomic_xsub(&mut ptr.count, 1) - 1; + //assert new_count >= 0; if new_count == 0 { // We were the last owner. Can unwrap immediately. // Also we have to free the server endpoints. @@ -452,7 +468,7 @@ pub unsafe fn clone_shared_mutable_state(rc: &SharedMutableState) -> SharedMutableState { unsafe { let ptr: ~ArcData = cast::reinterpret_cast(&(*rc).data); - let new_count = rustrt::rust_atomic_increment(&mut ptr.count); + let new_count = rusti::atomic_xadd(&mut ptr.count, 1) + 1; assert new_count >= 2; cast::forget(move ptr); } diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp index 5baa95c7323..67281cbee5a 100644 --- a/src/rt/rust_builtin.cpp +++ b/src/rt/rust_builtin.cpp @@ -830,16 +830,6 @@ rust_compare_and_swap_ptr(intptr_t *address, return sync::compare_and_swap(address, oldval, newval); } -extern "C" CDECL intptr_t -rust_atomic_increment(intptr_t *address) { - return sync::increment(address); -} - -extern "C" CDECL intptr_t -rust_atomic_decrement(intptr_t *address) { - return sync::decrement(address); -} - extern "C" CDECL void rust_task_weaken(rust_port_id chan) { rust_task *task = rust_get_current_task(); diff --git a/src/rt/rustrt.def.in b/src/rt/rustrt.def.in index 6a2bdd622cb..3760c9ff09f 100644 --- a/src/rt/rustrt.def.in +++ b/src/rt/rustrt.def.in @@ -178,8 +178,6 @@ rust_dbg_do_nothing rust_dbg_breakpoint rust_osmain_sched_id rust_compare_and_swap_ptr -rust_atomic_increment -rust_atomic_decrement rust_global_env_chan_ptr rust_port_take rust_port_drop @@ -207,4 +205,4 @@ rust_gc_metadata rust_uv_ip4_port rust_uv_ip6_port rust_uv_tcp_getpeername -rust_uv_tcp_getpeername6 \ No newline at end of file +rust_uv_tcp_getpeername6 diff --git a/src/rustc/lib/llvm.rs b/src/rustc/lib/llvm.rs index 0d92c19b952..f1397006b16 100644 --- a/src/rustc/lib/llvm.rs +++ b/src/rustc/lib/llvm.rs @@ -843,6 +843,9 @@ extern mod llvm { Name: *c_char) -> ValueRef; /* Atomic Operations */ + fn LLVMBuildAtomicCmpXchg(B: BuilderRef, LHS: ValueRef, + CMP: ValueRef, RHS: ValueRef, + ++Order: AtomicOrdering) -> ValueRef; fn LLVMBuildAtomicRMW(B: BuilderRef, ++Op: AtomicBinOp, LHS: ValueRef, RHS: ValueRef, ++Order: AtomicOrdering) -> ValueRef; diff --git a/src/rustc/middle/trans/build.rs b/src/rustc/middle/trans/build.rs index ea992600ae1..f7690b7bc93 100644 --- a/src/rustc/middle/trans/build.rs +++ b/src/rustc/middle/trans/build.rs @@ -813,6 +813,11 @@ fn Resume(cx: block, Exn: ValueRef) -> ValueRef { } // Atomic Operations +fn AtomicCmpXchg(cx: block, dst: ValueRef, + cmp: ValueRef, src: ValueRef, + order: AtomicOrdering) -> ValueRef { + llvm::LLVMBuildAtomicCmpXchg(B(cx), dst, cmp, src, order) +} fn AtomicRMW(cx: block, op: AtomicBinOp, dst: ValueRef, src: ValueRef, order: AtomicOrdering) -> ValueRef { diff --git a/src/rustc/middle/trans/foreign.rs b/src/rustc/middle/trans/foreign.rs index d307fcb5dca..8fa23ef8fab 100644 --- a/src/rustc/middle/trans/foreign.rs +++ b/src/rustc/middle/trans/foreign.rs @@ -799,6 +799,30 @@ fn trans_intrinsic(ccx: @crate_ctxt, decl: ValueRef, item: @ast::foreign_item, Some(substs), Some(item.span)); let mut bcx = top_scope_block(fcx, None), lltop = bcx.llbb; match ccx.sess.str_of(item.ident) { + ~"atomic_cxchg" => { + let old = AtomicCmpXchg(bcx, + get_param(decl, first_real_arg), + get_param(decl, first_real_arg + 1u), + get_param(decl, first_real_arg + 2u), + SequentiallyConsistent); + Store(bcx, old, fcx.llretptr); + } + ~"atomic_cxchg_acq" => { + let old = AtomicCmpXchg(bcx, + get_param(decl, first_real_arg), + get_param(decl, first_real_arg + 1u), + get_param(decl, first_real_arg + 2u), + Acquire); + Store(bcx, old, fcx.llretptr); + } + ~"atomic_cxchg_rel" => { + let old = AtomicCmpXchg(bcx, + get_param(decl, first_real_arg), + get_param(decl, first_real_arg + 1u), + get_param(decl, first_real_arg + 2u), + Release); + Store(bcx, old, fcx.llretptr); + } ~"atomic_xchg" => { let old = AtomicRMW(bcx, Xchg, get_param(decl, first_real_arg), diff --git a/src/rustc/middle/trans/type_use.rs b/src/rustc/middle/trans/type_use.rs index 8b2efacd4d1..ddd50d47c08 100644 --- a/src/rustc/middle/trans/type_use.rs +++ b/src/rustc/middle/trans/type_use.rs @@ -98,11 +98,12 @@ fn type_uses_for(ccx: @crate_ctxt, fn_id: def_id, n_tps: uint) ~"get_tydesc" | ~"needs_drop" => use_tydesc, - ~"atomic_xchg" | ~"atomic_xadd" | - ~"atomic_xsub" | ~"atomic_xchg_acq" | - ~"atomic_xadd_acq" | ~"atomic_xsub_acq" | - ~"atomic_xchg_rel" | ~"atomic_xadd_rel" | - ~"atomic_xsub_rel" => 0, + ~"atomic_cxchg" | ~"atomic_cxchg_acq"| + ~"atomic_cxchg_rel"| ~"atomic_xchg" | + ~"atomic_xadd" | ~"atomic_xsub" | + ~"atomic_xchg_acq" | ~"atomic_xadd_acq" | + ~"atomic_xsub_acq" | ~"atomic_xchg_rel" | + ~"atomic_xadd_rel" | ~"atomic_xsub_rel" => 0, ~"visit_tydesc" | ~"forget" | ~"addr_of" | ~"frame_address" | ~"morestack_addr" => 0, diff --git a/src/rustc/middle/typeck/check.rs b/src/rustc/middle/typeck/check.rs index 266fd1ba4ea..8d65cb9fdac 100644 --- a/src/rustc/middle/typeck/check.rs +++ b/src/rustc/middle/typeck/check.rs @@ -2638,7 +2638,15 @@ fn check_intrinsic_type(ccx: @crate_ctxt, it: @ast::foreign_item) { } ~"needs_drop" => (1u, ~[], ty::mk_bool(tcx)), - ~"atomic_xchg" | ~"atomic_xadd" | ~"atomic_xsub" | + ~"atomic_cxchg" | ~"atomic_cxchg_acq"| ~"atomic_cxchg_rel" => { + (0u, ~[arg(ast::by_copy, + ty::mk_mut_rptr(tcx, ty::re_bound(ty::br_anon(0)), + ty::mk_int(tcx))), + arg(ast::by_copy, ty::mk_int(tcx)), + arg(ast::by_copy, ty::mk_int(tcx))], + ty::mk_int(tcx)) + } + ~"atomic_xchg" | ~"atomic_xadd" | ~"atomic_xsub" | ~"atomic_xchg_acq" | ~"atomic_xadd_acq" | ~"atomic_xsub_acq" | ~"atomic_xchg_rel" | ~"atomic_xadd_rel" | ~"atomic_xsub_rel" => { (0u, ~[arg(ast::by_copy, diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index 498a4e137f0..39a707ad320 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -482,6 +482,14 @@ extern "C" LLVMTypeRef LLVMMetadataType(void) { return LLVMMetadataTypeInContext(LLVMGetGlobalContext()); } +extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, + LLVMValueRef target, + LLVMValueRef old, + LLVMValueRef source, + AtomicOrdering order) { + return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(target), unwrap(old), + unwrap(source), order)); +} extern "C" LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B, AtomicRMWInst::BinOp op, LLVMValueRef target, diff --git a/src/rustllvm/rustllvm.def.in b/src/rustllvm/rustllvm.def.in index 36833e5175e..44636f4f36b 100644 --- a/src/rustllvm/rustllvm.def.in +++ b/src/rustllvm/rustllvm.def.in @@ -84,6 +84,7 @@ LLVMArrayType LLVMBasicBlockAsValue LLVMBlockAddress LLVMBuildAShr +LLVMBuildAtomicCmpXchg LLVMBuildAtomicRMW LLVMBuildAdd LLVMBuildAggregateRet diff --git a/src/test/auxiliary/cci_intrinsic.rs b/src/test/auxiliary/cci_intrinsic.rs index 1cde1a049fe..f3ea9fd531c 100644 --- a/src/test/auxiliary/cci_intrinsic.rs +++ b/src/test/auxiliary/cci_intrinsic.rs @@ -2,6 +2,10 @@ #[abi = "rust-intrinsic"] extern mod rusti { #[legacy_exports]; + fn atomic_cxchg(dst: &mut int, old: int, src: int) -> int; + fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int; + fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int; + fn atomic_xchg(dst: &mut int, src: int) -> int; fn atomic_xchg_acq(dst: &mut int, src: int) -> int; fn atomic_xchg_rel(dst: &mut int, src: int) -> int; diff --git a/src/test/run-pass/intrinsic-atomics.rs b/src/test/run-pass/intrinsic-atomics.rs index 2629afa4909..8438ecf383e 100644 --- a/src/test/run-pass/intrinsic-atomics.rs +++ b/src/test/run-pass/intrinsic-atomics.rs @@ -1,6 +1,10 @@ #[abi = "rust-intrinsic"] extern mod rusti { #[legacy_exports]; + fn atomic_cxchg(dst: &mut int, old: int, src: int) -> int; + fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int; + fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int; + fn atomic_xchg(dst: &mut int, src: int) -> int; fn atomic_xchg_acq(dst: &mut int, src: int) -> int; fn atomic_xchg_rel(dst: &mut int, src: int) -> int; @@ -17,6 +21,15 @@ extern mod rusti { fn main() { let x = ~mut 1; + assert rusti::atomic_cxchg(x, 1, 2) == 1; + assert *x == 2; + + assert rusti::atomic_cxchg_acq(x, 1, 3) == 2; + assert *x == 2; + + assert rusti::atomic_cxchg_rel(x, 2, 1) == 2; + assert *x == 1; + assert rusti::atomic_xchg(x, 0) == 1; assert *x == 0;