mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #34860 - jseyfried:encapsulate_hygiene, r=nrc

Browse Source 
Clean up and encapsulate `syntax::ext::mtwt`, rename `mtwt` to `hygiene`

r? @nrc
This commit is contained in:
bors 2016-07-17 22:12:59 -07:00 committed by GitHub
commit 6cc49e51de
11 changed files with 156 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/librustc/session/config.rs
View File

@ -735,8 +735,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
"for every macro invocation, print its name and arguments"),
enable_nonzeroing_move_hints: bool = (false, parse_bool,
"force nonzeroing move optimization on"),
keep_mtwt_tables: bool = (false, parse_bool,
"don't clear the resolution tables after analysis"),
keep_hygiene_data: bool = (false, parse_bool,
"don't clear the hygiene data after analysis"),
keep_ast: bool = (false, parse_bool,
"keep the AST after lowering it to HIR"),
show_span: Option<String> = (None, parse_opt_string,

15
src/librustc_driver/driver.rs
View File

@ -236,8 +236,8 @@ pub fn compile_input(sess: &Session,
Ok(())
}
fn keep_mtwt_tables(sess: &Session) -> bool {
sess.opts.debugging_opts.keep_mtwt_tables
fn keep_hygiene_data(sess: &Session) -> bool {
sess.opts.debugging_opts.keep_hygiene_data
}
fn keep_ast(sess: &Session) -> bool {
@ -479,9 +479,8 @@ pub fn phase_1_parse_input<'a>(sess: &'a Session,
input: &Input)
-> PResult<'a, ast::Crate> {
// These may be left in an incoherent state after a previous compile.
// `clear_tables` and `clear_ident_interner` can be used to free
// memory, but they do not restore the initial state.
syntax::ext::mtwt::reset_tables();
syntax::ext::hygiene::reset_hygiene_data();
// `clear_ident_interner` can be used to free memory, but it does not restore the initial state.
token::reset_ident_interner();
let continue_after_error = sess.opts.continue_parse_after_error;
sess.diagnostic().set_continue_after_error(continue_after_error);
@ -761,9 +760,9 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
hir_map::Forest::new(lower_crate(sess, &krate, &mut resolver), &sess.dep_graph)
});
// Discard MTWT tables that aren't required past lowering to HIR.
if !keep_mtwt_tables(sess) {
syntax::ext::mtwt::clear_tables();
// Discard hygiene data, which isn't required past lowering to HIR.
if !keep_hygiene_data(sess) {
syntax::ext::hygiene::reset_hygiene_data();
}
Ok(ExpansionResult {

2
src/librustc_driver/pretty.rs
View File

@ -456,7 +456,7 @@ impl<'ast> pprust::PpAnn for HygieneAnnotation<'ast> {
pp::space(&mut s.s)?;
// FIXME #16420: this doesn't display the connections
// between syntax contexts
s.synth_comment(format!("{}#{}", nm, ctxt.0))
s.synth_comment(format!("{}{:?}", nm, ctxt))
}
pprust::NodeName(&ast::Name(nm)) => {
pp::space(&mut s.s)?;

6
src/librustc_resolve/assign_ids.rs
View File

@ -11,7 +11,7 @@
use Resolver;
use rustc::session::Session;
use syntax::ast;
use syntax::ext::mtwt;
use syntax::ext::hygiene::Mark;
use syntax::fold::{self, Folder};
use syntax::ptr::P;
use syntax::util::move_map::MoveMap;
@ -31,7 +31,7 @@ impl<'a> Resolver<'a> {
struct NodeIdAssigner<'a> {
sess: &'a Session,
macros_at_scope: &'a mut HashMap<ast::NodeId, Vec<ast::Mrk>>,
macros_at_scope: &'a mut HashMap<ast::NodeId, Vec<Mark>>,
}
impl<'a> Folder for NodeIdAssigner<'a> {
@ -49,7 +49,7 @@ impl<'a> Folder for NodeIdAssigner<'a> {
block.stmts = block.stmts.move_flat_map(|stmt| {
if let ast::StmtKind::Item(ref item) = stmt.node {
if let ast::ItemKind::Mac(..) = item.node {
macros.push(mtwt::outer_mark(item.ident.ctxt));
macros.push(item.ident.ctxt.data().outer_mark);
return None;
}
}

20
src/librustc_resolve/lib.rs
View File

@ -53,7 +53,7 @@ use rustc::ty::subst::{ParamSpace, FnSpace, TypeSpace};
use rustc::hir::{Freevar, FreevarMap, TraitCandidate, TraitMap, GlobMap};
use rustc::util::nodemap::{NodeMap, NodeSet, FnvHashMap, FnvHashSet};
use syntax::ext::mtwt;
use syntax::ext::hygiene::Mark;
use syntax::ast::{self, FloatTy};
use syntax::ast::{CRATE_NODE_ID, Name, NodeId, CrateNum, IntTy, UintTy};
use syntax::parse::token::{self, keywords};
@ -654,7 +654,7 @@ enum RibKind<'a> {
ModuleRibKind(Module<'a>),
// We passed through a `macro_rules!` statement with the given expansion
MacroDefinition(ast::Mrk),
MacroDefinition(Mark),
}
#[derive(Copy, Clone)]
@ -933,7 +933,7 @@ pub struct Resolver<'a> {
// Maps the node id of a statement to the expansions of the `macro_rules!`s
// immediately above the statement (if appropriate).
macros_at_scope: HashMap<NodeId, Vec<ast::Mrk>>,
macros_at_scope: HashMap<NodeId, Vec<Mark>>,
graph_root: Module<'a>,
@ -1434,10 +1434,9 @@ impl<'a> Resolver<'a> {
if let MacroDefinition(mac) = self.get_ribs(ns)[i].kind {
// If an invocation of this macro created `ident`, give up on `ident`
// and switch to `ident`'s source from the macro definition.
if let Some((source_ident, source_macro)) = mtwt::source(ident) {
if mac == source_macro {
ident = source_ident;
}
let (source_ctxt, source_macro) = ident.ctxt.source();
if source_macro == mac {
ident.ctxt = source_ctxt;
}
}
}
@ -1585,10 +1584,9 @@ impl<'a> Resolver<'a> {
MacroDefinition(mac) => {
// If an invocation of this macro created `ident`, give up on `ident`
// and switch to `ident`'s source from the macro definition.
if let Some((source_ident, source_macro)) = mtwt::source(ident) {
if mac == source_macro {
ident = source_ident;
}
let (source_ctxt, source_macro) = ident.ctxt.source();
if source_macro == mac {
ident.ctxt = source_ctxt;
}
}
_ => {

22
src/libsyntax/ast.rs
View File

@ -19,6 +19,7 @@ pub use util::ThinVec;
use syntax_pos::{mk_sp, Span, DUMMY_SP, ExpnId};
use codemap::{respan, Spanned};
use abi::Abi;
use ext::hygiene::SyntaxContext;
use parse::token::{self, keywords, InternedString};
use print::pprust;
use ptr::P;
@ -33,15 +34,6 @@ use serialize::{Encodable, Decodable, Encoder, Decoder};
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Name(pub u32);
/// A SyntaxContext represents a chain of macro-expandings
/// and renamings. Each macro expansion corresponds to
/// a fresh u32. This u32 is a reference to a table stored
/// in thread-local storage.
/// The special value EMPTY_CTXT is used to indicate an empty
/// syntax context.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
pub struct SyntaxContext(pub u32);
/// An identifier contains a Name (index into the interner
/// table) and a SyntaxContext to track renaming and
/// macro expansion per Flatt et al., "Macros That Work Together"
@ -81,20 +73,15 @@ impl Decodable for Name {
}
}
pub const EMPTY_CTXT : SyntaxContext = SyntaxContext(0);
impl Ident {
pub fn new(name: Name, ctxt: SyntaxContext) -> Ident {
Ident {name: name, ctxt: ctxt}
}
pub const fn with_empty_ctxt(name: Name) -> Ident {
Ident {name: name, ctxt: EMPTY_CTXT}
Ident { name: name, ctxt: SyntaxContext::empty() }
}
}
impl fmt::Debug for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}#{}", self.name, self.ctxt.0)
write!(f, "{}{:?}", self.name, self.ctxt)
}
}
@ -116,9 +103,6 @@ impl Decodable for Ident {
}
}
/// A mark represents a unique id associated with a macro expansion
pub type Mrk = u32;
#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy)]
pub struct Lifetime {
pub id: NodeId,

42
src/libsyntax/ext/expand.rs
View File

@ -9,20 +9,19 @@
// except according to those terms.
use ast::{Block, Crate, Ident, Mac_, Name, PatKind};
use ast::{MacStmtStyle, Mrk, Stmt, StmtKind, ItemKind};
use ast::{MacStmtStyle, Stmt, StmtKind, ItemKind};
use ast;
use attr::HasAttrs;
use ext::mtwt;
use attr;
use ext::hygiene::Mark;
use attr::{self, HasAttrs};
use attr::AttrMetaMethods;
use codemap::{dummy_spanned, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
use codemap::{dummy_spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
use syntax_pos::{self, Span, ExpnId};
use config::StripUnconfigured;
use ext::base::*;
use feature_gate::{self, Features};
use fold;
use fold::*;
use parse::token::{fresh_mark, intern, keywords};
use parse::token::{intern, keywords};
use ptr::P;
use tokenstream::TokenTree;
use util::small_vector::SmallVector;
@ -130,9 +129,9 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
// It would almost certainly be cleaner to pass the whole macro invocation in,
// rather than pulling it apart and marking the tts and the ctxt separately.
let Mac_ { path, tts, .. } = mac.node;
let mark = fresh_mark();
let mark = Mark::fresh();
fn mac_result<'a>(path: &ast::Path, ident: Option<Ident>, tts: Vec<TokenTree>, mark: Mrk,
fn mac_result<'a>(path: &ast::Path, ident: Option<Ident>, tts: Vec<TokenTree>, mark: Mark,
attrs: Vec<ast::Attribute>, call_site: Span, fld: &'a mut MacroExpander)
-> Option<Box<MacResult + 'a>> {
// Detect use of feature-gated or invalid attributes on macro invoations
@ -708,30 +707,17 @@ pub fn expand_crate(mut cx: ExtCtxt,
return (ret, cx.syntax_env.names);
}
// HYGIENIC CONTEXT EXTENSION:
// all of these functions are for walking over
// ASTs and making some change to the context of every
// element that has one. a CtxtFn is a trait-ified
// version of a closure in (SyntaxContext -> SyntaxContext).
// the ones defined here include:
// Marker - add a mark to a context
// A Marker adds the given mark to the syntax context and
// sets spans' `expn_id` to the given expn_id (unless it is `None`).
struct Marker { mark: Mrk, expn_id: Option<ExpnId> }
struct Marker { mark: Mark, expn_id: Option<ExpnId> }
impl Folder for Marker {
fn fold_ident(&mut self, id: Ident) -> Ident {
ast::Ident::new(id.name, mtwt::apply_mark(self.mark, id.ctxt))
fn fold_ident(&mut self, mut ident: Ident) -> Ident {
ident.ctxt = ident.ctxt.apply_mark(self.mark);
ident
}
fn fold_mac(&mut self, Spanned {node, span}: ast::Mac) -> ast::Mac {
Spanned {
node: Mac_ {
path: self.fold_path(node.path),
tts: self.fold_tts(&node.tts),
},
span: self.new_span(span),
}
fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
noop_fold_mac(mac, self)
}
fn new_span(&mut self, mut span: Span) -> Span {
@ -743,7 +729,7 @@ impl Folder for Marker {
}
// apply a given mark to the given token trees. Used prior to expansion of a macro.
fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec<TokenTree> {
fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> {
noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
}

116
src/libsyntax/ext/hygiene.rs Normal file
View File

@ -0,0 +1,116 @@
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Machinery for hygienic macros, inspired by the MTWT[1] paper.
//!
//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
use std::cell::RefCell;
use std::collections::HashMap;
use std::fmt;
/// A SyntaxContext represents a chain of macro expansions (represented by marks).
#[derive(Clone, Copy, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Default)]
pub struct SyntaxContext(u32);
#[derive(Copy, Clone)]
pub struct SyntaxContextData {
pub outer_mark: Mark,
pub prev_ctxt: SyntaxContext,
}
/// A mark represents a unique id associated with a macro expansion.
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default)]
pub struct Mark(u32);
impl Mark {
pub fn fresh() -> Self {
HygieneData::with(|data| {
let next_mark = Mark(data.next_mark.0 + 1);
::std::mem::replace(&mut data.next_mark, next_mark)
})
}
}
struct HygieneData {
syntax_contexts: Vec<SyntaxContextData>,
markings: HashMap<(SyntaxContext, Mark), SyntaxContext>,
next_mark: Mark,
}
impl HygieneData {
fn new() -> Self {
HygieneData {
syntax_contexts: vec![SyntaxContextData {
outer_mark: Mark(0), // the null mark
prev_ctxt: SyntaxContext(0), // the empty context
}],
markings: HashMap::new(),
next_mark: Mark(1),
}
}
fn with<T, F: FnOnce(&mut HygieneData) -> T>(f: F) -> T {
thread_local! {
static HYGIENE_DATA: RefCell<HygieneData> = RefCell::new(HygieneData::new());
}
HYGIENE_DATA.with(|data| f(&mut *data.borrow_mut()))
}
}
pub fn reset_hygiene_data() {
HygieneData::with(|data| *data = HygieneData::new())
}
impl SyntaxContext {
pub const fn empty() -> Self {
SyntaxContext(0)
}
pub fn data(self) -> SyntaxContextData {
HygieneData::with(|data| data.syntax_contexts[self.0 as usize])
}
/// Extend a syntax context with a given mark
pub fn apply_mark(self, mark: Mark) -> SyntaxContext {
// Applying the same mark twice is a no-op
let ctxt_data = self.data();
if mark == ctxt_data.outer_mark {
return ctxt_data.prev_ctxt;
}
HygieneData::with(|data| {
let syntax_contexts = &mut data.syntax_contexts;
*data.markings.entry((self, mark)).or_insert_with(|| {
syntax_contexts.push(SyntaxContextData {
outer_mark: mark,
prev_ctxt: self,
});
SyntaxContext(syntax_contexts.len() as u32 - 1)
})
})
}
/// If `ident` is macro expanded, return the source ident from the macro definition
/// and the mark of the expansion that created the macro definition.
pub fn source(self) -> (Self /* source context */, Mark /* source macro */) {
let macro_def_ctxt = self.data().prev_ctxt.data();
(macro_def_ctxt.prev_ctxt, macro_def_ctxt.outer_mark)
}
}
impl fmt::Debug for SyntaxContext {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "#{}", self.0)
}
}

154
src/libsyntax/ext/mtwt.rs
View File

@ -1,154 +0,0 @@
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Machinery for hygienic macros, as described in the MTWT[1] paper.
//!
//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
pub use self::SyntaxContext_::*;
use ast::{Ident, Mrk, SyntaxContext};
use std::cell::RefCell;
use std::collections::HashMap;
/// The SCTable contains a table of SyntaxContext_'s. It
/// represents a flattened tree structure, to avoid having
/// managed pointers everywhere (that caused an ICE).
/// The `marks` ensures that adding the same mark to the
/// same context gives you back the same context as before.
pub struct SCTable {
table: RefCell<Vec<SyntaxContext_>>,
marks: RefCell<HashMap<(SyntaxContext,Mrk),SyntaxContext>>,
}
#[derive(PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy, Clone)]
pub enum SyntaxContext_ {
EmptyCtxt,
Mark (Mrk,SyntaxContext),
}
/// Extend a syntax context with a given mark
pub fn apply_mark(m: Mrk, ctxt: SyntaxContext) -> SyntaxContext {
with_sctable(|table| apply_mark_internal(m, ctxt, table))
}
/// Extend a syntax context with a given mark and sctable (explicit memoization)
fn apply_mark_internal(m: Mrk, ctxt: SyntaxContext, table: &SCTable) -> SyntaxContext {
let ctxts = &mut *table.table.borrow_mut();
match ctxts[ctxt.0 as usize] {
// Applying the same mark twice is a no-op.
Mark(outer_mark, prev_ctxt) if outer_mark == m => return prev_ctxt,
_ => *table.marks.borrow_mut().entry((ctxt, m)).or_insert_with(|| {
SyntaxContext(idx_push(ctxts, Mark(m, ctxt)))
}),
}
}
/// Fetch the SCTable from TLS, create one if it doesn't yet exist.
pub fn with_sctable<T, F>(op: F) -> T where
F: FnOnce(&SCTable) -> T,
{
thread_local!(static SCTABLE_KEY: SCTable = new_sctable_internal());
SCTABLE_KEY.with(move |slot| op(slot))
}
// Make a fresh syntax context table with EmptyCtxt in slot zero.
fn new_sctable_internal() -> SCTable {
SCTable {
table: RefCell::new(vec![EmptyCtxt]),
marks: RefCell::new(HashMap::new()),
}
}
/// Clear the tables from TLD to reclaim memory.
pub fn clear_tables() {
with_sctable(|table| {
*table.table.borrow_mut() = Vec::new();
*table.marks.borrow_mut() = HashMap::new();
});
}
/// Reset the tables to their initial state
pub fn reset_tables() {
with_sctable(|table| {
*table.table.borrow_mut() = vec![EmptyCtxt];
*table.marks.borrow_mut() = HashMap::new();
});
}
/// Add a value to the end of a vec, return its index
fn idx_push<T>(vec: &mut Vec<T>, val: T) -> u32 {
vec.push(val);
(vec.len() - 1) as u32
}
/// Return the outer mark for a context with a mark at the outside.
/// FAILS when outside is not a mark.
pub fn outer_mark(ctxt: SyntaxContext) -> Mrk {
with_sctable(|sctable| {
match (*sctable.table.borrow())[ctxt.0 as usize] {
Mark(mrk, _) => mrk,
_ => panic!("can't retrieve outer mark when outside is not a mark")
}
})
}
/// If `ident` is macro expanded, return the source ident from the macro definition
/// and the mark of the expansion that created the macro definition.
pub fn source(ident: Ident) -> Option<(Ident /* source ident */, Mrk /* source macro */)> {
with_sctable(|sctable| {
let ctxts = sctable.table.borrow();
if let Mark(_expansion_mark, macro_ctxt) = ctxts[ident.ctxt.0 as usize] {
if let Mark(definition_mark, orig_ctxt) = ctxts[macro_ctxt.0 as usize] {
return Some((Ident::new(ident.name, orig_ctxt), definition_mark));
}
}
None
})
}
#[cfg(test)]
mod tests {
use ast::{EMPTY_CTXT, Mrk, SyntaxContext};
use super::{apply_mark_internal, new_sctable_internal, Mark, SCTable};
// extend a syntax context with a sequence of marks given
// in a vector. v[0] will be the outermost mark.
fn unfold_marks(mrks: Vec<Mrk> , tail: SyntaxContext, table: &SCTable)
-> SyntaxContext {
mrks.iter().rev().fold(tail, |tail:SyntaxContext, mrk:&Mrk|
{apply_mark_internal(*mrk,tail,table)})
}
#[test] fn unfold_marks_test() {
let mut t = new_sctable_internal();
assert_eq!(unfold_marks(vec!(3,7),EMPTY_CTXT,&mut t),SyntaxContext(2));
{
let table = t.table.borrow();
assert!((*table)[1] == Mark(7,EMPTY_CTXT));
assert!((*table)[2] == Mark(3,SyntaxContext(1)));
}
}
#[test]
fn hashing_tests () {
let mut t = new_sctable_internal();
assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
assert_eq!(apply_mark_internal(13,EMPTY_CTXT,&mut t),SyntaxContext(2));
// using the same one again should result in the same index:
assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
// I'm assuming that the rename table will behave the same....
}
}

2
src/libsyntax/lib.rs
View File

@ -127,7 +127,7 @@ pub mod ext {
pub mod base;
pub mod build;
pub mod expand;
pub mod mtwt;
pub mod hygiene;
pub mod quote;
pub mod source_util;

5
src/libsyntax/parse/token.rs
View File

@ -633,8 +633,3 @@ pub fn fresh_name(src: ast::Ident) -> ast::Name {
/*let num = rand::thread_rng().gen_uint_range(0,0xffff);
gensym(format!("{}_{}",ident_to_string(src),num))*/
}
// create a fresh mark.
pub fn fresh_mark() -> ast::Mrk {
gensym("mark").0
}