Auto merge of #118958 - c410-f3r:concat-again, r=petrochenkov

Add a new concat metavar expr

Revival of #111930

Giving it another try now that #117050 was merged.

With the new rules, meta-variable expressions must be referenced with a dollar sign (`$`) and this can cause misunderstands with `$concat`.

```rust
macro_rules! foo {
    ( $bar:ident ) => {
        const ${concat(VAR, bar)}: i32 = 1;
    };
}

// Will produce `VARbar` instead of `VAR_123`
foo!(_123);
```

In other words, forgetting the dollar symbol can produce undesired outputs.

cc #29599
cc https://github.com/rust-lang/rust/issues/124225
This commit is contained in:
bors 2024-06-14 16:41:39 +00:00
commit f8e5660532
15 changed files with 551 additions and 29 deletions

View File

@ -8,9 +8,14 @@
use rustc_span::symbol::Ident;
use rustc_span::Span;
pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers";
/// A meta-variable expression, for expansions based on properties of meta-variables.
#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
#[derive(Debug, PartialEq, Encodable, Decodable)]
pub(crate) enum MetaVarExpr {
/// Unification of two or more identifiers.
Concat(Box<[MetaVarExprConcatElem]>),
/// The number of repetitions of an identifier.
Count(Ident, usize),
@ -42,6 +47,31 @@ pub(crate) fn parse<'psess>(
check_trailing_token(&mut tts, psess)?;
let mut iter = args.trees();
let rslt = match ident.as_str() {
"concat" => {
let mut result = Vec::new();
loop {
let is_var = try_eat_dollar(&mut iter);
let element_ident = parse_ident(&mut iter, psess, outer_span)?;
let element = if is_var {
MetaVarExprConcatElem::Var(element_ident)
} else {
MetaVarExprConcatElem::Ident(element_ident)
};
result.push(element);
if iter.look_ahead(0).is_none() {
break;
}
if !try_eat_comma(&mut iter) {
return Err(psess.dcx.struct_span_err(outer_span, "expected comma"));
}
}
if result.len() < 2 {
return Err(psess
.dcx
.struct_span_err(ident.span, "`concat` must have at least two elements"));
}
MetaVarExpr::Concat(result.into())
}
"count" => parse_count(&mut iter, psess, ident.span)?,
"ignore" => {
eat_dollar(&mut iter, psess, ident.span)?;
@ -68,11 +98,21 @@ pub(crate) fn parse<'psess>(
pub(crate) fn ident(&self) -> Option<Ident> {
match *self {
MetaVarExpr::Count(ident, _) | MetaVarExpr::Ignore(ident) => Some(ident),
MetaVarExpr::Index(..) | MetaVarExpr::Len(..) => None,
MetaVarExpr::Concat { .. } | MetaVarExpr::Index(..) | MetaVarExpr::Len(..) => None,
}
}
}
#[derive(Debug, Decodable, Encodable, PartialEq)]
pub(crate) enum MetaVarExprConcatElem {
/// There is NO preceding dollar sign, which means that this identifier should be interpreted
/// as a literal.
Ident(Ident),
/// There is a preceding dollar sign, which means that this identifier should be expanded
/// and interpreted as a variable.
Var(Ident),
}
// Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
fn check_trailing_token<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
@ -138,26 +178,30 @@ fn parse_depth<'psess>(
fn parse_ident<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
psess: &'psess ParseSess,
span: Span,
fallback_span: Span,
) -> PResult<'psess, Ident> {
if let Some(tt) = iter.next()
&& let TokenTree::Token(token, _) = tt
{
if let Some((elem, IdentIsRaw::No)) = token.ident() {
return Ok(elem);
let Some(tt) = iter.next() else {
return Err(psess.dcx.struct_span_err(fallback_span, "expected identifier"));
};
let TokenTree::Token(token, _) = tt else {
return Err(psess.dcx.struct_span_err(tt.span(), "expected identifier"));
};
if let Some((elem, is_raw)) = token.ident() {
if let IdentIsRaw::Yes = is_raw {
return Err(psess.dcx.struct_span_err(elem.span, RAW_IDENT_ERR));
}
let token_str = pprust::token_to_string(token);
let mut err =
psess.dcx.struct_span_err(span, format!("expected identifier, found `{}`", &token_str));
err.span_suggestion(
token.span,
format!("try removing `{}`", &token_str),
"",
Applicability::MaybeIncorrect,
);
return Err(err);
return Ok(elem);
}
Err(psess.dcx.struct_span_err(span, "expected identifier"))
let token_str = pprust::token_to_string(token);
let mut err =
psess.dcx.struct_span_err(token.span, format!("expected identifier, found `{token_str}`"));
err.span_suggestion(
token.span,
format!("try removing `{token_str}`"),
"",
Applicability::MaybeIncorrect,
);
Err(err)
}
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
@ -170,6 +214,17 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
false
}
/// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
/// iterator is not modified and the result is `false`.
fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
{
let _ = iter.next();
return true;
}
false
}
/// Expects that the next item is a dollar sign.
fn eat_dollar<'psess>(
iter: &mut RefTokenTreeCursor<'_>,

View File

@ -155,6 +155,13 @@ fn maybe_emit_macro_metavar_expr_feature(features: &Features, sess: &Session, sp
}
}
fn maybe_emit_macro_metavar_expr_concat_feature(features: &Features, sess: &Session, span: Span) {
if !features.macro_metavar_expr_concat {
let msg = "the `concat` meta-variable expression is unstable";
feature_err(sess, sym::macro_metavar_expr_concat, span, msg).emit();
}
}
/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a
/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree`
/// for use in parsing a macro.
@ -217,11 +224,19 @@ fn parse_tree<'a>(
return TokenTree::token(token::Dollar, dollar_span);
}
Ok(elem) => {
maybe_emit_macro_metavar_expr_feature(
features,
sess,
delim_span.entire(),
);
if let MetaVarExpr::Concat(_) = elem {
maybe_emit_macro_metavar_expr_concat_feature(
features,
sess,
delim_span.entire(),
);
} else {
maybe_emit_macro_metavar_expr_feature(
features,
sess,
delim_span.entire(),
);
}
return TokenTree::MetaVarExpr(delim_span, elem);
}
}

View File

@ -3,18 +3,19 @@
NoSyntaxVarsExprRepeat, VarStillRepeating,
};
use crate::mbe::macro_parser::{NamedMatch, NamedMatch::*};
use crate::mbe::metavar_expr::{MetaVarExprConcatElem, RAW_IDENT_ERR};
use crate::mbe::{self, KleeneOp, MetaVarExpr};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::IdentIsRaw;
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{pluralize, Diag, DiagCtxt, PResult};
use rustc_parse::parser::ParseNtResult;
use rustc_session::parse::ParseSess;
use rustc_span::hygiene::{LocalExpnId, Transparency};
use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
use rustc_span::{with_metavar_spans, Span, SyntaxContext};
use rustc_session::parse::ParseSess;
use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext};
use smallvec::{smallvec, SmallVec};
use std::mem;
@ -675,6 +676,23 @@ fn transcribe_metavar_expr<'a>(
span
};
match *expr {
MetaVarExpr::Concat(ref elements) => {
let mut concatenated = String::new();
for element in elements.into_iter() {
let string = match element {
MetaVarExprConcatElem::Ident(ident) => ident.to_string(),
MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?,
};
concatenated.push_str(&string);
}
// The current implementation marks the span as coming from the macro regardless of
// contexts of the concatenated identifiers but this behavior may change in the
// future.
result.push(TokenTree::Token(
Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())),
Spacing::Alone,
));
}
MetaVarExpr::Count(original_ident, depth) => {
let matched = matched_from_ident(dcx, original_ident, interp)?;
let count = count_repetitions(dcx, depth, matched, repeats, sp)?;
@ -709,3 +727,33 @@ fn transcribe_metavar_expr<'a>(
}
Ok(())
}
/// Extracts an identifier that can be originated from a `$var:ident` variable or from a token tree.
fn extract_ident<'a>(
dcx: &'a DiagCtxt,
ident: Ident,
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
) -> PResult<'a, String> {
if let NamedMatch::MatchedSingle(pnr) = matched_from_ident(dcx, ident, interp)? {
if let ParseNtResult::Ident(nt_ident, is_raw) = pnr {
if let IdentIsRaw::Yes = is_raw {
return Err(dcx.struct_span_err(ident.span, RAW_IDENT_ERR));
}
return Ok(nt_ident.to_string());
}
if let ParseNtResult::Tt(TokenTree::Token(
Token { kind: TokenKind::Ident(token_ident, is_raw), .. },
_,
)) = pnr
{
if let IdentIsRaw::Yes = is_raw {
return Err(dcx.struct_span_err(ident.span, RAW_IDENT_ERR));
}
return Ok(token_ident.to_string());
}
}
Err(dcx.struct_span_err(
ident.span,
"`${concat(..)}` currently only accepts identifiers or meta-variables as parameters",
))
}

View File

@ -516,6 +516,8 @@ pub fn internal(&self, feature: Symbol) -> bool {
(unstable, lint_reasons, "1.31.0", Some(54503)),
/// Give access to additional metadata about declarative macro meta-variables.
(unstable, macro_metavar_expr, "1.61.0", Some(83527)),
/// Provides a way to concatenate identifiers using metavariable expressions.
(unstable, macro_metavar_expr_concat, "CURRENT_RUSTC_VERSION", Some(124225)),
/// Allows `#[marker]` on certain traits allowing overlapping implementations.
(unstable, marker_trait_attr, "1.30.0", Some(29864)),
/// Allows exhaustive pattern matching on types that contain uninhabited types in cases that are

View File

@ -1118,6 +1118,7 @@
macro_lifetime_matcher,
macro_literal_matcher,
macro_metavar_expr,
macro_metavar_expr_concat,
macro_reexport,
macro_use,
macro_vis_matcher,

View File

@ -0,0 +1,9 @@
macro_rules! join {
($lhs:ident, $rhs:ident) => {
let ${concat($lhs, $rhs)}: () = ();
//~^ ERROR the `concat` meta-variable expression is unstable
};
}
fn main() {
}

View File

@ -0,0 +1,13 @@
error[E0658]: the `concat` meta-variable expression is unstable
--> $DIR/feature-gate-macro-metavar-expr-concat.rs:3:14
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^^^^^^^^^^^^^^^^^^
|
= note: see issue #124225 <https://github.com/rust-lang/rust/issues/124225> for more information
= help: add `#![feature(macro_metavar_expr_concat)]` to the crate attributes to enable
= note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
error: aborting due to 1 previous error
For more information about this error, try `rustc --explain E0658`.

View File

@ -0,0 +1,58 @@
//@ run-pass
#![allow(dead_code, non_camel_case_types, non_upper_case_globals)]
#![feature(macro_metavar_expr_concat)]
macro_rules! create_things {
($lhs:ident) => {
struct ${concat($lhs, _separated_idents_in_a_struct)} {
foo: i32,
${concat($lhs, _separated_idents_in_a_field)}: i32,
}
mod ${concat($lhs, _separated_idents_in_a_module)} {
pub const FOO: () = ();
}
fn ${concat($lhs, _separated_idents_in_a_fn)}() {}
};
}
macro_rules! many_idents {
($a:ident, $c:ident) => {
const ${concat($a, B, $c, D)}: i32 = 1;
};
}
macro_rules! valid_tts {
($_0:tt, $_1:tt) => {
const ${concat($_0, $_1)}: i32 = 1;
}
}
macro_rules! without_dollar_sign_is_an_ident {
($ident:ident) => {
const ${concat(VAR, ident)}: i32 = 1;
const ${concat(VAR, $ident)}: i32 = 2;
};
}
fn main() {
create_things!(behold);
behold_separated_idents_in_a_fn();
let _ = behold_separated_idents_in_a_module::FOO;
let _ = behold_separated_idents_in_a_struct {
foo: 1,
behold_separated_idents_in_a_field: 2,
};
many_idents!(A, C);
assert_eq!(ABCD, 1);
valid_tts!(X, YZ);
assert_eq!(XYZ, 1);
without_dollar_sign_is_an_ident!(_123);
assert_eq!(VARident, 1);
assert_eq!(VAR_123, 2);
}

View File

@ -0,0 +1,13 @@
#![feature(macro_metavar_expr_concat)]
macro_rules! join {
($lhs:ident, $rhs:ident) => {
${concat($lhs, $rhs)}
//~^ ERROR cannot find value `abcdef` in this scope
};
}
fn main() {
let abcdef = 1;
let _another = join!(abc, def);
}

View File

@ -0,0 +1,14 @@
error[E0425]: cannot find value `abcdef` in this scope
--> $DIR/hygiene.rs:5:10
|
LL | ${concat($lhs, $rhs)}
| ^^^^^^^^^^^^^^^^^^^^ not found in this scope
...
LL | let _another = join!(abc, def);
| --------------- in this macro invocation
|
= note: this error originates in the macro `join` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 1 previous error
For more information about this error, try `rustc --explain E0425`.

View File

@ -0,0 +1,81 @@
#![feature(macro_metavar_expr_concat)]
macro_rules! idents_01 {
($rhs:ident) => {
let ${concat(abc, $rhs)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
};
}
macro_rules! idents_10 {
($lhs:ident) => {
let ${concat($lhs, abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
};
}
macro_rules! idents_11 {
($lhs:ident, $rhs:ident) => {
let ${concat($lhs, $rhs)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
//~| ERROR `${concat(..)}` currently does not support raw identifiers
//~| ERROR `${concat(..)}` currently does not support raw identifiers
};
}
macro_rules! no_params {
() => {
let ${concat(r#abc, abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
//~| ERROR expected pattern, found `$`
let ${concat(abc, r#abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
let ${concat(r#abc, r#abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
};
}
macro_rules! tts_01 {
($rhs:tt) => {
let ${concat(abc, $rhs)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
};
}
macro_rules! tts_10 {
($lhs:tt) => {
let ${concat($lhs, abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
};
}
macro_rules! tts_11 {
($lhs:tt, $rhs:tt) => {
let ${concat($lhs, $rhs)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
//~| ERROR `${concat(..)}` currently does not support raw identifiers
//~| ERROR `${concat(..)}` currently does not support raw identifiers
};
}
fn main() {
idents_01!(r#_c);
idents_10!(r#_c);
idents_11!(r#_c, d);
idents_11!(_e, r#f);
idents_11!(r#_g, r#h);
tts_01!(r#_c);
tts_10!(r#_c);
tts_11!(r#_c, d);
tts_11!(_e, r#f);
tts_11!(r#_g, r#h);
no_params!();
}

View File

@ -0,0 +1,95 @@
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:28:22
|
LL | let ${concat(r#abc, abc)}: () = ();
| ^^^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:32:27
|
LL | let ${concat(abc, r#abc)}: () = ();
| ^^^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:35:22
|
LL | let ${concat(r#abc, r#abc)}: () = ();
| ^^^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:5:28
|
LL | let ${concat(abc, $rhs)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:12:23
|
LL | let ${concat($lhs, abc)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:19:23
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:19:29
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:19:23
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^
|
= note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:42:28
|
LL | let ${concat(abc, $rhs)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:49:23
|
LL | let ${concat($lhs, abc)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:56:23
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:56:29
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^
error: `${concat(..)}` currently does not support raw identifiers
--> $DIR/raw-identifiers.rs:56:23
|
LL | let ${concat($lhs, $rhs)}: () = ();
| ^^^
|
= note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
error: expected pattern, found `$`
--> $DIR/raw-identifiers.rs:28:13
|
LL | let ${concat(r#abc, abc)}: () = ();
| ^ expected pattern
...
LL | no_params!();
| ------------ in this macro invocation
|
= note: this error originates in the macro `no_params` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 14 previous errors

View File

@ -0,0 +1,50 @@
#![feature(macro_metavar_expr_concat)]
macro_rules! wrong_concat_declarations {
($ex:expr) => {
${concat()}
//~^ ERROR expected identifier
${concat(aaaa)}
//~^ ERROR `concat` must have at least two elements
${concat(aaaa,)}
//~^ ERROR expected identifier
${concat(aaaa, 1)}
//~^ ERROR expected identifier
${concat(_, aaaa)}
${concat(aaaa aaaa)}
//~^ ERROR expected comma
${concat($ex)}
//~^ ERROR `concat` must have at least two elements
${concat($ex, aaaa)}
//~^ ERROR `${concat(..)}` currently only accepts identifiers
${concat($ex, aaaa 123)}
//~^ ERROR expected comma
${concat($ex, aaaa,)}
//~^ ERROR expected identifier
${concat($ex, aaaa, 123)}
//~^ ERROR expected identifier
};
}
macro_rules! dollar_sign_without_referenced_ident {
($ident:ident) => {
const ${concat(FOO, $foo)}: i32 = 2;
//~^ ERROR variable `foo` is not recognized in meta-variable expression
};
}
fn main() {
wrong_concat_declarations!(1);
dollar_sign_without_referenced_ident!(VAR);
}

View File

@ -0,0 +1,68 @@
error: expected identifier
--> $DIR/syntax-errors.rs:5:10
|
LL | ${concat()}
| ^^^^^^^^^^
error: `concat` must have at least two elements
--> $DIR/syntax-errors.rs:8:11
|
LL | ${concat(aaaa)}
| ^^^^^^
error: expected identifier
--> $DIR/syntax-errors.rs:11:10
|
LL | ${concat(aaaa,)}
| ^^^^^^^^^^^^^^^
error: expected identifier, found `1`
--> $DIR/syntax-errors.rs:14:24
|
LL | ${concat(aaaa, 1)}
| ^ help: try removing `1`
error: expected comma
--> $DIR/syntax-errors.rs:19:10
|
LL | ${concat(aaaa aaaa)}
| ^^^^^^^^^^^^^^^^^^^
error: `concat` must have at least two elements
--> $DIR/syntax-errors.rs:22:11
|
LL | ${concat($ex)}
| ^^^^^^
error: expected comma
--> $DIR/syntax-errors.rs:28:10
|
LL | ${concat($ex, aaaa 123)}
| ^^^^^^^^^^^^^^^^^^^^^^^
error: expected identifier
--> $DIR/syntax-errors.rs:31:10
|
LL | ${concat($ex, aaaa,)}
| ^^^^^^^^^^^^^^^^^^^^
error: expected identifier, found `123`
--> $DIR/syntax-errors.rs:34:29
|
LL | ${concat($ex, aaaa, 123)}
| ^^^ help: try removing `123`
error: `${concat(..)}` currently only accepts identifiers or meta-variables as parameters
--> $DIR/syntax-errors.rs:25:19
|
LL | ${concat($ex, aaaa)}
| ^^
error: variable `foo` is not recognized in meta-variable expression
--> $DIR/syntax-errors.rs:41:30
|
LL | const ${concat(FOO, $foo)}: i32 = 2;
| ^^^
error: aborting due to 11 previous errors

View File

@ -191,10 +191,10 @@ LL | ( $( $i:ident ),* ) => { ${ aaaaaaaaaaaaaa(i) } };
| ^^^^^^^^^^^^^^ help: supported expressions are count, ignore, index and len
error: expected identifier
--> $DIR/syntax-errors.rs:118:31
--> $DIR/syntax-errors.rs:118:33
|
LL | ( $( $i:ident ),* ) => { ${ {} } };
| ^^^^^^
| ^^
error: `count` can not be placed inside the inner-most repetition
--> $DIR/syntax-errors.rs:12:24