Rollup merge of #126841 - c410-f3r:concat-again, r=petrochenkov

[`macro_metavar_expr_concat`] Add support for literals

Adds support for things like `${concat($variable, 123)}` or `${concat("hello", "_world")}` .

cc #124225
This commit is contained in:
许杰友 Jieyou Xu (Joe) 2024-07-08 13:04:30 +08:00 committed by GitHub
commit 2c16d65c1e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 272 additions and 59 deletions

View File

@ -1,4 +1,4 @@
use rustc_ast::token::{self, Delimiter, IdentIsRaw};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind};
use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
use rustc_ast::{LitIntType, LitKind};
use rustc_ast_pretty::pprust;
@ -6,9 +6,10 @@
use rustc_macros::{Decodable, Encodable};
use rustc_session::parse::ParseSess;
use rustc_span::symbol::Ident;
use rustc_span::Span;
use rustc_span::{Span, Symbol};
pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers";
pub(crate) const UNSUPPORTED_CONCAT_ELEM_ERR: &str = "expected identifier or string literal";
/// A meta-variable expression, for expansions based on properties of meta-variables.
#[derive(Debug, PartialEq, Encodable, Decodable)]
@ -51,11 +52,26 @@ pub(crate) fn parse<'psess>(
let mut result = Vec::new();
loop {
let is_var = try_eat_dollar(&mut iter);
let element_ident = parse_ident(&mut iter, psess, outer_span)?;
let token = parse_token(&mut iter, psess, outer_span)?;
let element = if is_var {
MetaVarExprConcatElem::Var(element_ident)
MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?)
} else if let TokenKind::Literal(Lit {
kind: token::LitKind::Str,
symbol,
suffix: None,
}) = token.kind
{
MetaVarExprConcatElem::Literal(symbol)
} else {
MetaVarExprConcatElem::Ident(element_ident)
match parse_ident_from_token(psess, token) {
Err(err) => {
err.cancel();
return Err(psess
.dcx()
.struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR));
}
Ok(elem) => MetaVarExprConcatElem::Ident(elem),
}
};
result.push(element);
if iter.look_ahead(0).is_none() {
@ -105,11 +121,13 @@ pub(crate) fn ident(&self) -> Option<Ident> {
#[derive(Debug, Decodable, Encodable, PartialEq)]
pub(crate) enum MetaVarExprConcatElem {
/// There is NO preceding dollar sign, which means that this identifier should be interpreted
/// as a literal.
/// Identifier WITHOUT a preceding dollar sign, which means that this identifier should be
/// interpreted as a literal.
Ident(Ident),
/// There is a preceding dollar sign, which means that this identifier should be expanded
/// and interpreted as a variable.
/// For example, a number or a string.
Literal(Symbol),
/// Identifier WITH a preceding dollar sign, which means that this identifier should be
/// expanded and interpreted as a variable.
Var(Ident),
}
@ -158,7 +176,7 @@ fn parse_depth<'psess>(
span: Span,
) -> PResult<'psess, usize> {
let Some(tt) = iter.next() else { return Ok(0) };
let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else {
let TokenTree::Token(Token { kind: TokenKind::Literal(lit), .. }, _) = tt else {
return Err(psess
.dcx()
.struct_span_err(span, "meta-variable expression depth must be a literal"));
@ -180,12 +198,14 @@ fn parse_ident<'psess>(
psess: &'psess ParseSess,
fallback_span: Span,
) -> PResult<'psess, Ident> {
let Some(tt) = iter.next() else {
return Err(psess.dcx().struct_span_err(fallback_span, "expected identifier"));
};
let TokenTree::Token(token, _) = tt else {
return Err(psess.dcx().struct_span_err(tt.span(), "expected identifier"));
};
let token = parse_token(iter, psess, fallback_span)?;
parse_ident_from_token(psess, token)
}
fn parse_ident_from_token<'psess>(
psess: &'psess ParseSess,
token: &Token,
) -> PResult<'psess, Ident> {
if let Some((elem, is_raw)) = token.ident() {
if let IdentIsRaw::Yes = is_raw {
return Err(psess.dcx().struct_span_err(elem.span, RAW_IDENT_ERR));
@ -205,10 +225,24 @@ fn parse_ident<'psess>(
Err(err)
}
fn parse_token<'psess, 't>(
iter: &mut RefTokenTreeCursor<'t>,
psess: &'psess ParseSess,
fallback_span: Span,
) -> PResult<'psess, &'t Token> {
let Some(tt) = iter.next() else {
return Err(psess.dcx().struct_span_err(fallback_span, UNSUPPORTED_CONCAT_ELEM_ERR));
};
let TokenTree::Token(token, _) = tt else {
return Err(psess.dcx().struct_span_err(tt.span(), UNSUPPORTED_CONCAT_ELEM_ERR));
};
Ok(token)
}
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
/// iterator is not modified and the result is `false`.
fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
if let Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
let _ = iter.next();
return true;
}
@ -218,8 +252,7 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
/// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
/// iterator is not modified and the result is `false`.
fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
{
if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
let _ = iter.next();
return true;
}
@ -232,8 +265,7 @@ fn eat_dollar<'psess>(
psess: &'psess ParseSess,
span: Span,
) -> PResult<'psess, ()> {
if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
{
if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
let _ = iter.next();
return Ok(());
}

View File

@ -11,11 +11,13 @@
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{pluralize, Diag, DiagCtxtHandle, PResult};
use rustc_parse::lexer::nfc_normalize;
use rustc_parse::parser::ParseNtResult;
use rustc_session::parse::ParseSess;
use rustc_session::parse::SymbolGallery;
use rustc_span::hygiene::{LocalExpnId, Transparency};
use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext};
use rustc_span::{with_metavar_spans, Span, SyntaxContext};
use smallvec::{smallvec, SmallVec};
use std::mem;
@ -312,7 +314,16 @@ pub(super) fn transcribe<'a>(
// Replace meta-variable expressions with the result of their expansion.
mbe::TokenTree::MetaVarExpr(sp, expr) => {
transcribe_metavar_expr(dcx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
transcribe_metavar_expr(
dcx,
expr,
interp,
&mut marker,
&repeats,
&mut result,
sp,
&psess.symbol_gallery,
)?;
}
// If we are entering a new delimiter, we push its contents to the `stack` to be
@ -669,6 +680,7 @@ fn transcribe_metavar_expr<'a>(
repeats: &[(usize, usize)],
result: &mut Vec<TokenTree>,
sp: &DelimSpan,
symbol_gallery: &SymbolGallery,
) -> PResult<'a, ()> {
let mut visited_span = || {
let mut span = sp.entire();
@ -680,16 +692,26 @@ fn transcribe_metavar_expr<'a>(
let mut concatenated = String::new();
for element in elements.into_iter() {
let string = match element {
MetaVarExprConcatElem::Ident(ident) => ident.to_string(),
MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?,
MetaVarExprConcatElem::Ident(elem) => elem.to_string(),
MetaVarExprConcatElem::Literal(elem) => elem.as_str().into(),
MetaVarExprConcatElem::Var(elem) => extract_ident(dcx, *elem, interp)?,
};
concatenated.push_str(&string);
}
let symbol = nfc_normalize(&concatenated);
let concatenated_span = visited_span();
if !rustc_lexer::is_ident(symbol.as_str()) {
return Err(dcx.struct_span_err(
concatenated_span,
"`${concat(..)}` is not generating a valid identifier",
));
}
symbol_gallery.insert(symbol, concatenated_span);
// The current implementation marks the span as coming from the macro regardless of
// contexts of the concatenated identifiers but this behavior may change in the
// future.
result.push(TokenTree::Token(
Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())),
Token::from_ast_ident(Ident::new(symbol, concatenated_span)),
Spacing::Alone,
));
}

View File

@ -37,6 +37,16 @@ macro_rules! without_dollar_sign_is_an_ident {
};
}
macro_rules! literals {
($ident:ident) => {{
let ${concat(_a, "_b")}: () = ();
let ${concat("_b", _a)}: () = ();
let ${concat($ident, "_b")}: () = ();
let ${concat("_b", $ident)}: () = ();
}};
}
fn main() {
create_things!(behold);
behold_separated_idents_in_a_fn();
@ -55,4 +65,6 @@ fn main() {
without_dollar_sign_is_an_ident!(_123);
assert_eq!(VARident, 1);
assert_eq!(VAR_123, 2);
literals!(_hello);
}

View File

@ -26,14 +26,14 @@ macro_rules! idents_11 {
macro_rules! no_params {
() => {
let ${concat(r#abc, abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
//~^ ERROR expected identifier or string literal
//~| ERROR expected pattern, found `$`
let ${concat(abc, r#abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
//~^ ERROR expected identifier or string literal
let ${concat(r#abc, r#abc)}: () = ();
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
//~^ ERROR expected identifier or string literal
};
}

View File

@ -1,16 +1,16 @@
error: `${concat(..)}` currently does not support raw identifiers
error: expected identifier or string literal
--> $DIR/raw-identifiers.rs:28:22
|
LL | let ${concat(r#abc, abc)}: () = ();
| ^^^^^
error: `${concat(..)}` currently does not support raw identifiers
error: expected identifier or string literal
--> $DIR/raw-identifiers.rs:32:27
|
LL | let ${concat(abc, r#abc)}: () = ();
| ^^^^^
error: `${concat(..)}` currently does not support raw identifiers
error: expected identifier or string literal
--> $DIR/raw-identifiers.rs:35:22
|
LL | let ${concat(r#abc, r#abc)}: () = ();

View File

@ -11,9 +11,6 @@ macro_rules! wrong_concat_declarations {
${concat(aaaa,)}
//~^ ERROR expected identifier
${concat(aaaa, 1)}
//~^ ERROR expected identifier
${concat(_, aaaa)}
${concat(aaaa aaaa)}
@ -30,9 +27,6 @@ macro_rules! wrong_concat_declarations {
${concat($ex, aaaa,)}
//~^ ERROR expected identifier
${concat($ex, aaaa, 123)}
//~^ ERROR expected identifier
};
}
@ -43,8 +37,80 @@ macro_rules! dollar_sign_without_referenced_ident {
};
}
macro_rules! starting_number {
($ident:ident) => {{
let ${concat("1", $ident)}: () = ();
//~^ ERROR `${concat(..)}` is not generating a valid identifier
}};
}
macro_rules! starting_valid_unicode {
($ident:ident) => {{
let ${concat("Ý", $ident)}: () = ();
}};
}
macro_rules! starting_invalid_unicode {
($ident:ident) => {{
let ${concat("\u{00BD}", $ident)}: () = ();
//~^ ERROR `${concat(..)}` is not generating a valid identifier
}};
}
macro_rules! ending_number {
($ident:ident) => {{
let ${concat($ident, "1")}: () = ();
}};
}
macro_rules! ending_valid_unicode {
($ident:ident) => {{
let ${concat($ident, "Ý")}: () = ();
}};
}
macro_rules! ending_invalid_unicode {
($ident:ident) => {{
let ${concat($ident, "\u{00BD}")}: () = ();
//~^ ERROR `${concat(..)}` is not generating a valid identifier
}};
}
macro_rules! empty {
() => {{
let ${concat("", "")}: () = ();
//~^ ERROR `${concat(..)}` is not generating a valid identifier
}};
}
macro_rules! unsupported_literals {
($ident:ident) => {{
let ${concat(_a, 'b')}: () = ();
//~^ ERROR expected identifier or string literal
//~| ERROR expected pattern
let ${concat(_a, 1)}: () = ();
//~^ ERROR expected identifier or string literal
let ${concat($ident, 'b')}: () = ();
//~^ ERROR expected identifier or string literal
let ${concat($ident, 1)}: () = ();
//~^ ERROR expected identifier or string literal
}};
}
fn main() {
wrong_concat_declarations!(1);
dollar_sign_without_referenced_ident!(VAR);
starting_number!(_abc);
starting_valid_unicode!(_abc);
starting_invalid_unicode!(_abc);
ending_number!(_abc);
ending_valid_unicode!(_abc);
ending_invalid_unicode!(_abc);
unsupported_literals!(_abc);
empty!();
}

View File

@ -1,4 +1,4 @@
error: expected identifier
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:5:10
|
LL | ${concat()}
@ -10,59 +10,126 @@ error: `concat` must have at least two elements
LL | ${concat(aaaa)}
| ^^^^^^
error: expected identifier
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:11:10
|
LL | ${concat(aaaa,)}
| ^^^^^^^^^^^^^^^
error: expected identifier, found `1`
--> $DIR/syntax-errors.rs:14:24
|
LL | ${concat(aaaa, 1)}
| ^ help: try removing `1`
error: expected comma
--> $DIR/syntax-errors.rs:19:10
--> $DIR/syntax-errors.rs:16:10
|
LL | ${concat(aaaa aaaa)}
| ^^^^^^^^^^^^^^^^^^^
error: `concat` must have at least two elements
--> $DIR/syntax-errors.rs:22:11
--> $DIR/syntax-errors.rs:19:11
|
LL | ${concat($ex)}
| ^^^^^^
error: expected comma
--> $DIR/syntax-errors.rs:28:10
--> $DIR/syntax-errors.rs:25:10
|
LL | ${concat($ex, aaaa 123)}
| ^^^^^^^^^^^^^^^^^^^^^^^
error: expected identifier
--> $DIR/syntax-errors.rs:31:10
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:28:10
|
LL | ${concat($ex, aaaa,)}
| ^^^^^^^^^^^^^^^^^^^^
error: expected identifier, found `123`
--> $DIR/syntax-errors.rs:34:29
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:88:26
|
LL | ${concat($ex, aaaa, 123)}
| ^^^ help: try removing `123`
LL | let ${concat(_a, 'b')}: () = ();
| ^^^
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:91:26
|
LL | let ${concat(_a, 1)}: () = ();
| ^
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:94:30
|
LL | let ${concat($ident, 'b')}: () = ();
| ^^^
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:96:30
|
LL | let ${concat($ident, 1)}: () = ();
| ^
error: `${concat(..)}` currently only accepts identifiers or meta-variables as parameters
--> $DIR/syntax-errors.rs:25:19
--> $DIR/syntax-errors.rs:22:19
|
LL | ${concat($ex, aaaa)}
| ^^
error: variable `foo` is not recognized in meta-variable expression
--> $DIR/syntax-errors.rs:41:30
--> $DIR/syntax-errors.rs:35:30
|
LL | const ${concat(FOO, $foo)}: i32 = 2;
| ^^^
error: aborting due to 11 previous errors
error: `${concat(..)}` is not generating a valid identifier
--> $DIR/syntax-errors.rs:42:14
|
LL | let ${concat("1", $ident)}: () = ();
| ^^^^^^^^^^^^^^^^^^^^^
...
LL | starting_number!(_abc);
| ---------------------- in this macro invocation
|
= note: this error originates in the macro `starting_number` (in Nightly builds, run with -Z macro-backtrace for more info)
error: `${concat(..)}` is not generating a valid identifier
--> $DIR/syntax-errors.rs:55:14
|
LL | let ${concat("\u{00BD}", $ident)}: () = ();
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...
LL | starting_invalid_unicode!(_abc);
| ------------------------------- in this macro invocation
|
= note: this error originates in the macro `starting_invalid_unicode` (in Nightly builds, run with -Z macro-backtrace for more info)
error: `${concat(..)}` is not generating a valid identifier
--> $DIR/syntax-errors.rs:74:14
|
LL | let ${concat($ident, "\u{00BD}")}: () = ();
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...
LL | ending_invalid_unicode!(_abc);
| ----------------------------- in this macro invocation
|
= note: this error originates in the macro `ending_invalid_unicode` (in Nightly builds, run with -Z macro-backtrace for more info)
error: expected pattern, found `$`
--> $DIR/syntax-errors.rs:88:13
|
LL | let ${concat(_a, 'b')}: () = ();
| ^ expected pattern
...
LL | unsupported_literals!(_abc);
| --------------------------- in this macro invocation
|
= note: this error originates in the macro `unsupported_literals` (in Nightly builds, run with -Z macro-backtrace for more info)
error: `${concat(..)}` is not generating a valid identifier
--> $DIR/syntax-errors.rs:81:14
|
LL | let ${concat("", "")}: () = ();
| ^^^^^^^^^^^^^^^^
...
LL | empty!();
| -------- in this macro invocation
|
= note: this error originates in the macro `empty` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 18 previous errors

View File

@ -0,0 +1,14 @@
//@ run-pass
#![feature(macro_metavar_expr_concat)]
macro_rules! turn_to_page {
($ident:ident) => {
const ${concat("", $ident)}: i32 = 394;
};
}
fn main() {
turn_to_page!(P);
assert_eq!(P, 394);
}

View File

@ -190,7 +190,7 @@ error: unrecognized meta-variable expression
LL | ( $( $i:ident ),* ) => { ${ aaaaaaaaaaaaaa(i) } };
| ^^^^^^^^^^^^^^ help: supported expressions are count, ignore, index and len
error: expected identifier
error: expected identifier or string literal
--> $DIR/syntax-errors.rs:118:33
|
LL | ( $( $i:ident ),* ) => { ${ {} } };