From de4c8975aaa05063129196e470d3dcf7558f19b1 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 18 Sep 2024 15:01:22 -0700 Subject: [PATCH 01/11] bootstrap: Set the dylib path when building books with rustdoc The library path is needed when the toolchain has been configured with `[rust] rpath = false`. Otherwise, building the reference book will get an error when it tries to run rustdoc, like: rustdoc: error while loading shared libraries: librustc_driver-2ec457c3b8826b72.so --- src/bootstrap/src/core/build_steps/doc.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 3755f4a33cb..4daccf6fdb4 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -63,7 +63,7 @@ fn run(self, builder: &Builder<'_>) { src: builder.src.join($path), parent: Some(self), languages: $lang.into(), - rustdoc: None, + rustdoc_compiler: None, }) } } @@ -113,7 +113,7 @@ fn run(self, builder: &Builder<'_>) { src: builder.md_doc_out(self.target).join("unstable-book"), parent: Some(self), languages: vec![], - rustdoc: None, + rustdoc_compiler: None, }) } } @@ -125,7 +125,7 @@ struct RustbookSrc { src: PathBuf, parent: Option

, languages: Vec<&'static str>, - rustdoc: Option, + rustdoc_compiler: Option, } impl Step for RustbookSrc

{ @@ -157,7 +157,9 @@ fn run(self, builder: &Builder<'_>) { let _ = fs::remove_dir_all(&out); let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - if let Some(mut rustdoc) = self.rustdoc { + + if let Some(compiler) = self.rustdoc_compiler { + let mut rustdoc = builder.rustdoc(compiler); rustdoc.pop(); let old_path = env::var_os("PATH").unwrap_or_default(); let new_path = @@ -165,6 +167,7 @@ fn run(self, builder: &Builder<'_>) { .expect("could not add rustdoc to PATH"); rustbook_cmd.env("PATH", new_path); + builder.add_rustc_lib_path(compiler, &mut rustbook_cmd); } rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out).run(builder); @@ -240,7 +243,7 @@ fn run(self, builder: &Builder<'_>) { src: absolute_path.clone(), parent: Some(self), languages: vec![], - rustdoc: None, + rustdoc_compiler: None, }); // building older edition redirects @@ -253,7 +256,7 @@ fn run(self, builder: &Builder<'_>) { // treat the other editions as not having a parent. parent: Option::::None, languages: vec![], - rustdoc: None, + rustdoc_compiler: None, }); } @@ -1229,7 +1232,7 @@ fn run(self, builder: &Builder<'_>) { src: out_base, parent: Some(self), languages: vec![], - rustdoc: None, + rustdoc_compiler: None, }); } } @@ -1263,16 +1266,15 @@ fn run(self, builder: &Builder<'_>) { // This is needed for generating links to the standard library using // the mdbook-spec plugin. builder.ensure(compile::Std::new(self.compiler, builder.config.build)); - let rustdoc = builder.rustdoc(self.compiler); // Run rustbook/mdbook to generate the HTML pages. builder.ensure(RustbookSrc { target: self.target, name: "reference".to_owned(), src: builder.src.join("src/doc/reference"), + rustdoc_compiler: Some(self.compiler), parent: Some(self), languages: vec![], - rustdoc: Some(rustdoc), }); } } From 019435b265eaa85f3b0b6de2957c9a3af2c3ceb1 Mon Sep 17 00:00:00 2001 From: Artyom Tetyukhin <51746822+arttet@users.noreply.github.com> Date: Sat, 21 Sep 2024 13:29:00 +0400 Subject: [PATCH 02/11] Remove x86_64-fuchsia and aarch64-fuchsia target aliases --- compiler/rustc_target/src/spec/mod.rs | 4 ---- compiler/rustc_target/src/spec/targets/aarch64_fuchsia.rs | 1 - compiler/rustc_target/src/spec/targets/x86_64_fuchsia.rs | 1 - src/doc/rustc/src/platform-support.md | 2 -- tests/assembly/targets/targets-elf.rs | 6 ------ 5 files changed, 14 deletions(-) delete mode 100644 compiler/rustc_target/src/spec/targets/aarch64_fuchsia.rs delete mode 100644 compiler/rustc_target/src/spec/targets/x86_64_fuchsia.rs diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 1d478f84c43..44fb177daad 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -1690,12 +1690,8 @@ fn $module() { ("x86_64h-apple-darwin", x86_64h_apple_darwin), ("i686-apple-darwin", i686_apple_darwin), - // FIXME(#106649): Remove aarch64-fuchsia in favor of aarch64-unknown-fuchsia - ("aarch64-fuchsia", aarch64_fuchsia), ("aarch64-unknown-fuchsia", aarch64_unknown_fuchsia), ("riscv64gc-unknown-fuchsia", riscv64gc_unknown_fuchsia), - // FIXME(#106649): Remove x86_64-fuchsia in favor of x86_64-unknown-fuchsia - ("x86_64-fuchsia", x86_64_fuchsia), ("x86_64-unknown-fuchsia", x86_64_unknown_fuchsia), ("avr-unknown-gnu-atmega328", avr_unknown_gnu_atmega328), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_fuchsia.rs b/compiler/rustc_target/src/spec/targets/aarch64_fuchsia.rs deleted file mode 100644 index 144ac85622e..00000000000 --- a/compiler/rustc_target/src/spec/targets/aarch64_fuchsia.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) use crate::spec::targets::aarch64_unknown_fuchsia::target; diff --git a/compiler/rustc_target/src/spec/targets/x86_64_fuchsia.rs b/compiler/rustc_target/src/spec/targets/x86_64_fuchsia.rs deleted file mode 100644 index ce3e1e159b7..00000000000 --- a/compiler/rustc_target/src/spec/targets/x86_64_fuchsia.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) use crate::spec::targets::x86_64_unknown_fuchsia::target; diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index 827a7065f3e..3728d1f5160 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -138,7 +138,6 @@ target | std | notes [`aarch64-apple-ios`](platform-support/apple-ios.md) | ✓ | ARM64 iOS [`aarch64-apple-ios-macabi`](platform-support/apple-ios-macabi.md) | ✓ | Mac Catalyst on ARM64 [`aarch64-apple-ios-sim`](platform-support/apple-ios.md) | ✓ | Apple iOS Simulator on ARM64 -`aarch64-fuchsia` | ✓ | Alias for `aarch64-unknown-fuchsia` [`aarch64-unknown-fuchsia`](platform-support/fuchsia.md) | ✓ | ARM64 Fuchsia [`aarch64-linux-android`](platform-support/android.md) | ✓ | ARM64 Android [`aarch64-pc-windows-gnullvm`](platform-support/pc-windows-gnullvm.md) | ✓ | ARM64 MinGW (Windows 10+), LLVM ABI @@ -199,7 +198,6 @@ target | std | notes [`x86_64-apple-ios`](platform-support/apple-ios.md) | ✓ | 64-bit x86 iOS [`x86_64-apple-ios-macabi`](platform-support/apple-ios-macabi.md) | ✓ | Mac Catalyst on x86_64 [`x86_64-fortanix-unknown-sgx`](platform-support/x86_64-fortanix-unknown-sgx.md) | ✓ | [Fortanix ABI] for 64-bit Intel SGX -`x86_64-fuchsia` | ✓ | Alias for `x86_64-unknown-fuchsia` [`x86_64-unknown-fuchsia`](platform-support/fuchsia.md) | ✓ | 64-bit x86 Fuchsia [`x86_64-linux-android`](platform-support/android.md) | ✓ | 64-bit x86 Android `x86_64-pc-solaris` | ✓ | 64-bit Solaris 11, illumos diff --git a/tests/assembly/targets/targets-elf.rs b/tests/assembly/targets/targets-elf.rs index 4e1c5e6806e..6ffa34f802e 100644 --- a/tests/assembly/targets/targets-elf.rs +++ b/tests/assembly/targets/targets-elf.rs @@ -9,9 +9,6 @@ //@ revisions: aarch64_be_unknown_netbsd //@ [aarch64_be_unknown_netbsd] compile-flags: --target aarch64_be-unknown-netbsd //@ [aarch64_be_unknown_netbsd] needs-llvm-components: aarch64 -//@ revisions: aarch64_fuchsia -//@ [aarch64_fuchsia] compile-flags: --target aarch64-fuchsia -//@ [aarch64_fuchsia] needs-llvm-components: aarch64 //@ revisions: aarch64_kmc_solid_asp3 //@ [aarch64_kmc_solid_asp3] compile-flags: --target aarch64-kmc-solid_asp3 //@ [aarch64_kmc_solid_asp3] needs-llvm-components: aarch64 @@ -525,9 +522,6 @@ //@ revisions: x86_64_fortanix_unknown_sgx //@ [x86_64_fortanix_unknown_sgx] compile-flags: --target x86_64-fortanix-unknown-sgx //@ [x86_64_fortanix_unknown_sgx] needs-llvm-components: x86 -//@ revisions: x86_64_fuchsia -//@ [x86_64_fuchsia] compile-flags: --target x86_64-fuchsia -//@ [x86_64_fuchsia] needs-llvm-components: x86 //@ revisions: x86_64_linux_android //@ [x86_64_linux_android] compile-flags: --target x86_64-linux-android //@ [x86_64_linux_android] needs-llvm-components: x86 From 8d2809957e361a3405edc192389c07ab7d953e1f Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Sun, 22 Sep 2024 23:29:25 +0300 Subject: [PATCH 03/11] Add more test cases for block-no-opening-brace --- tests/ui/parser/block-no-opening-brace.rs | 36 ++++++++++++++----- tests/ui/parser/block-no-opening-brace.stderr | 24 +++++++++++-- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/tests/ui/parser/block-no-opening-brace.rs b/tests/ui/parser/block-no-opening-brace.rs index e90a34104e8..2fde37ce6ac 100644 --- a/tests/ui/parser/block-no-opening-brace.rs +++ b/tests/ui/parser/block-no-opening-brace.rs @@ -4,28 +4,46 @@ fn main() {} -fn f1() { +fn in_loop() { loop let x = 0; //~ ERROR expected `{`, found keyword `let` drop(0); - } +} -fn f2() { +fn in_while() { while true let x = 0; //~ ERROR expected `{`, found keyword `let` - } +} -fn f3() { +fn in_for() { for x in 0..1 let x = 0; //~ ERROR expected `{`, found keyword `let` - } +} -fn f4() { + +// FIXME +fn in_try() { try //~ ERROR expected expression, found reserved keyword `try` let x = 0; - } +} -fn f5() { +// FIXME(#80931) +fn in_async() { async let x = 0; //~ ERROR expected one of `move`, `|`, or `||`, found keyword `let` +} + +// FIXME(#78168) +fn in_const() { + let x = const 2; //~ ERROR expected expression, found keyword `const` +} + +// FIXME(#78168) +fn in_const_in_match() { + let x = 2; + match x { + const 2 => {} + //~^ ERROR expected identifier, found keyword `const` + //~| ERROR expected one of `=>`, `if`, or `|`, found `2` } +} diff --git a/tests/ui/parser/block-no-opening-brace.stderr b/tests/ui/parser/block-no-opening-brace.stderr index f232f480ce9..83360944ed5 100644 --- a/tests/ui/parser/block-no-opening-brace.stderr +++ b/tests/ui/parser/block-no-opening-brace.stderr @@ -38,18 +38,36 @@ LL | { let x = 0; } | + + error: expected expression, found reserved keyword `try` - --> $DIR/block-no-opening-brace.rs:24:5 + --> $DIR/block-no-opening-brace.rs:26:5 | LL | try | ^^^ expected expression error: expected one of `move`, `|`, or `||`, found keyword `let` - --> $DIR/block-no-opening-brace.rs:30:9 + --> $DIR/block-no-opening-brace.rs:33:9 | LL | async | - expected one of `move`, `|`, or `||` LL | let x = 0; | ^^^ unexpected token -error: aborting due to 5 previous errors +error: expected expression, found keyword `const` + --> $DIR/block-no-opening-brace.rs:38:13 + | +LL | let x = const 2; + | ^^^^^ expected expression + +error: expected identifier, found keyword `const` + --> $DIR/block-no-opening-brace.rs:45:9 + | +LL | const 2 => {} + | ^^^^^ expected identifier, found keyword + +error: expected one of `=>`, `if`, or `|`, found `2` + --> $DIR/block-no-opening-brace.rs:45:15 + | +LL | const 2 => {} + | ^ expected one of `=>`, `if`, or `|` + +error: aborting due to 8 previous errors From 73cc5751773d4c49cc9d938548762520037926ba Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 19 Sep 2024 19:32:17 +1000 Subject: [PATCH 04/11] Fix `break_last_token`. It currently doesn't handle the three-char tokens `>>=` and `<<=` correctly. These can be broken twice, resulting in three individual tokens. This is a latent bug that currently doesn't cause any problems, but does cause problems for #124141, because that PR increases the usage of lazy token streams. --- compiler/rustc_ast/src/token.rs | 64 ++++++++++--------- .../rustc_parse/src/parser/attr_wrapper.rs | 36 ++++++----- compiler/rustc_parse/src/parser/mod.rs | 39 +++++------ tests/ui/macros/break-last-token-twice.rs | 16 +++++ 4 files changed, 91 insertions(+), 64 deletions(-) create mode 100644 tests/ui/macros/break-last-token-twice.rs diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index a0082a41713..2904bae00b3 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -385,35 +385,41 @@ pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { Literal(Lit::new(kind, symbol, suffix)) } - /// An approximation to proc-macro-style single-character operators used by rustc parser. - /// If the operator token can be broken into two tokens, the first of which is single-character, - /// then this function performs that operation, otherwise it returns `None`. - pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> { - Some(match *self { - Le => (Lt, Eq), - EqEq => (Eq, Eq), - Ne => (Not, Eq), - Ge => (Gt, Eq), - AndAnd => (BinOp(And), BinOp(And)), - OrOr => (BinOp(Or), BinOp(Or)), - BinOp(Shl) => (Lt, Lt), - BinOp(Shr) => (Gt, Gt), - BinOpEq(Plus) => (BinOp(Plus), Eq), - BinOpEq(Minus) => (BinOp(Minus), Eq), - BinOpEq(Star) => (BinOp(Star), Eq), - BinOpEq(Slash) => (BinOp(Slash), Eq), - BinOpEq(Percent) => (BinOp(Percent), Eq), - BinOpEq(Caret) => (BinOp(Caret), Eq), - BinOpEq(And) => (BinOp(And), Eq), - BinOpEq(Or) => (BinOp(Or), Eq), - BinOpEq(Shl) => (Lt, Le), - BinOpEq(Shr) => (Gt, Ge), - DotDot => (Dot, Dot), - DotDotDot => (Dot, DotDot), - PathSep => (Colon, Colon), - RArrow => (BinOp(Minus), Gt), - LArrow => (Lt, BinOp(Minus)), - FatArrow => (Eq, Gt), + /// An approximation to proc-macro-style single-character operators used by + /// rustc parser. If the operator token can be broken into two tokens, the + /// first of which has `n` (1 or 2) chars, then this function performs that + /// operation, otherwise it returns `None`. + pub fn break_two_token_op(&self, n: u32) -> Option<(TokenKind, TokenKind)> { + assert!(n == 1 || n == 2); + Some(match (self, n) { + (Le, 1) => (Lt, Eq), + (EqEq, 1) => (Eq, Eq), + (Ne, 1) => (Not, Eq), + (Ge, 1) => (Gt, Eq), + (AndAnd, 1) => (BinOp(And), BinOp(And)), + (OrOr, 1) => (BinOp(Or), BinOp(Or)), + (BinOp(Shl), 1) => (Lt, Lt), + (BinOp(Shr), 1) => (Gt, Gt), + (BinOpEq(Plus), 1) => (BinOp(Plus), Eq), + (BinOpEq(Minus), 1) => (BinOp(Minus), Eq), + (BinOpEq(Star), 1) => (BinOp(Star), Eq), + (BinOpEq(Slash), 1) => (BinOp(Slash), Eq), + (BinOpEq(Percent), 1) => (BinOp(Percent), Eq), + (BinOpEq(Caret), 1) => (BinOp(Caret), Eq), + (BinOpEq(And), 1) => (BinOp(And), Eq), + (BinOpEq(Or), 1) => (BinOp(Or), Eq), + (BinOpEq(Shl), 1) => (Lt, Le), // `<` + `<=` + (BinOpEq(Shl), 2) => (BinOp(Shl), Eq), // `<<` + `=` + (BinOpEq(Shr), 1) => (Gt, Ge), // `>` + `>=` + (BinOpEq(Shr), 2) => (BinOp(Shr), Eq), // `>>` + `=` + (DotDot, 1) => (Dot, Dot), + (DotDotDot, 1) => (Dot, DotDot), // `.` + `..` + (DotDotDot, 2) => (DotDot, Dot), // `..` + `.` + (DotDotEq, 2) => (DotDot, Eq), + (PathSep, 1) => (Colon, Colon), + (RArrow, 1) => (BinOp(Minus), Gt), + (LArrow, 1) => (Lt, BinOp(Minus)), + (FatArrow, 1) => (Eq, Gt), _ => return None, }) } diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 6a241be0a15..ee045b70d75 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -108,7 +108,7 @@ struct LazyAttrTokenStreamImpl { start_token: (Token, Spacing), cursor_snapshot: TokenCursor, num_calls: u32, - break_last_token: bool, + break_last_token: u32, node_replacements: Box<[NodeReplacement]>, } @@ -339,17 +339,20 @@ pub(super) fn collect_tokens( let parser_replacements_end = self.capture_state.parser_replacements.len(); assert!( - !(self.break_last_token && matches!(capture_trailing, Trailing::Yes)), - "Cannot set break_last_token and have trailing token" + !(self.break_last_token > 0 && matches!(capture_trailing, Trailing::Yes)), + "Cannot have break_last_token > 0 and have trailing token" ); + assert!(self.break_last_token <= 2, "cannot break token more than twice"); let end_pos = self.num_bump_calls + capture_trailing as u32 - // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then - // extend the range of captured tokens to include it, since the parser was not actually - // bumped past it. When the `LazyAttrTokenStream` gets converted into an - // `AttrTokenStream`, we will create the proper token. - + self.break_last_token as u32; + // If we "broke" the last token (e.g. breaking a `>>` token once into `>` + `>`, or + // breaking a `>>=` token twice into `>` + `>` + `=`), then extend the range of + // captured tokens to include it, because the parser was not actually bumped past it. + // (Even if we broke twice, it was still just one token originally, hence the `1`.) + // When the `LazyAttrTokenStream` gets converted into an `AttrTokenStream`, we will + // rebreak that final token once or twice. + + if self.break_last_token == 0 { 0 } else { 1 }; let num_calls = end_pos - collect_pos.start_pos; @@ -425,7 +428,7 @@ pub(super) fn collect_tokens( // for the `#[cfg]` and/or `#[cfg_attr]` attrs. This allows us to run // eager cfg-expansion on the captured token stream. if definite_capture_mode { - assert!(!self.break_last_token, "Should not have unglued last token with cfg attr"); + assert!(self.break_last_token == 0, "Should not have unglued last token with cfg attr"); // What is the status here when parsing the example code at the top of this method? // @@ -471,7 +474,7 @@ pub(super) fn collect_tokens( /// close delims. fn make_attr_token_stream( iter: impl Iterator, - break_last_token: bool, + break_last_token: u32, ) -> AttrTokenStream { #[derive(Debug)] struct FrameData { @@ -513,18 +516,17 @@ struct FrameData { } } - if break_last_token { + if break_last_token > 0 { let last_token = stack_top.inner.pop().unwrap(); if let AttrTokenTree::Token(last_token, spacing) = last_token { - let unglued_first = last_token.kind.break_two_token_op().unwrap().0; + let (unglued, _) = last_token.kind.break_two_token_op(break_last_token).unwrap(); - // An 'unglued' token is always two ASCII characters + // Tokens are always ASCII chars, so we can use byte arithmetic here. let mut first_span = last_token.span.shrink_to_lo(); - first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1)); + first_span = + first_span.with_hi(first_span.lo() + rustc_span::BytePos(break_last_token)); - stack_top - .inner - .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing)); + stack_top.inner.push(AttrTokenTree::Token(Token::new(unglued, first_span), spacing)); } else { panic!("Unexpected last token {last_token:?}") } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 9d9265d5318..ca0838a7929 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -146,21 +146,25 @@ pub struct Parser<'a> { token_cursor: TokenCursor, // The number of calls to `bump`, i.e. the position in the token stream. num_bump_calls: u32, - // During parsing we may sometimes need to 'unglue' a glued token into two - // component tokens (e.g. '>>' into '>' and '>), so the parser can consume - // them one at a time. This process bypasses the normal capturing mechanism - // (e.g. `num_bump_calls` will not be incremented), since the 'unglued' - // tokens due not exist in the original `TokenStream`. + // During parsing we may sometimes need to "unglue" a glued token into two + // or three component tokens (e.g. `>>` into `>` and `>`, or `>>=` into `>` + // and `>` and `=`), so the parser can consume them one at a time. This + // process bypasses the normal capturing mechanism (e.g. `num_bump_calls` + // will not be incremented), since the "unglued" tokens due not exist in + // the original `TokenStream`. // - // If we end up consuming both unglued tokens, this is not an issue. We'll - // end up capturing the single 'glued' token. + // If we end up consuming all the component tokens, this is not an issue, + // because we'll end up capturing the single "glued" token. // - // However, sometimes we may want to capture just the first 'unglued' + // However, sometimes we may want to capture not all of the original // token. For example, capturing the `Vec` in `Option>` // requires us to unglue the trailing `>>` token. The `break_last_token` - // field is used to track this token. It gets appended to the captured + // field is used to track these tokens. They get appended to the captured // stream when we evaluate a `LazyAttrTokenStream`. - break_last_token: bool, + // + // This value is always 0, 1, or 2. It can only reach 2 when splitting + // `>>=` or `<<=`. + break_last_token: u32, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error /// appropriately. @@ -453,7 +457,7 @@ pub fn new( expected_tokens: Vec::new(), token_cursor: TokenCursor { tree_cursor: stream.into_trees(), stack: Vec::new() }, num_bump_calls: 0, - break_last_token: false, + break_last_token: 0, unmatched_angle_bracket_count: 0, angle_bracket_nesting: 0, last_unexpected_token_span: None, @@ -773,7 +777,7 @@ fn break_and_eat(&mut self, expected: TokenKind) -> bool { self.bump(); return true; } - match self.token.kind.break_two_token_op() { + match self.token.kind.break_two_token_op(1) { Some((first, second)) if first == expected => { let first_span = self.psess.source_map().start_point(self.token.span); let second_span = self.token.span.with_lo(first_span.hi()); @@ -783,8 +787,8 @@ fn break_and_eat(&mut self, expected: TokenKind) -> bool { // // If we consume any additional tokens, then this token // is not needed (we'll capture the entire 'glued' token), - // and `bump` will set this field to `None` - self.break_last_token = true; + // and `bump` will set this field to 0. + self.break_last_token += 1; // Use the spacing of the glued token as the spacing of the // unglued second token. self.bump_with((Token::new(second, second_span), self.token_spacing)); @@ -1148,10 +1152,9 @@ pub fn bump(&mut self) { // than `.0`/`.1` access. let mut next = self.token_cursor.inlined_next(); self.num_bump_calls += 1; - // We've retrieved an token from the underlying - // cursor, so we no longer need to worry about - // an unglued token. See `break_and_eat` for more details - self.break_last_token = false; + // We got a token from the underlying cursor and no longer need to + // worry about an unglued token. See `break_and_eat` for more details. + self.break_last_token = 0; if next.0.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. let fallback_span = self.token.span; diff --git a/tests/ui/macros/break-last-token-twice.rs b/tests/ui/macros/break-last-token-twice.rs new file mode 100644 index 00000000000..791f349ab38 --- /dev/null +++ b/tests/ui/macros/break-last-token-twice.rs @@ -0,0 +1,16 @@ +//@ check-pass + +macro_rules! m { + (static $name:ident: $t:ty = $e:expr) => { + let $name: $t = $e; + } +} + +fn main() { + m! { + // Tricky: the trailing `>>=` token here is broken twice: + // - into `>` and `>=` + // - then the `>=` is broken into `>` and `=` + static _x: Vec>= vec![] + } +} From f7735f99cf1f29e7ec68331f0badacb390a00280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 23 Sep 2024 10:02:04 +0200 Subject: [PATCH 05/11] Add rustfmt 2024 reformatting to git blame ignore --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index ec76a1a42ef..8612a40923a 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -27,3 +27,5 @@ ec2cc761bc7067712ecc7734502f703fe3b024c8 84ac80f1921afc243d71fd0caaa4f2838c294102 # bless mir-opt tests to add `copy` 99cb0c6bc399fb94a0ddde7e9b38e9c00d523bad +# reformat with rustfmt edition 2024 +c682aa162b0d41e21cc6748f4fecfe01efb69d1f From 5c1c72572479afe98734d5f78fa862abe662c41a Mon Sep 17 00:00:00 2001 From: joboet Date: Thu, 15 Aug 2024 13:28:02 +0200 Subject: [PATCH 06/11] std: implement the `random` feature Implements the ACP https://github.com/rust-lang/libs-team/issues/393. --- library/core/src/lib.rs | 2 + library/core/src/random.rs | 62 ++++ library/std/src/hash/random.rs | 5 +- library/std/src/lib.rs | 4 + library/std/src/random.rs | 100 ++++++ library/std/src/sys/mod.rs | 1 + library/std/src/sys/pal/hermit/mod.rs | 14 - .../std/src/sys/pal/sgx/abi/usercalls/mod.rs | 4 +- library/std/src/sys/pal/sgx/mod.rs | 18 -- library/std/src/sys/pal/solid/mod.rs | 10 - library/std/src/sys/pal/teeos/mod.rs | 3 - library/std/src/sys/pal/teeos/rand.rs | 21 -- library/std/src/sys/pal/uefi/mod.rs | 33 -- library/std/src/sys/pal/unix/mod.rs | 2 - library/std/src/sys/pal/unix/rand.rs | 302 ------------------ library/std/src/sys/pal/unsupported/common.rs | 4 - library/std/src/sys/pal/wasi/helpers.rs | 12 +- library/std/src/sys/pal/wasi/mod.rs | 2 +- library/std/src/sys/pal/wasip2/mod.rs | 2 +- library/std/src/sys/pal/windows/mod.rs | 2 - library/std/src/sys/pal/windows/pipe.rs | 7 +- library/std/src/sys/pal/windows/rand.rs | 27 -- library/std/src/sys/pal/zkvm/mod.rs | 8 - library/std/src/sys/random/apple.rs | 22 ++ library/std/src/sys/random/espidf.rs | 9 + library/std/src/sys/random/fuchsia.rs | 13 + library/std/src/sys/random/hermit.rs | 7 + library/std/src/sys/random/horizon.rs | 7 + library/std/src/sys/random/linux.rs | 170 ++++++++++ library/std/src/sys/random/mod.rs | 98 ++++++ library/std/src/sys/random/netbsd.rs | 19 ++ library/std/src/sys/random/redox.rs | 12 + library/std/src/sys/random/sgx.rs | 67 ++++ library/std/src/sys/random/solid.rs | 8 + library/std/src/sys/random/teeos.rs | 7 + library/std/src/sys/random/uefi.rs | 27 ++ library/std/src/sys/random/unix.rs | 33 ++ library/std/src/sys/random/unix_legacy.rs | 20 ++ library/std/src/sys/random/unsupported.rs | 15 + library/std/src/sys/random/vxworks.rs | 25 ++ library/std/src/sys/random/wasi.rs | 5 + library/std/src/sys/random/windows.rs | 20 ++ library/std/src/sys/random/zkvm.rs | 21 ++ 43 files changed, 786 insertions(+), 464 deletions(-) create mode 100644 library/core/src/random.rs create mode 100644 library/std/src/random.rs delete mode 100644 library/std/src/sys/pal/teeos/rand.rs delete mode 100644 library/std/src/sys/pal/unix/rand.rs delete mode 100644 library/std/src/sys/pal/windows/rand.rs create mode 100644 library/std/src/sys/random/apple.rs create mode 100644 library/std/src/sys/random/espidf.rs create mode 100644 library/std/src/sys/random/fuchsia.rs create mode 100644 library/std/src/sys/random/hermit.rs create mode 100644 library/std/src/sys/random/horizon.rs create mode 100644 library/std/src/sys/random/linux.rs create mode 100644 library/std/src/sys/random/mod.rs create mode 100644 library/std/src/sys/random/netbsd.rs create mode 100644 library/std/src/sys/random/redox.rs create mode 100644 library/std/src/sys/random/sgx.rs create mode 100644 library/std/src/sys/random/solid.rs create mode 100644 library/std/src/sys/random/teeos.rs create mode 100644 library/std/src/sys/random/uefi.rs create mode 100644 library/std/src/sys/random/unix.rs create mode 100644 library/std/src/sys/random/unix_legacy.rs create mode 100644 library/std/src/sys/random/unsupported.rs create mode 100644 library/std/src/sys/random/vxworks.rs create mode 100644 library/std/src/sys/random/wasi.rs create mode 100644 library/std/src/sys/random/windows.rs create mode 100644 library/std/src/sys/random/zkvm.rs diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index a30b57c19d4..b4393889e75 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -394,6 +394,8 @@ pub mod assert_matches { #[unstable(feature = "core_pattern_types", issue = "123646")] pub mod pat; pub mod pin; +#[unstable(feature = "random", issue = "none")] +pub mod random; #[unstable(feature = "new_range_api", issue = "125687")] pub mod range; pub mod result; diff --git a/library/core/src/random.rs b/library/core/src/random.rs new file mode 100644 index 00000000000..9e0d0d0c58b --- /dev/null +++ b/library/core/src/random.rs @@ -0,0 +1,62 @@ +//! Random value generation. +//! +//! The [`Random`] trait allows generating a random value for a type using a +//! given [`RandomSource`]. + +/// A source of randomness. +#[unstable(feature = "random", issue = "none")] +pub trait RandomSource { + /// Fills `bytes` with random bytes. + fn fill_bytes(&mut self, bytes: &mut [u8]); +} + +/// A trait for getting a random value for a type. +/// +/// **Warning:** Be careful when manipulating random values! The +/// [`random`](Random::random) method on integers samples them with a uniform +/// distribution, so a value of 1 is just as likely as [`i32::MAX`]. By using +/// modulo operations, some of the resulting values can become more likely than +/// others. Use audited crates when in doubt. +#[unstable(feature = "random", issue = "none")] +pub trait Random: Sized { + /// Generates a random value. + fn random(source: &mut (impl RandomSource + ?Sized)) -> Self; +} + +impl Random for bool { + fn random(source: &mut (impl RandomSource + ?Sized)) -> Self { + u8::random(source) & 1 == 1 + } +} + +macro_rules! impl_primitive { + ($t:ty) => { + impl Random for $t { + /// Generates a random value. + /// + /// **Warning:** Be careful when manipulating the resulting value! This + /// method samples according to a uniform distribution, so a value of 1 is + /// just as likely as [`MAX`](Self::MAX). By using modulo operations, some + /// values can become more likely than others. Use audited crates when in + /// doubt. + fn random(source: &mut (impl RandomSource + ?Sized)) -> Self { + let mut bytes = (0 as Self).to_ne_bytes(); + source.fill_bytes(&mut bytes); + Self::from_ne_bytes(bytes) + } + } + }; +} + +impl_primitive!(u8); +impl_primitive!(i8); +impl_primitive!(u16); +impl_primitive!(i16); +impl_primitive!(u32); +impl_primitive!(i32); +impl_primitive!(u64); +impl_primitive!(i64); +impl_primitive!(u128); +impl_primitive!(i128); +impl_primitive!(usize); +impl_primitive!(isize); diff --git a/library/std/src/hash/random.rs b/library/std/src/hash/random.rs index 8ef45172eac..40f3a90f60c 100644 --- a/library/std/src/hash/random.rs +++ b/library/std/src/hash/random.rs @@ -10,7 +10,8 @@ #[allow(deprecated)] use super::{BuildHasher, Hasher, SipHasher13}; use crate::cell::Cell; -use crate::{fmt, sys}; +use crate::fmt; +use crate::sys::random::hashmap_random_keys; /// `RandomState` is the default state for [`HashMap`] types. /// @@ -65,7 +66,7 @@ pub fn new() -> RandomState { // increment one of the seeds on every RandomState creation, giving // every corresponding HashMap a different iteration order. thread_local!(static KEYS: Cell<(u64, u64)> = { - Cell::new(sys::hashmap_random_keys()) + Cell::new(hashmap_random_keys()) }); KEYS.with(|keys| { diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index 2add88da9a7..ce05d816604 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -318,6 +318,7 @@ // // Library features (core): // tidy-alphabetical-start +#![feature(array_chunks)] #![feature(c_str_module)] #![feature(char_internals)] #![feature(clone_to_uninit)] @@ -348,6 +349,7 @@ #![feature(prelude_2024)] #![feature(ptr_as_uninit)] #![feature(ptr_mask)] +#![feature(random)] #![feature(slice_internals)] #![feature(slice_ptr_get)] #![feature(slice_range)] @@ -595,6 +597,8 @@ #[unstable(feature = "anonymous_pipe", issue = "127154")] pub mod pipe; pub mod process; +#[unstable(feature = "random", issue = "none")] +pub mod random; pub mod sync; pub mod time; diff --git a/library/std/src/random.rs b/library/std/src/random.rs new file mode 100644 index 00000000000..24d1dac4dae --- /dev/null +++ b/library/std/src/random.rs @@ -0,0 +1,100 @@ +//! Random value generation. +//! +//! The [`Random`] trait allows generating a random value for a type using a +//! given [`RandomSource`]. + +#[unstable(feature = "random", issue = "none")] +pub use core::random::*; + +use crate::sys::random as sys; + +/// The default random source. +/// +/// This asks the system for random data suitable for cryptographic purposes +/// such as key generation. If security is a concern, consult the platform +/// documentation below for the specific guarantees your target provides. +/// +/// The high quality of randomness provided by this source means it can be quite +/// slow. If you need a large quantity of random numbers and security is not a +/// concern, consider using an alternative random number generator (potentially +/// seeded from this one). +/// +/// # Underlying sources +/// +/// Platform | Source +/// -----------------------|--------------------------------------------------------------- +/// Linux | [`getrandom`] or [`/dev/urandom`] after polling `/dev/random` +/// Windows | [`ProcessPrng`] +/// macOS and other UNIXes | [`getentropy`] +/// other Apple platforms | `CCRandomGenerateBytes` +/// ESP-IDF | [`esp_fill_random`] +/// Fuchsia | [`cprng_draw`] +/// Hermit | `read_entropy` +/// Horizon | `getrandom` shim +/// Hurd, L4Re, QNX | `/dev/urandom` +/// NetBSD before 10.0 | [`kern.arandom`] +/// Redox | `/scheme/rand` +/// SGX | [`rdrand`] +/// SOLID | `SOLID_RNG_SampleRandomBytes` +/// TEEOS | `TEE_GenerateRandom` +/// UEFI | [`EFI_RNG_PROTOCOL`] +/// VxWorks | `randABytes` after waiting for `randSecure` to become ready +/// WASI | `random_get` +/// ZKVM | `sys_rand` +/// +/// **Disclaimer:** The sources used might change over time. +/// +/// [`getrandom`]: https://www.man7.org/linux/man-pages/man2/getrandom.2.html +/// [`/dev/urandom`]: https://www.man7.org/linux/man-pages/man4/random.4.html +/// [`ProcessPrng`]: https://learn.microsoft.com/en-us/windows/win32/seccng/processprng +/// [`getentropy`]: https://pubs.opengroup.org/onlinepubs/9799919799/functions/getentropy.html +/// [`esp_fill_random`]: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/random.html#_CPPv415esp_fill_randomPv6size_t +/// [`cprng_draw`]: https://fuchsia.dev/reference/syscalls/cprng_draw +/// [`kern.arandom`]: https://man.netbsd.org/rnd.4 +/// [`rdrand`]: https://en.wikipedia.org/wiki/RDRAND +/// [`EFI_RNG_PROTOCOL`]: https://uefi.org/specs/UEFI/2.10/37_Secure_Technologies.html#random-number-generator-protocol +#[derive(Default, Debug, Clone, Copy)] +#[unstable(feature = "random", issue = "none")] +pub struct DefaultRandomSource; + +#[unstable(feature = "random", issue = "none")] +impl RandomSource for DefaultRandomSource { + fn fill_bytes(&mut self, bytes: &mut [u8]) { + sys::fill_bytes(bytes) + } +} + +/// Generates a random value with the default random source. +/// +/// This is a convenience function for `T::random(&mut DefaultRandomSource)` and +/// will sample according to the same distribution as the underlying [`Random`] +/// trait implementation. +/// +/// **Warning:** Be careful when manipulating random values! The +/// [`random`](Random::random) method on integers samples them with a uniform +/// distribution, so a value of 1 is just as likely as [`i32::MAX`]. By using +/// modulo operations, some of the resulting values can become more likely than +/// others. Use audited crates when in doubt. +/// +/// # Examples +/// +/// Generating a [version 4/variant 1 UUID] represented as text: +/// ``` +/// #![feature(random)] +/// +/// use std::random::random; +/// +/// let bits = random::(); +/// let g1 = (bits >> 96) as u32; +/// let g2 = (bits >> 80) as u16; +/// let g3 = (0x4000 | (bits >> 64) & 0x0fff) as u16; +/// let g4 = (0x8000 | (bits >> 48) & 0x3fff) as u16; +/// let g5 = (bits & 0xffffffffffff) as u64; +/// let uuid = format!("{g1:08x}-{g2:04x}-{g3:04x}-{g4:04x}-{g5:012x}"); +/// println!("{uuid}"); +/// ``` +/// +/// [version 4/variant 1 UUID]: https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_(random) +pub fn random() -> T { + T::random(&mut DefaultRandomSource) +} diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs index 96d6f2c87c4..df25b84fbbe 100644 --- a/library/std/src/sys/mod.rs +++ b/library/std/src/sys/mod.rs @@ -15,6 +15,7 @@ pub mod exit_guard; pub mod os_str; pub mod path; +pub mod random; pub mod sync; pub mod thread_local; diff --git a/library/std/src/sys/pal/hermit/mod.rs b/library/std/src/sys/pal/hermit/mod.rs index 1f2e5d9469f..f49ef947174 100644 --- a/library/std/src/sys/pal/hermit/mod.rs +++ b/library/std/src/sys/pal/hermit/mod.rs @@ -52,20 +52,6 @@ pub fn abort_internal() -> ! { unsafe { hermit_abi::abort() } } -pub fn hashmap_random_keys() -> (u64, u64) { - let mut buf = [0; 16]; - let mut slice = &mut buf[..]; - while !slice.is_empty() { - let res = cvt(unsafe { hermit_abi::read_entropy(slice.as_mut_ptr(), slice.len(), 0) }) - .expect("failed to generate random hashmap keys"); - slice = &mut slice[res as usize..]; - } - - let key1 = buf[..8].try_into().unwrap(); - let key2 = buf[8..].try_into().unwrap(); - (u64::from_ne_bytes(key1), u64::from_ne_bytes(key2)) -} - // This function is needed by the panic runtime. The symbol is named in // pre-link args for the target specification, so keep that in sync. #[cfg(not(test))] diff --git a/library/std/src/sys/pal/sgx/abi/usercalls/mod.rs b/library/std/src/sys/pal/sgx/abi/usercalls/mod.rs index def1ccdf81a..90b9b59471a 100644 --- a/library/std/src/sys/pal/sgx/abi/usercalls/mod.rs +++ b/library/std/src/sys/pal/sgx/abi/usercalls/mod.rs @@ -1,6 +1,6 @@ use crate::cmp; use crate::io::{Error as IoError, ErrorKind, IoSlice, IoSliceMut, Result as IoResult}; -use crate::sys::rand::rdrand64; +use crate::random::{DefaultRandomSource, Random}; use crate::time::{Duration, Instant}; pub(crate) mod alloc; @@ -164,7 +164,7 @@ pub fn wait(event_mask: u64, mut timeout: u64) -> IoResult { // trusted to ensure accurate timeouts. if let Ok(timeout_signed) = i64::try_from(timeout) { let tenth = timeout_signed / 10; - let deviation = (rdrand64() as i64).checked_rem(tenth).unwrap_or(0); + let deviation = i64::random(&mut DefaultRandomSource).checked_rem(tenth).unwrap_or(0); timeout = timeout_signed.saturating_add(deviation) as _; } } diff --git a/library/std/src/sys/pal/sgx/mod.rs b/library/std/src/sys/pal/sgx/mod.rs index 8d29b2ec619..586ccd18c2f 100644 --- a/library/std/src/sys/pal/sgx/mod.rs +++ b/library/std/src/sys/pal/sgx/mod.rs @@ -132,24 +132,6 @@ pub extern "C" fn __rust_abort() { abort_internal(); } -pub mod rand { - pub fn rdrand64() -> u64 { - unsafe { - let mut ret: u64 = 0; - for _ in 0..10 { - if crate::arch::x86_64::_rdrand64_step(&mut ret) == 1 { - return ret; - } - } - rtabort!("Failed to obtain random data"); - } - } -} - -pub fn hashmap_random_keys() -> (u64, u64) { - (self::rand::rdrand64(), self::rand::rdrand64()) -} - pub use crate::sys_common::{AsInner, FromInner, IntoInner}; pub trait TryIntoInner: Sized { diff --git a/library/std/src/sys/pal/solid/mod.rs b/library/std/src/sys/pal/solid/mod.rs index 6ebcf5b7c48..d41042be518 100644 --- a/library/std/src/sys/pal/solid/mod.rs +++ b/library/std/src/sys/pal/solid/mod.rs @@ -62,13 +62,3 @@ pub fn decode_error_kind(code: i32) -> crate::io::ErrorKind { pub fn abort_internal() -> ! { unsafe { libc::abort() } } - -pub fn hashmap_random_keys() -> (u64, u64) { - unsafe { - let mut out = crate::mem::MaybeUninit::<[u64; 2]>::uninit(); - let result = abi::SOLID_RNG_SampleRandomBytes(out.as_mut_ptr() as *mut u8, 16); - assert_eq!(result, 0, "SOLID_RNG_SampleRandomBytes failed: {result}"); - let [x1, x2] = out.assume_init(); - (x1, x2) - } -} diff --git a/library/std/src/sys/pal/teeos/mod.rs b/library/std/src/sys/pal/teeos/mod.rs index 00e38604240..60a227afb84 100644 --- a/library/std/src/sys/pal/teeos/mod.rs +++ b/library/std/src/sys/pal/teeos/mod.rs @@ -6,8 +6,6 @@ #![allow(unused_variables)] #![allow(dead_code)] -pub use self::rand::hashmap_random_keys; - #[path = "../unsupported/args.rs"] pub mod args; #[path = "../unsupported/env.rs"] @@ -23,7 +21,6 @@ pub mod pipe; #[path = "../unsupported/process.rs"] pub mod process; -mod rand; pub mod stdio; pub mod thread; #[allow(non_upper_case_globals)] diff --git a/library/std/src/sys/pal/teeos/rand.rs b/library/std/src/sys/pal/teeos/rand.rs deleted file mode 100644 index b45c3bb40e7..00000000000 --- a/library/std/src/sys/pal/teeos/rand.rs +++ /dev/null @@ -1,21 +0,0 @@ -pub fn hashmap_random_keys() -> (u64, u64) { - const KEY_LEN: usize = core::mem::size_of::(); - - let mut v = [0u8; KEY_LEN * 2]; - imp::fill_bytes(&mut v); - - let key1 = v[0..KEY_LEN].try_into().unwrap(); - let key2 = v[KEY_LEN..].try_into().unwrap(); - - (u64::from_ne_bytes(key1), u64::from_ne_bytes(key2)) -} - -mod imp { - extern "C" { - fn TEE_GenerateRandom(randomBuffer: *mut core::ffi::c_void, randomBufferLen: libc::size_t); - } - - pub fn fill_bytes(v: &mut [u8]) { - unsafe { TEE_GenerateRandom(v.as_mut_ptr() as _, v.len() * crate::mem::size_of::()) } - } -} diff --git a/library/std/src/sys/pal/uefi/mod.rs b/library/std/src/sys/pal/uefi/mod.rs index ac22f4ded88..c0ab52f650a 100644 --- a/library/std/src/sys/pal/uefi/mod.rs +++ b/library/std/src/sys/pal/uefi/mod.rs @@ -179,39 +179,6 @@ pub extern "C" fn __rust_abort() { abort_internal(); } -#[inline] -pub fn hashmap_random_keys() -> (u64, u64) { - get_random().unwrap() -} - -fn get_random() -> Option<(u64, u64)> { - use r_efi::protocols::rng; - - let mut buf = [0u8; 16]; - let handles = helpers::locate_handles(rng::PROTOCOL_GUID).ok()?; - for handle in handles { - if let Ok(protocol) = helpers::open_protocol::(handle, rng::PROTOCOL_GUID) { - let r = unsafe { - ((*protocol.as_ptr()).get_rng)( - protocol.as_ptr(), - crate::ptr::null_mut(), - buf.len(), - buf.as_mut_ptr(), - ) - }; - if r.is_error() { - continue; - } else { - return Some(( - u64::from_le_bytes(buf[..8].try_into().ok()?), - u64::from_le_bytes(buf[8..].try_into().ok()?), - )); - } - } - } - None -} - /// Disable access to BootServices if `EVT_SIGNAL_EXIT_BOOT_SERVICES` is signaled extern "efiapi" fn exit_boot_service_handler(_e: r_efi::efi::Event, _ctx: *mut crate::ffi::c_void) { uefi::env::disable_boot_services(); diff --git a/library/std/src/sys/pal/unix/mod.rs b/library/std/src/sys/pal/unix/mod.rs index e8428eccb16..1c9159e5fba 100644 --- a/library/std/src/sys/pal/unix/mod.rs +++ b/library/std/src/sys/pal/unix/mod.rs @@ -1,6 +1,5 @@ #![allow(missing_docs, nonstandard_style)] -pub use self::rand::hashmap_random_keys; use crate::io::ErrorKind; #[cfg(not(target_os = "espidf"))] @@ -26,7 +25,6 @@ pub mod os; pub mod pipe; pub mod process; -pub mod rand; pub mod stack_overflow; pub mod stdio; pub mod thread; diff --git a/library/std/src/sys/pal/unix/rand.rs b/library/std/src/sys/pal/unix/rand.rs deleted file mode 100644 index cc0852aab43..00000000000 --- a/library/std/src/sys/pal/unix/rand.rs +++ /dev/null @@ -1,302 +0,0 @@ -pub fn hashmap_random_keys() -> (u64, u64) { - const KEY_LEN: usize = core::mem::size_of::(); - - let mut v = [0u8; KEY_LEN * 2]; - if let Err(err) = read(&mut v) { - panic!("failed to retrieve random hash map seed: {err}"); - } - - let key1 = v[0..KEY_LEN].try_into().unwrap(); - let key2 = v[KEY_LEN..].try_into().unwrap(); - - (u64::from_ne_bytes(key1), u64::from_ne_bytes(key2)) -} - -cfg_if::cfg_if! { - if #[cfg(any( - target_vendor = "apple", - target_os = "openbsd", - target_os = "emscripten", - target_os = "vita", - all(target_os = "netbsd", not(netbsd10)), - target_os = "fuchsia", - target_os = "vxworks", - ))] { - // Some systems have a syscall that directly retrieves random data. - // If that is guaranteed to be available, use it. - use imp::syscall as read; - } else { - // Otherwise, try the syscall to see if it exists only on some systems - // and fall back to reading from the random device otherwise. - fn read(bytes: &mut [u8]) -> crate::io::Result<()> { - use crate::fs::File; - use crate::io::Read; - use crate::sync::OnceLock; - - #[cfg(any( - target_os = "linux", - target_os = "android", - target_os = "espidf", - target_os = "horizon", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "solaris", - target_os = "illumos", - netbsd10, - ))] - if let Some(res) = imp::syscall(bytes) { - return res; - } - - const PATH: &'static str = if cfg!(target_os = "redox") { - "/scheme/rand" - } else { - "/dev/urandom" - }; - - static FILE: OnceLock = OnceLock::new(); - - FILE.get_or_try_init(|| File::open(PATH))?.read_exact(bytes) - } - } -} - -// All these systems a `getrandom` syscall. -// -// It is not guaranteed to be available, so return None to fallback to the file -// implementation. -#[cfg(any( - target_os = "linux", - target_os = "android", - target_os = "espidf", - target_os = "horizon", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "solaris", - target_os = "illumos", - netbsd10, -))] -mod imp { - use crate::io::{Error, Result}; - use crate::sync::atomic::{AtomicBool, Ordering}; - use crate::sys::os::errno; - - #[cfg(any(target_os = "linux", target_os = "android"))] - fn getrandom(buf: &mut [u8]) -> libc::ssize_t { - use crate::sys::weak::syscall; - - // A weak symbol allows interposition, e.g. for perf measurements that want to - // disable randomness for consistency. Otherwise, we'll try a raw syscall. - // (`getrandom` was added in glibc 2.25, musl 1.1.20, android API level 28) - syscall! { - fn getrandom( - buffer: *mut libc::c_void, - length: libc::size_t, - flags: libc::c_uint - ) -> libc::ssize_t - } - - // This provides the best quality random numbers available at the given moment - // without ever blocking, and is preferable to falling back to /dev/urandom. - static GRND_INSECURE_AVAILABLE: AtomicBool = AtomicBool::new(true); - if GRND_INSECURE_AVAILABLE.load(Ordering::Relaxed) { - let ret = unsafe { getrandom(buf.as_mut_ptr().cast(), buf.len(), libc::GRND_INSECURE) }; - if ret == -1 && errno() as libc::c_int == libc::EINVAL { - GRND_INSECURE_AVAILABLE.store(false, Ordering::Relaxed); - } else { - return ret; - } - } - - unsafe { getrandom(buf.as_mut_ptr().cast(), buf.len(), libc::GRND_NONBLOCK) } - } - - #[cfg(any( - target_os = "dragonfly", - target_os = "espidf", - target_os = "horizon", - target_os = "freebsd", - netbsd10, - target_os = "illumos", - target_os = "solaris" - ))] - fn getrandom(buf: &mut [u8]) -> libc::ssize_t { - unsafe { libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), 0) } - } - - pub fn syscall(v: &mut [u8]) -> Option> { - static GETRANDOM_UNAVAILABLE: AtomicBool = AtomicBool::new(false); - - if GETRANDOM_UNAVAILABLE.load(Ordering::Relaxed) { - return None; - } - - let mut read = 0; - while read < v.len() { - let result = getrandom(&mut v[read..]); - if result == -1 { - let err = errno() as libc::c_int; - if err == libc::EINTR { - continue; - } else if err == libc::ENOSYS || err == libc::EPERM { - // `getrandom` is not supported on the current system. - // - // Also fall back in case it is disabled by something like - // seccomp or inside of docker. - // - // If the `getrandom` syscall is not implemented in the current kernel version it should return an - // `ENOSYS` error. Docker also blocks the whole syscall inside unprivileged containers, and - // returns `EPERM` (instead of `ENOSYS`) when a program tries to invoke the syscall. Because of - // that we need to check for *both* `ENOSYS` and `EPERM`. - // - // Note that Docker's behavior is breaking other projects (notably glibc), so they're planning - // to update their filtering to return `ENOSYS` in a future release: - // - // https://github.com/moby/moby/issues/42680 - // - GETRANDOM_UNAVAILABLE.store(true, Ordering::Relaxed); - return None; - } else if err == libc::EAGAIN { - // getrandom has failed because it would have blocked as the - // non-blocking pool (urandom) has not been initialized in - // the kernel yet due to a lack of entropy. Fallback to - // reading from `/dev/urandom` which will return potentially - // insecure random data to avoid blocking applications which - // could depend on this call without ever knowing they do and - // don't have a work around. - return None; - } else { - return Some(Err(Error::from_raw_os_error(err))); - } - } else { - read += result as usize; - } - } - - Some(Ok(())) - } -} - -#[cfg(any( - target_os = "macos", // Supported since macOS 10.12+. - target_os = "openbsd", - target_os = "emscripten", - target_os = "vita", -))] -mod imp { - use crate::io::{Error, Result}; - - pub fn syscall(v: &mut [u8]) -> Result<()> { - // getentropy(2) permits a maximum buffer size of 256 bytes - for s in v.chunks_mut(256) { - let ret = unsafe { libc::getentropy(s.as_mut_ptr().cast(), s.len()) }; - if ret == -1 { - return Err(Error::last_os_error()); - } - } - - Ok(()) - } -} - -// On Apple platforms, `CCRandomGenerateBytes` and `SecRandomCopyBytes` simply -// call into `CCRandomCopyBytes` with `kCCRandomDefault`. `CCRandomCopyBytes` -// manages a CSPRNG which is seeded from the kernel's CSPRNG and which runs on -// its own thread accessed via GCD. This seems needlessly heavyweight for our purposes -// so we only use it when `getentropy` is blocked, which appears to be the case -// on all platforms except macOS (see #102643). -// -// `CCRandomGenerateBytes` is used instead of `SecRandomCopyBytes` because the former is accessible -// via `libSystem` (libc) while the other needs to link to `Security.framework`. -#[cfg(all(target_vendor = "apple", not(target_os = "macos")))] -mod imp { - use libc::size_t; - - use crate::ffi::{c_int, c_void}; - use crate::io::{Error, Result}; - - pub fn syscall(v: &mut [u8]) -> Result<()> { - extern "C" { - fn CCRandomGenerateBytes(bytes: *mut c_void, count: size_t) -> c_int; - } - - let ret = unsafe { CCRandomGenerateBytes(v.as_mut_ptr().cast(), v.len()) }; - if ret != -1 { Ok(()) } else { Err(Error::last_os_error()) } - } -} - -// FIXME: once the 10.x release becomes the minimum, this can be dropped for simplification. -#[cfg(all(target_os = "netbsd", not(netbsd10)))] -mod imp { - use crate::io::{Error, Result}; - use crate::ptr; - - pub fn syscall(v: &mut [u8]) -> Result<()> { - let mib = [libc::CTL_KERN, libc::KERN_ARND]; - // kern.arandom permits a maximum buffer size of 256 bytes - for s in v.chunks_mut(256) { - let mut s_len = s.len(); - let ret = unsafe { - libc::sysctl( - mib.as_ptr(), - mib.len() as libc::c_uint, - s.as_mut_ptr() as *mut _, - &mut s_len, - ptr::null(), - 0, - ) - }; - if ret == -1 { - return Err(Error::last_os_error()); - } else if s_len != s.len() { - // FIXME(joboet): this can't actually happen, can it? - panic!("read less bytes than requested from kern.arandom"); - } - } - - Ok(()) - } -} - -#[cfg(target_os = "fuchsia")] -mod imp { - use crate::io::Result; - - #[link(name = "zircon")] - extern "C" { - fn zx_cprng_draw(buffer: *mut u8, len: usize); - } - - pub fn syscall(v: &mut [u8]) -> Result<()> { - unsafe { zx_cprng_draw(v.as_mut_ptr(), v.len()) }; - Ok(()) - } -} - -#[cfg(target_os = "vxworks")] -mod imp { - use core::sync::atomic::AtomicBool; - use core::sync::atomic::Ordering::Relaxed; - - use crate::io::{Error, Result}; - - pub fn syscall(v: &mut [u8]) -> Result<()> { - static RNG_INIT: AtomicBool = AtomicBool::new(false); - while !RNG_INIT.load(Relaxed) { - let ret = unsafe { libc::randSecure() }; - if ret < 0 { - return Err(Error::last_os_error()); - } else if ret > 0 { - RNG_INIT.store(true, Relaxed); - break; - } - - unsafe { libc::usleep(10) }; - } - - let ret = unsafe { - libc::randABytes(v.as_mut_ptr() as *mut libc::c_uchar, v.len() as libc::c_int) - }; - if ret >= 0 { Ok(()) } else { Err(Error::last_os_error()) } - } -} diff --git a/library/std/src/sys/pal/unsupported/common.rs b/library/std/src/sys/pal/unsupported/common.rs index 76f80291f0e..34a76668383 100644 --- a/library/std/src/sys/pal/unsupported/common.rs +++ b/library/std/src/sys/pal/unsupported/common.rs @@ -27,7 +27,3 @@ pub fn decode_error_kind(_code: i32) -> crate::io::ErrorKind { pub fn abort_internal() -> ! { core::intrinsics::abort(); } - -pub fn hashmap_random_keys() -> (u64, u64) { - (1, 2) -} diff --git a/library/std/src/sys/pal/wasi/helpers.rs b/library/std/src/sys/pal/wasi/helpers.rs index 37ef17858cb..404747f0dc7 100644 --- a/library/std/src/sys/pal/wasi/helpers.rs +++ b/library/std/src/sys/pal/wasi/helpers.rs @@ -1,6 +1,6 @@ #![forbid(unsafe_op_in_unsafe_fn)] -use crate::{io as std_io, mem}; +use crate::io as std_io; #[inline] pub fn is_interrupted(errno: i32) -> bool { @@ -108,16 +108,6 @@ pub fn abort_internal() -> ! { unsafe { libc::abort() } } -pub fn hashmap_random_keys() -> (u64, u64) { - let mut ret = (0u64, 0u64); - unsafe { - let base = &mut ret as *mut (u64, u64) as *mut u8; - let len = mem::size_of_val(&ret); - wasi::random_get(base, len).expect("random_get failure"); - } - ret -} - #[inline] pub(crate) fn err2io(err: wasi::Errno) -> std_io::Error { std_io::Error::from_raw_os_error(err.raw().into()) diff --git a/library/std/src/sys/pal/wasi/mod.rs b/library/std/src/sys/pal/wasi/mod.rs index 8051021a588..5d54c790306 100644 --- a/library/std/src/sys/pal/wasi/mod.rs +++ b/library/std/src/sys/pal/wasi/mod.rs @@ -47,4 +47,4 @@ // then the compiler complains about conflicts. use helpers::err2io; -pub use helpers::{abort_internal, decode_error_kind, hashmap_random_keys, is_interrupted}; +pub use helpers::{abort_internal, decode_error_kind, is_interrupted}; diff --git a/library/std/src/sys/pal/wasip2/mod.rs b/library/std/src/sys/pal/wasip2/mod.rs index 546fadbe501..17b26543bd7 100644 --- a/library/std/src/sys/pal/wasip2/mod.rs +++ b/library/std/src/sys/pal/wasip2/mod.rs @@ -50,6 +50,6 @@ // then the compiler complains about conflicts. use helpers::err2io; -pub use helpers::{abort_internal, decode_error_kind, hashmap_random_keys, is_interrupted}; +pub use helpers::{abort_internal, decode_error_kind, is_interrupted}; mod cabi_realloc; diff --git a/library/std/src/sys/pal/windows/mod.rs b/library/std/src/sys/pal/windows/mod.rs index f5ed3e4628e..1ea253e5e52 100644 --- a/library/std/src/sys/pal/windows/mod.rs +++ b/library/std/src/sys/pal/windows/mod.rs @@ -1,7 +1,6 @@ #![allow(missing_docs, nonstandard_style)] #![forbid(unsafe_op_in_unsafe_fn)] -pub use self::rand::hashmap_random_keys; use crate::ffi::{OsStr, OsString}; use crate::io::ErrorKind; use crate::mem::MaybeUninit; @@ -27,7 +26,6 @@ pub mod os; pub mod pipe; pub mod process; -pub mod rand; pub mod stdio; pub mod thread; pub mod time; diff --git a/library/std/src/sys/pal/windows/pipe.rs b/library/std/src/sys/pal/windows/pipe.rs index 7d1b5aca1d5..d8200ef9ca4 100644 --- a/library/std/src/sys/pal/windows/pipe.rs +++ b/library/std/src/sys/pal/windows/pipe.rs @@ -2,12 +2,13 @@ use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut}; use crate::os::windows::prelude::*; use crate::path::Path; +use crate::random::{DefaultRandomSource, Random}; use crate::sync::atomic::AtomicUsize; use crate::sync::atomic::Ordering::Relaxed; +use crate::sys::c; use crate::sys::fs::{File, OpenOptions}; use crate::sys::handle::Handle; use crate::sys::pal::windows::api::{self, WinError}; -use crate::sys::{c, hashmap_random_keys}; use crate::sys_common::{FromInner, IntoInner}; use crate::{mem, ptr}; @@ -79,7 +80,7 @@ pub fn anon_pipe(ours_readable: bool, their_handle_inheritable: bool) -> io::Res name = format!( r"\\.\pipe\__rust_anonymous_pipe1__.{}.{}", c::GetCurrentProcessId(), - random_number() + random_number(), ); let wide_name = OsStr::new(&name).encode_wide().chain(Some(0)).collect::>(); let mut flags = c::FILE_FLAG_FIRST_PIPE_INSTANCE | c::FILE_FLAG_OVERLAPPED; @@ -214,7 +215,7 @@ fn random_number() -> usize { return N.fetch_add(1, Relaxed); } - N.store(hashmap_random_keys().0 as usize, Relaxed); + N.store(usize::random(&mut DefaultRandomSource), Relaxed); } } diff --git a/library/std/src/sys/pal/windows/rand.rs b/library/std/src/sys/pal/windows/rand.rs deleted file mode 100644 index e366bb99562..00000000000 --- a/library/std/src/sys/pal/windows/rand.rs +++ /dev/null @@ -1,27 +0,0 @@ -use core::{mem, ptr}; - -use crate::sys::c; - -#[cfg(not(target_vendor = "win7"))] -#[inline] -pub fn hashmap_random_keys() -> (u64, u64) { - let mut v = (0, 0); - let ret = unsafe { c::ProcessPrng(ptr::addr_of_mut!(v).cast::(), mem::size_of_val(&v)) }; - // ProcessPrng is documented as always returning `TRUE`. - // https://learn.microsoft.com/en-us/windows/win32/seccng/processprng#return-value - debug_assert_eq!(ret, c::TRUE); - v -} - -#[cfg(target_vendor = "win7")] -pub fn hashmap_random_keys() -> (u64, u64) { - use crate::ffi::c_void; - use crate::io; - - let mut v = (0, 0); - let ret = unsafe { - c::RtlGenRandom(ptr::addr_of_mut!(v).cast::(), mem::size_of_val(&v) as u32) - }; - - if ret != 0 { v } else { panic!("RNG broken: {}", io::Error::last_os_error()) } -} diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs index 20fdb7468a4..6ea05772029 100644 --- a/library/std/src/sys/pal/zkvm/mod.rs +++ b/library/std/src/sys/pal/zkvm/mod.rs @@ -60,11 +60,3 @@ pub fn decode_error_kind(_code: i32) -> crate::io::ErrorKind { pub fn abort_internal() -> ! { core::intrinsics::abort(); } - -pub fn hashmap_random_keys() -> (u64, u64) { - let mut buf = [0u32; 4]; - unsafe { - abi::sys_rand(buf.as_mut_ptr(), 4); - }; - ((buf[0] as u64) << 32 + buf[1] as u64, (buf[2] as u64) << 32 + buf[3] as u64) -} diff --git a/library/std/src/sys/random/apple.rs b/library/std/src/sys/random/apple.rs new file mode 100644 index 00000000000..09b6d0d51ab --- /dev/null +++ b/library/std/src/sys/random/apple.rs @@ -0,0 +1,22 @@ +//! Random data on non-macOS Apple platforms. +//! +//! Apple recommends the usage of `getentropy` in their security documentation[^1] +//! and mark it as being available in iOS 10.0, but we cannot use it on non-macOS +//! platforms as Apple in their *infinite wisdom* decided to consider this API +//! private, meaning its use will lead to App Store rejections (see #102643). +//! +//! Thus, we need to do the next best thing: +//! +//! Both `CCRandomGenerateBytes` and `SecRandomCopyBytes` simply call into +//! `CCRandomCopyBytes` with `kCCRandomDefault`. `CCRandomCopyBytes` manages a +//! CSPRNG which is seeded from the kernel's CSPRNG and which runs on its own +//! thread accessed via GCD (this is so wasteful...). Both are available on +//! iOS, but we use `CCRandomGenerateBytes` because it is accessible via +//! `libSystem` (libc) while the other needs to link to `Security.framework`. +//! +//! [^1]: + +pub fn fill_bytes(bytes: &mut [u8]) { + let ret = unsafe { libc::CCRandomGenerateBytes(bytes.as_mut_ptr().cast(), bytes.len()) }; + assert_eq!(ret, libc::kCCSuccess, "failed to generate random data"); +} diff --git a/library/std/src/sys/random/espidf.rs b/library/std/src/sys/random/espidf.rs new file mode 100644 index 00000000000..fd52cb5559c --- /dev/null +++ b/library/std/src/sys/random/espidf.rs @@ -0,0 +1,9 @@ +use crate::ffi::c_void; + +extern "C" { + fn esp_fill_random(buf: *mut c_void, len: usize); +} + +pub fn fill_bytes(bytes: &mut [u8]) { + unsafe { esp_fill_random(bytes.as_mut_ptr().cast(), bytes.len()) } +} diff --git a/library/std/src/sys/random/fuchsia.rs b/library/std/src/sys/random/fuchsia.rs new file mode 100644 index 00000000000..77d72b3c5b7 --- /dev/null +++ b/library/std/src/sys/random/fuchsia.rs @@ -0,0 +1,13 @@ +//! Random data generation using the Zircon kernel. +//! +//! Fuchsia, as always, is quite nice and provides exactly the API we need: +//! . + +#[link(name = "zircon")] +extern "C" { + fn zx_cprng_draw(buffer: *mut u8, len: usize); +} + +pub fn fill_bytes(bytes: &mut [u8]) { + unsafe { zx_cprng_draw(bytes.as_mut_ptr(), bytes.len()) } +} diff --git a/library/std/src/sys/random/hermit.rs b/library/std/src/sys/random/hermit.rs new file mode 100644 index 00000000000..92c0550d2d5 --- /dev/null +++ b/library/std/src/sys/random/hermit.rs @@ -0,0 +1,7 @@ +pub fn fill_bytes(mut bytes: &mut [u8]) { + while !bytes.is_empty() { + let res = unsafe { hermit_abi::read_entropy(bytes.as_mut_ptr(), bytes.len(), 0) }; + assert_ne!(res, -1, "failed to generate random data"); + bytes = &mut bytes[res as usize..]; + } +} diff --git a/library/std/src/sys/random/horizon.rs b/library/std/src/sys/random/horizon.rs new file mode 100644 index 00000000000..0be2eae20a7 --- /dev/null +++ b/library/std/src/sys/random/horizon.rs @@ -0,0 +1,7 @@ +pub fn fill_bytes(mut bytes: &mut [u8]) { + while !bytes.is_empty() { + let r = unsafe { libc::getrandom(bytes.as_mut_ptr().cast(), bytes.len(), 0) }; + assert_ne!(r, -1, "failed to generate random data"); + bytes = &mut bytes[r as usize..]; + } +} diff --git a/library/std/src/sys/random/linux.rs b/library/std/src/sys/random/linux.rs new file mode 100644 index 00000000000..4ede0af6494 --- /dev/null +++ b/library/std/src/sys/random/linux.rs @@ -0,0 +1,170 @@ +//! Random data generation with the Linux kernel. +//! +//! The first interface random data interface to be introduced on Linux were +//! the `/dev/random` and `/dev/urandom` special files. As paths can become +//! unreachable when inside a chroot and when the file descriptors are exhausted, +//! this was not enough to provide userspace with a reliable source of randomness, +//! so when the OpenBSD 5.6 introduced the `getentropy` syscall, Linux 3.17 got +//! its very own `getrandom` syscall to match.[^1] Unfortunately, even if our +//! minimum supported version were high enough, we still couldn't rely on the +//! syscall being available, as it is blocked in `seccomp` by default. +//! +//! The question is therefore which of the random sources to use. Historically, +//! the kernel contained two pools: the blocking and non-blocking pool. The +//! blocking pool used entropy estimation to limit the amount of available +//! bytes, while the non-blocking pool, once initialized using the blocking +//! pool, uses a CPRNG to return an unlimited number of random bytes. With a +//! strong enough CPRNG however, the entropy estimation didn't contribute that +//! much towards security while being an excellent vector for DoS attacs. Thus, +//! the blocking pool was removed in kernel version 5.6.[^2] That patch did not +//! magically increase the quality of the non-blocking pool, however, so we can +//! safely consider it strong enough even in older kernel versions and use it +//! unconditionally. +//! +//! One additional consideration to make is that the non-blocking pool is not +//! always initialized during early boot. We want the best quality of randomness +//! for the output of `DefaultRandomSource` so we simply wait until it is +//! initialized. When `HashMap` keys however, this represents a potential source +//! of deadlocks, as the additional entropy may only be generated once the +//! program makes forward progress. In that case, we just use the best random +//! data the system has available at the time. +//! +//! So in conclusion, we always want the output of the non-blocking pool, but +//! may need to wait until it is initalized. The default behaviour of `getrandom` +//! is to wait until the non-blocking pool is initialized and then draw from there, +//! so if `getrandom` is available, we use its default to generate the bytes. For +//! `HashMap`, however, we need to specify the `GRND_INSECURE` flags, but that +//! is only available starting with kernel version 5.6. Thus, if we detect that +//! the flag is unsupported, we try `GRND_NONBLOCK` instead, which will only +//! succeed if the pool is initialized. If it isn't, we fall back to the file +//! access method. +//! +//! The behaviour of `/dev/urandom` is inverse to that of `getrandom`: it always +//! yields data, even when the pool is not initialized. For generating `HashMap` +//! keys, this is not important, so we can use it directly. For secure data +//! however, we need to wait until initialization, which we can do by `poll`ing +//! `/dev/random`. +//! +//! TLDR: our fallback strategies are: +//! +//! Secure data | `HashMap` keys +//! --------------------------------------------|------------------ +//! getrandom(0) | getrandom(GRND_INSECURE) +//! poll("/dev/random") && read("/dev/urandom") | getrandom(GRND_NONBLOCK) +//! | read("/dev/urandom") +//! +//! [^1]: +//! [^2]: +//! +// FIXME(in 2040 or so): once the minimum kernel version is 5.6, remove the +// `GRND_NONBLOCK` fallback and use `/dev/random` instead of `/dev/urandom` +// when secure data is required. + +use crate::fs::File; +use crate::io::Read; +use crate::os::fd::AsRawFd; +use crate::sync::atomic::AtomicBool; +use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; +use crate::sync::OnceLock; +use crate::sys::pal::os::errno; +use crate::sys::pal::weak::syscall; + +fn getrandom(mut bytes: &mut [u8], insecure: bool) { + // A weak symbol allows interposition, e.g. for perf measurements that want to + // disable randomness for consistency. Otherwise, we'll try a raw syscall. + // (`getrandom` was added in glibc 2.25, musl 1.1.20, android API level 28) + syscall! { + fn getrandom( + buffer: *mut libc::c_void, + length: libc::size_t, + flags: libc::c_uint + ) -> libc::ssize_t + } + + static GETRANDOM_AVAILABLE: AtomicBool = AtomicBool::new(true); + static GRND_INSECURE_AVAILABLE: AtomicBool = AtomicBool::new(true); + static URANDOM_READY: AtomicBool = AtomicBool::new(false); + static DEVICE: OnceLock = OnceLock::new(); + + if GETRANDOM_AVAILABLE.load(Relaxed) { + loop { + if bytes.is_empty() { + return; + } + + let flags = if insecure { + if GRND_INSECURE_AVAILABLE.load(Relaxed) { + libc::GRND_INSECURE + } else { + libc::GRND_NONBLOCK + } + } else { + 0 + }; + + let ret = unsafe { getrandom(bytes.as_mut_ptr().cast(), bytes.len(), flags) }; + if ret != -1 { + bytes = &mut bytes[ret as usize..]; + } else { + match errno() { + libc::EINTR => continue, + // `GRND_INSECURE` is not available, try + // `GRND_NONBLOCK`. + libc::EINVAL if flags == libc::GRND_INSECURE => { + GRND_INSECURE_AVAILABLE.store(false, Relaxed); + continue; + } + // The pool is not initialized yet, fall back to + // /dev/urandom for now. + libc::EAGAIN if flags == libc::GRND_NONBLOCK => break, + // `getrandom` is unavailable or blocked by seccomp. + // Don't try it again and fall back to /dev/urandom. + libc::ENOSYS | libc::EPERM => { + GETRANDOM_AVAILABLE.store(false, Relaxed); + break; + } + _ => panic!("failed to generate random data"), + } + } + } + } + + // When we want cryptographic strength, we need to wait for the CPRNG-pool + // to become initialized. Do this by polling `/dev/random` until it is ready. + if !insecure { + if !URANDOM_READY.load(Acquire) { + let random = File::open("/dev/random").expect("failed to open /dev/random"); + let mut fd = libc::pollfd { fd: random.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + + while !URANDOM_READY.load(Acquire) { + let ret = unsafe { libc::poll(&mut fd, 1, -1) }; + match ret { + 1 => { + assert_eq!(fd.revents, libc::POLLIN); + URANDOM_READY.store(true, Release); + break; + } + -1 if errno() == libc::EINTR => continue, + _ => panic!("poll(\"/dev/random\") failed"), + } + } + } + } + + DEVICE + .get_or_try_init(|| File::open("/dev/urandom")) + .and_then(|mut dev| dev.read_exact(bytes)) + .expect("failed to generate random data"); +} + +pub fn fill_bytes(bytes: &mut [u8]) { + getrandom(bytes, false); +} + +pub fn hashmap_random_keys() -> (u64, u64) { + let mut bytes = [0; 16]; + getrandom(&mut bytes, true); + let k1 = u64::from_ne_bytes(bytes[..8].try_into().unwrap()); + let k2 = u64::from_ne_bytes(bytes[8..].try_into().unwrap()); + (k1, k2) +} diff --git a/library/std/src/sys/random/mod.rs b/library/std/src/sys/random/mod.rs new file mode 100644 index 00000000000..d39e78bb7b3 --- /dev/null +++ b/library/std/src/sys/random/mod.rs @@ -0,0 +1,98 @@ +cfg_if::cfg_if! { + // Tier 1 + if #[cfg(any(target_os = "linux", target_os = "android"))] { + mod linux; + pub use linux::{fill_bytes, hashmap_random_keys}; + } else if #[cfg(target_os = "windows")] { + mod windows; + pub use windows::fill_bytes; + } else if #[cfg(any( + target_os = "openbsd", + target_os = "freebsd", + target_os = "macos", + all(target_os = "netbsd", netbsd10), + target_os = "dragonfly", + target_os = "illumos", + target_os = "solaris", + target_os = "emscripten", + target_os = "vita", + target_os = "haiku", + ))] { + mod unix; + pub use unix::fill_bytes; + // Others, in alphabetical ordering. + } else if #[cfg(all(target_vendor = "apple", not(target_os = "macos")))] { + mod apple; + pub use apple::fill_bytes; + } else if #[cfg(target_os = "espidf")] { + mod espidf; + pub use espidf::fill_bytes; + } else if #[cfg(target_os = "fuchsia")] { + mod fuchsia; + pub use fuchsia::fill_bytes; + } else if #[cfg(target_os = "hermit")] { + mod hermit; + pub use hermit::fill_bytes; + } else if #[cfg(target_os = "horizon")] { + // FIXME: add getentropy to shim-3ds + mod horizon; + pub use horizon::fill_bytes; + } else if #[cfg(any( + target_os = "hurd", + target_os = "l4re", + target_os = "nto", + ))] { + mod unix_legacy; + pub use unix_legacy::fill_bytes; + } else if #[cfg(all(target_os = "netbsd", not(netbsd10)))] { + // FIXME: remove once NetBSD 10 is the minimum + mod netbsd; + pub use netbsd::fill_bytes; + } else if #[cfg(target_os = "redox")] { + mod redox; + pub use redox::fill_bytes; + } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] { + mod sgx; + pub use sgx::fill_bytes; + } else if #[cfg(target_os = "solid_asp3")] { + mod solid; + pub use solid::fill_bytes; + } else if #[cfg(target_os = "teeos")] { + mod teeos; + pub use teeos::fill_bytes; + } else if #[cfg(target_os = "uefi")] { + mod uefi; + pub use uefi::fill_bytes; + } else if #[cfg(target_os = "vxworks")] { + mod vxworks; + pub use vxworks::fill_bytes; + } else if #[cfg(target_os = "wasi")] { + mod wasi; + pub use wasi::fill_bytes; + } else if #[cfg(target_os = "zkvm")] { + mod zkvm; + pub use zkvm::fill_bytes; + } else if #[cfg(any( + all(target_family = "wasm", target_os = "unknown"), + target_os = "xous", + ))] { + // FIXME: finally remove std support for wasm32-unknown-unknown + // FIXME: add random data generation to xous + mod unsupported; + pub use unsupported::{fill_bytes, hashmap_random_keys}; + } +} + +#[cfg(not(any( + target_os = "linux", + target_os = "android", + all(target_family = "wasm", target_os = "unknown"), + target_os = "xous", +)))] +pub fn hashmap_random_keys() -> (u64, u64) { + let mut buf = [0; 16]; + fill_bytes(&mut buf); + let k1 = u64::from_ne_bytes(buf[..8].try_into().unwrap()); + let k2 = u64::from_ne_bytes(buf[8..].try_into().unwrap()); + (k1, k2) +} diff --git a/library/std/src/sys/random/netbsd.rs b/library/std/src/sys/random/netbsd.rs new file mode 100644 index 00000000000..2c5d9c72f30 --- /dev/null +++ b/library/std/src/sys/random/netbsd.rs @@ -0,0 +1,19 @@ +use crate::ptr; + +pub fn fill_bytes(bytes: &mut [u8]) { + let mib = [libc::CTL_KERN, libc::KERN_ARND]; + for chunk in bytes.chunks_mut(256) { + let mut len = chunk.len(); + let ret = unsafe { + libc::sysctl( + mib.as_ptr(), + mib.len() as libc::c_uint, + chunk.as_mut_ptr().cast(), + &mut len, + ptr::null(), + 0, + ) + }; + assert!(ret != -1 && len == chunk.len(), "failed to generate random data"); + } +} diff --git a/library/std/src/sys/random/redox.rs b/library/std/src/sys/random/redox.rs new file mode 100644 index 00000000000..b004335a351 --- /dev/null +++ b/library/std/src/sys/random/redox.rs @@ -0,0 +1,12 @@ +use crate::fs::File; +use crate::io::Read; +use crate::sync::OnceLock; + +static SCHEME: OnceLock = OnceLock::new(); + +pub fn fill_bytes(bytes: &mut [u8]) { + SCHEME + .get_or_try_init(|| File::open("/scheme/rand")) + .and_then(|mut scheme| scheme.read_exact(bytes)) + .expect("failed to generate random data"); +} diff --git a/library/std/src/sys/random/sgx.rs b/library/std/src/sys/random/sgx.rs new file mode 100644 index 00000000000..c3647a8df22 --- /dev/null +++ b/library/std/src/sys/random/sgx.rs @@ -0,0 +1,67 @@ +use crate::arch::x86_64::{_rdrand16_step, _rdrand32_step, _rdrand64_step}; + +const RETRIES: u32 = 10; + +fn fail() -> ! { + panic!("failed to generate random data"); +} + +fn rdrand64() -> u64 { + unsafe { + let mut ret: u64 = 0; + for _ in 0..RETRIES { + if _rdrand64_step(&mut ret) == 1 { + return ret; + } + } + + fail(); + } +} + +fn rdrand32() -> u32 { + unsafe { + let mut ret: u32 = 0; + for _ in 0..RETRIES { + if _rdrand32_step(&mut ret) == 1 { + return ret; + } + } + + fail(); + } +} + +fn rdrand16() -> u16 { + unsafe { + let mut ret: u16 = 0; + for _ in 0..RETRIES { + if _rdrand16_step(&mut ret) == 1 { + return ret; + } + } + + fail(); + } +} + +pub fn fill_bytes(bytes: &mut [u8]) { + let mut chunks = bytes.array_chunks_mut(); + for chunk in &mut chunks { + *chunk = rdrand64().to_ne_bytes(); + } + + let mut chunks = chunks.into_remainder().array_chunks_mut(); + for chunk in &mut chunks { + *chunk = rdrand32().to_ne_bytes(); + } + + let mut chunks = chunks.into_remainder().array_chunks_mut(); + for chunk in &mut chunks { + *chunk = rdrand16().to_ne_bytes(); + } + + if let [byte] = chunks.into_remainder() { + *byte = rdrand16() as u8; + } +} diff --git a/library/std/src/sys/random/solid.rs b/library/std/src/sys/random/solid.rs new file mode 100644 index 00000000000..545771150e2 --- /dev/null +++ b/library/std/src/sys/random/solid.rs @@ -0,0 +1,8 @@ +use crate::sys::pal::abi; + +pub fn fill_bytes(bytes: &mut [u8]) { + unsafe { + let result = abi::SOLID_RNG_SampleRandomBytes(bytes.as_mut_ptr(), bytes.len()); + assert_eq!(result, 0, "failed to generate random data"); + } +} diff --git a/library/std/src/sys/random/teeos.rs b/library/std/src/sys/random/teeos.rs new file mode 100644 index 00000000000..fd6b24e19e9 --- /dev/null +++ b/library/std/src/sys/random/teeos.rs @@ -0,0 +1,7 @@ +extern "C" { + fn TEE_GenerateRandom(randomBuffer: *mut core::ffi::c_void, randomBufferLen: libc::size_t); +} + +pub fn fill_bytes(bytes: &mut [u8]) { + unsafe { TEE_GenerateRandom(bytes.as_mut_ptr().cast(), bytes.len()) } +} diff --git a/library/std/src/sys/random/uefi.rs b/library/std/src/sys/random/uefi.rs new file mode 100644 index 00000000000..a4d29e66f38 --- /dev/null +++ b/library/std/src/sys/random/uefi.rs @@ -0,0 +1,27 @@ +use r_efi::protocols::rng; + +use crate::sys::pal::helpers; + +pub fn fill_bytes(bytes: &mut [u8]) { + let handles = + helpers::locate_handles(rng::PROTOCOL_GUID).expect("failed to generate random data"); + for handle in handles { + if let Ok(protocol) = helpers::open_protocol::(handle, rng::PROTOCOL_GUID) { + let r = unsafe { + ((*protocol.as_ptr()).get_rng)( + protocol.as_ptr(), + crate::ptr::null_mut(), + bytes.len(), + bytes.as_mut_ptr(), + ) + }; + if r.is_error() { + continue; + } else { + return; + } + } + } + + panic!("failed to generate random data"); +} diff --git a/library/std/src/sys/random/unix.rs b/library/std/src/sys/random/unix.rs new file mode 100644 index 00000000000..a56847e5541 --- /dev/null +++ b/library/std/src/sys/random/unix.rs @@ -0,0 +1,33 @@ +//! Random data generation through `getentropy`. +//! +//! Since issue 8 (2024), the POSIX specification mandates the existence of the +//! `getentropy` function, which fills a slice of up to `GETENTROPY_MAX` bytes +//! (256 on all known platforms) with random data. Luckily, this function has +//! already been available on quite some BSDs before that, having appeared with +//! OpenBSD 5.7 and spread from there: +//! +//! platform | version | man-page +//! -----------|---------|---------- +//! OpenBSD | 5.6 | +//! FreeBSD | 12.0 | +//! macOS | 10.12 | +//! NetBSD | 10.0 | +//! DragonFly | 6.1 | +//! Illumos | ? | +//! Solaris | ? | +//! +//! As it is standardized we use it whereever possible, even when `getrandom` is +//! also available. NetBSD even warns that "Applications should avoid getrandom +//! and use getentropy(2) instead; getrandom may be removed from a later +//! release."[^1]. +//! +//! [^1]: + +pub fn fill_bytes(bytes: &mut [u8]) { + // GETENTROPY_MAX isn't defined yet on most platforms, but it's mandated + // to be at least 256, so just use that as limit. + for chunk in bytes.chunks_mut(256) { + let r = unsafe { libc::getentropy(chunk.as_mut_ptr().cast(), chunk.len()) }; + assert_ne!(r, -1, "failed to generate random data"); + } +} diff --git a/library/std/src/sys/random/unix_legacy.rs b/library/std/src/sys/random/unix_legacy.rs new file mode 100644 index 00000000000..dd6be43c173 --- /dev/null +++ b/library/std/src/sys/random/unix_legacy.rs @@ -0,0 +1,20 @@ +//! Random data from `/dev/urandom` +//! +//! Before `getentropy` was standardized in 2024, UNIX didn't have a standardized +//! way of getting random data, so systems just followed the precedent set by +//! Linux and exposed random devices at `/dev/random` and `/dev/urandom`. Thus, +//! for the few systems that do not support `getentropy` yet, we just read from +//! the file. + +use crate::fs::File; +use crate::io::Read; +use crate::sync::OnceLock; + +static DEVICE: OnceLock = OnceLock::new(); + +pub fn fill_bytes(bytes: &mut [u8]) { + DEVICE + .get_or_try_init(|| File::open("/dev/urandom")) + .and_then(|mut dev| dev.read_exact(bytes)) + .expect("failed to generate random data"); +} diff --git a/library/std/src/sys/random/unsupported.rs b/library/std/src/sys/random/unsupported.rs new file mode 100644 index 00000000000..d68ce4a9e87 --- /dev/null +++ b/library/std/src/sys/random/unsupported.rs @@ -0,0 +1,15 @@ +use crate::ptr; + +pub fn fill_bytes(_: &mut [u8]) { + panic!("this target does not support random data generation"); +} + +pub fn hashmap_random_keys() -> (u64, u64) { + // Use allocation addresses for a bit of randomness. This isn't + // particularily secure, but there isn't really an alternative. + let stack = 0u8; + let heap = Box::new(0u8); + let k1 = ptr::from_ref(&stack).addr() as u64; + let k2 = ptr::from_ref(&*heap).addr() as u64; + (k1, k2) +} diff --git a/library/std/src/sys/random/vxworks.rs b/library/std/src/sys/random/vxworks.rs new file mode 100644 index 00000000000..d549ccebdb2 --- /dev/null +++ b/library/std/src/sys/random/vxworks.rs @@ -0,0 +1,25 @@ +use crate::sync::atomic::AtomicBool; +use crate::sync::atomic::Ordering::Relaxed; + +static RNG_INIT: AtomicBool = AtomicBool::new(false); + +pub fn fill_bytes(mut bytes: &mut [u8]) { + while !RNG_INIT.load(Relaxed) { + let ret = unsafe { libc::randSecure() }; + if ret < 0 { + panic!("failed to generate random data"); + } else if ret > 0 { + RNG_INIT.store(true, Relaxed); + break; + } + + unsafe { libc::usleep(10) }; + } + + while !bytes.is_empty() { + let len = bytes.len().try_into().unwrap_or(libc::c_int::MAX); + let ret = unsafe { libc::randABytes(bytes.as_mut_ptr(), len) }; + assert!(ret >= 0, "failed to generate random data"); + bytes = &mut bytes[len as usize..]; + } +} diff --git a/library/std/src/sys/random/wasi.rs b/library/std/src/sys/random/wasi.rs new file mode 100644 index 00000000000..d41da3751fc --- /dev/null +++ b/library/std/src/sys/random/wasi.rs @@ -0,0 +1,5 @@ +pub fn fill_bytes(bytes: &mut [u8]) { + unsafe { + wasi::random_get(bytes.as_mut_ptr(), bytes.len()).expect("failed to generate random data") + } +} diff --git a/library/std/src/sys/random/windows.rs b/library/std/src/sys/random/windows.rs new file mode 100644 index 00000000000..7566000f9e6 --- /dev/null +++ b/library/std/src/sys/random/windows.rs @@ -0,0 +1,20 @@ +use crate::sys::c; + +#[cfg(not(target_vendor = "win7"))] +#[inline] +pub fn fill_bytes(bytes: &mut [u8]) { + let ret = unsafe { c::ProcessPrng(bytes.as_mut_ptr(), bytes.len()) }; + // ProcessPrng is documented as always returning `TRUE`. + // https://learn.microsoft.com/en-us/windows/win32/seccng/processprng#return-value + debug_assert_eq!(ret, c::TRUE); +} + +#[cfg(target_vendor = "win7")] +pub fn fill_bytes(mut bytes: &mut [u8]) { + while !bytes.is_empty() { + let len = bytes.len().try_into().unwrap_or(u32::MAX); + let ret = unsafe { c::RtlGenRandom(bytes.as_mut_ptr().cast(), len) }; + assert_ne!(ret, 0, "failed to generate random data"); + bytes = &mut bytes[len as usize..]; + } +} diff --git a/library/std/src/sys/random/zkvm.rs b/library/std/src/sys/random/zkvm.rs new file mode 100644 index 00000000000..3011942f6b2 --- /dev/null +++ b/library/std/src/sys/random/zkvm.rs @@ -0,0 +1,21 @@ +use crate::sys::pal::abi; + +pub fn fill_bytes(bytes: &mut [u8]) { + let (pre, words, post) = unsafe { bytes.align_to_mut::() }; + if !words.is_empty() { + unsafe { + abi::sys_rand(words.as_mut_ptr(), words.len()); + } + } + + let mut buf = [0u32; 2]; + let len = (pre.len() + post.len() + size_of::() - 1) / size_of::(); + if len != 0 { + unsafe { abi::sys_rand(buf.as_mut_ptr(), len) }; + } + + let buf = buf.map(u32::to_ne_bytes); + let buf = buf.as_flattened(); + pre.copy_from_slice(&buf[..pre.len()]); + post.copy_from_slice(&buf[pre.len()..pre.len() + post.len()]); +} From b9d47cfa9b8f0805bfab2c254a02e598a906f102 Mon Sep 17 00:00:00 2001 From: joboet Date: Sat, 17 Aug 2024 16:54:01 +0200 Subject: [PATCH 07/11] std: switch to faster random sources on macOS and most BSDs --- library/std/src/random.rs | 31 +++++++++--------- library/std/src/sys/random/apple.rs | 21 ++++-------- library/std/src/sys/random/arc4random.rs | 34 ++++++++++++++++++++ library/std/src/sys/random/getentropy.rs | 17 ++++++++++ library/std/src/sys/random/linux.rs | 2 +- library/std/src/sys/random/mod.rs | 39 +++++++++++------------ library/std/src/sys/random/netbsd.rs | 19 ----------- library/std/src/sys/random/unix.rs | 33 ------------------- library/std/src/sys/random/unix_legacy.rs | 4 +-- 9 files changed, 94 insertions(+), 106 deletions(-) create mode 100644 library/std/src/sys/random/arc4random.rs create mode 100644 library/std/src/sys/random/getentropy.rs delete mode 100644 library/std/src/sys/random/netbsd.rs delete mode 100644 library/std/src/sys/random/unix.rs diff --git a/library/std/src/random.rs b/library/std/src/random.rs index 24d1dac4dae..48b55a55b2f 100644 --- a/library/std/src/random.rs +++ b/library/std/src/random.rs @@ -24,35 +24,34 @@ /// Platform | Source /// -----------------------|--------------------------------------------------------------- /// Linux | [`getrandom`] or [`/dev/urandom`] after polling `/dev/random` -/// Windows | [`ProcessPrng`] -/// macOS and other UNIXes | [`getentropy`] -/// other Apple platforms | `CCRandomGenerateBytes` -/// ESP-IDF | [`esp_fill_random`] -/// Fuchsia | [`cprng_draw`] +/// Windows | [`ProcessPrng`](https://learn.microsoft.com/en-us/windows/win32/seccng/processprng) +/// Apple | `CCRandomGenerateBytes` +/// DragonFly | [`arc4random_buf`](https://man.dragonflybsd.org/?command=arc4random§ion=ANY) +/// ESP-IDF | [`esp_fill_random`](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/random.html#_CPPv415esp_fill_randomPv6size_t) +/// FreeBSD | [`arc4random_buf`](https://man.freebsd.org/cgi/man.cgi?query=arc4random&apropos=0&sektion=0&manpath=FreeBSD+15.0-CURRENT&arch=default&format=html) +/// Fuchsia | [`cprng_draw`](https://fuchsia.dev/reference/syscalls/cprng_draw) +/// Haiku | `arc4random_buf` +/// Illumos | [`arc4random_buf`](https://www.illumos.org/man/3C/arc4random) +/// NetBSD | [`arc4random_buf`](https://man.netbsd.org/arc4random.3) +/// OpenBSD | [`arc4random_buf`](https://man.openbsd.org/arc4random.3) +/// Solaris | [`arc4random_buf`](https://docs.oracle.com/cd/E88353_01/html/E37843/arc4random-3c.html) +/// Vita | `arc4random_buf` /// Hermit | `read_entropy` /// Horizon | `getrandom` shim /// Hurd, L4Re, QNX | `/dev/urandom` -/// NetBSD before 10.0 | [`kern.arandom`] /// Redox | `/scheme/rand` -/// SGX | [`rdrand`] +/// SGX | [`rdrand`](https://en.wikipedia.org/wiki/RDRAND) /// SOLID | `SOLID_RNG_SampleRandomBytes` /// TEEOS | `TEE_GenerateRandom` -/// UEFI | [`EFI_RNG_PROTOCOL`] +/// UEFI | [`EFI_RNG_PROTOCOL`](https://uefi.org/specs/UEFI/2.10/37_Secure_Technologies.html#random-number-generator-protocol) /// VxWorks | `randABytes` after waiting for `randSecure` to become ready -/// WASI | `random_get` +/// WASI | [`random_get`](https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md#-random_getbuf-pointeru8-buf_len-size---result-errno) /// ZKVM | `sys_rand` /// /// **Disclaimer:** The sources used might change over time. /// /// [`getrandom`]: https://www.man7.org/linux/man-pages/man2/getrandom.2.html /// [`/dev/urandom`]: https://www.man7.org/linux/man-pages/man4/random.4.html -/// [`ProcessPrng`]: https://learn.microsoft.com/en-us/windows/win32/seccng/processprng -/// [`getentropy`]: https://pubs.opengroup.org/onlinepubs/9799919799/functions/getentropy.html -/// [`esp_fill_random`]: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/random.html#_CPPv415esp_fill_randomPv6size_t -/// [`cprng_draw`]: https://fuchsia.dev/reference/syscalls/cprng_draw -/// [`kern.arandom`]: https://man.netbsd.org/rnd.4 -/// [`rdrand`]: https://en.wikipedia.org/wiki/RDRAND -/// [`EFI_RNG_PROTOCOL`]: https://uefi.org/specs/UEFI/2.10/37_Secure_Technologies.html#random-number-generator-protocol #[derive(Default, Debug, Clone, Copy)] #[unstable(feature = "random", issue = "none")] pub struct DefaultRandomSource; diff --git a/library/std/src/sys/random/apple.rs b/library/std/src/sys/random/apple.rs index 09b6d0d51ab..417198c9d85 100644 --- a/library/std/src/sys/random/apple.rs +++ b/library/std/src/sys/random/apple.rs @@ -1,20 +1,13 @@ -//! Random data on non-macOS Apple platforms. +//! Random data on Apple platforms. //! -//! Apple recommends the usage of `getentropy` in their security documentation[^1] -//! and mark it as being available in iOS 10.0, but we cannot use it on non-macOS -//! platforms as Apple in their *infinite wisdom* decided to consider this API -//! private, meaning its use will lead to App Store rejections (see #102643). -//! -//! Thus, we need to do the next best thing: -//! -//! Both `CCRandomGenerateBytes` and `SecRandomCopyBytes` simply call into -//! `CCRandomCopyBytes` with `kCCRandomDefault`. `CCRandomCopyBytes` manages a -//! CSPRNG which is seeded from the kernel's CSPRNG and which runs on its own -//! thread accessed via GCD (this is so wasteful...). Both are available on -//! iOS, but we use `CCRandomGenerateBytes` because it is accessible via +//! `CCRandomGenerateBytes` calls into `CCRandomCopyBytes` with `kCCRandomDefault`. +//! `CCRandomCopyBytes` manages a CSPRNG which is seeded from the kernel's CSPRNG. +//! We use `CCRandomGenerateBytes` instead of `SecCopyBytes` because it is accessible via //! `libSystem` (libc) while the other needs to link to `Security.framework`. //! -//! [^1]: +//! Note that technically, `arc4random_buf` is available as well, but that calls +//! into the same system service anyway, and `CCRandomGenerateBytes` has been +//! proven to be App Store-compatible. pub fn fill_bytes(bytes: &mut [u8]) { let ret = unsafe { libc::CCRandomGenerateBytes(bytes.as_mut_ptr().cast(), bytes.len()) }; diff --git a/library/std/src/sys/random/arc4random.rs b/library/std/src/sys/random/arc4random.rs new file mode 100644 index 00000000000..32467e9ebaa --- /dev/null +++ b/library/std/src/sys/random/arc4random.rs @@ -0,0 +1,34 @@ +//! Random data generation with `arc4random_buf`. +//! +//! Contrary to its name, `arc4random` doesn't actually use the horribly-broken +//! RC4 cypher anymore, at least not on modern systems, but rather something +//! like ChaCha20 with continual reseeding from the OS. That makes it an ideal +//! source of large quantities of cryptographically secure data, which is exactly +//! what we need for `DefaultRandomSource`. Unfortunately, it's not available +//! on all UNIX systems, most notably Linux (until recently, but it's just a +//! wrapper for `getrandom`. Since we need to hook into `getrandom` directly +//! for `HashMap` keys anyway, we just keep our version). + +#[cfg(not(any( + target_os = "haiku", + target_os = "illumos", + target_os = "solaris", + target_os = "vita", +)))] +use libc::arc4random_buf; + +// FIXME: move these to libc (except Haiku, that one needs to link to libbsd.so). +#[cfg(any( + target_os = "haiku", // See https://git.haiku-os.org/haiku/tree/headers/compatibility/bsd/stdlib.h + target_os = "illumos", // See https://www.illumos.org/man/3C/arc4random + target_os = "solaris", // See https://docs.oracle.com/cd/E88353_01/html/E37843/arc4random-3c.html + target_os = "vita", // See https://github.com/vitasdk/newlib/blob/b89e5bc183b516945f9ee07eef483ecb916e45ff/newlib/libc/include/stdlib.h#L74 +))] +#[cfg_attr(target_os = "haiku", link(name = "bsd"))] +extern "C" { + fn arc4random_buf(buf: *mut core::ffi::c_void, nbytes: libc::size_t); +} + +pub fn fill_bytes(bytes: &mut [u8]) { + unsafe { arc4random_buf(bytes.as_mut_ptr().cast(), bytes.len()) } +} diff --git a/library/std/src/sys/random/getentropy.rs b/library/std/src/sys/random/getentropy.rs new file mode 100644 index 00000000000..110ac134c1f --- /dev/null +++ b/library/std/src/sys/random/getentropy.rs @@ -0,0 +1,17 @@ +//! Random data generation through `getentropy`. +//! +//! Since issue 8 (2024), the POSIX specification mandates the existence of the +//! `getentropy` function, which fills a slice of up to `GETENTROPY_MAX` bytes +//! (256 on all known platforms) with random data. Unfortunately, it's only +//! meant to be used to seed other CPRNGs, which we don't have, so we only use +//! it where `arc4random_buf` and friends aren't available or secure (currently +//! that's only the case on Emscripten). + +pub fn fill_bytes(bytes: &mut [u8]) { + // GETENTROPY_MAX isn't defined yet on most platforms, but it's mandated + // to be at least 256, so just use that as limit. + for chunk in bytes.chunks_mut(256) { + let r = unsafe { libc::getentropy(chunk.as_mut_ptr().cast(), chunk.len()) }; + assert_ne!(r, -1, "failed to generate random data"); + } +} diff --git a/library/std/src/sys/random/linux.rs b/library/std/src/sys/random/linux.rs index 4ede0af6494..073fdc45e61 100644 --- a/library/std/src/sys/random/linux.rs +++ b/library/std/src/sys/random/linux.rs @@ -63,9 +63,9 @@ use crate::fs::File; use crate::io::Read; use crate::os::fd::AsRawFd; +use crate::sync::OnceLock; use crate::sync::atomic::AtomicBool; use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; -use crate::sync::OnceLock; use crate::sys::pal::os::errno; use crate::sys::pal::weak::syscall; diff --git a/library/std/src/sys/random/mod.rs b/library/std/src/sys/random/mod.rs index d39e78bb7b3..16fb8c64c9b 100644 --- a/library/std/src/sys/random/mod.rs +++ b/library/std/src/sys/random/mod.rs @@ -6,24 +6,25 @@ } else if #[cfg(target_os = "windows")] { mod windows; pub use windows::fill_bytes; - } else if #[cfg(any( - target_os = "openbsd", - target_os = "freebsd", - target_os = "macos", - all(target_os = "netbsd", netbsd10), - target_os = "dragonfly", - target_os = "illumos", - target_os = "solaris", - target_os = "emscripten", - target_os = "vita", - target_os = "haiku", - ))] { - mod unix; - pub use unix::fill_bytes; - // Others, in alphabetical ordering. - } else if #[cfg(all(target_vendor = "apple", not(target_os = "macos")))] { + } else if #[cfg(target_vendor = "apple")] { mod apple; pub use apple::fill_bytes; + // Others, in alphabetical ordering. + } else if #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "haiku", + target_os = "illumos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "solaris", + target_os = "vita", + ))] { + mod arc4random; + pub use arc4random::fill_bytes; + } else if #[cfg(target_os = "emscripten")] { + mod getentropy; + pub use getentropy::fill_bytes; } else if #[cfg(target_os = "espidf")] { mod espidf; pub use espidf::fill_bytes; @@ -34,7 +35,7 @@ mod hermit; pub use hermit::fill_bytes; } else if #[cfg(target_os = "horizon")] { - // FIXME: add getentropy to shim-3ds + // FIXME: add arc4random_buf to shim-3ds mod horizon; pub use horizon::fill_bytes; } else if #[cfg(any( @@ -44,10 +45,6 @@ ))] { mod unix_legacy; pub use unix_legacy::fill_bytes; - } else if #[cfg(all(target_os = "netbsd", not(netbsd10)))] { - // FIXME: remove once NetBSD 10 is the minimum - mod netbsd; - pub use netbsd::fill_bytes; } else if #[cfg(target_os = "redox")] { mod redox; pub use redox::fill_bytes; diff --git a/library/std/src/sys/random/netbsd.rs b/library/std/src/sys/random/netbsd.rs deleted file mode 100644 index 2c5d9c72f30..00000000000 --- a/library/std/src/sys/random/netbsd.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::ptr; - -pub fn fill_bytes(bytes: &mut [u8]) { - let mib = [libc::CTL_KERN, libc::KERN_ARND]; - for chunk in bytes.chunks_mut(256) { - let mut len = chunk.len(); - let ret = unsafe { - libc::sysctl( - mib.as_ptr(), - mib.len() as libc::c_uint, - chunk.as_mut_ptr().cast(), - &mut len, - ptr::null(), - 0, - ) - }; - assert!(ret != -1 && len == chunk.len(), "failed to generate random data"); - } -} diff --git a/library/std/src/sys/random/unix.rs b/library/std/src/sys/random/unix.rs deleted file mode 100644 index a56847e5541..00000000000 --- a/library/std/src/sys/random/unix.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Random data generation through `getentropy`. -//! -//! Since issue 8 (2024), the POSIX specification mandates the existence of the -//! `getentropy` function, which fills a slice of up to `GETENTROPY_MAX` bytes -//! (256 on all known platforms) with random data. Luckily, this function has -//! already been available on quite some BSDs before that, having appeared with -//! OpenBSD 5.7 and spread from there: -//! -//! platform | version | man-page -//! -----------|---------|---------- -//! OpenBSD | 5.6 | -//! FreeBSD | 12.0 | -//! macOS | 10.12 | -//! NetBSD | 10.0 | -//! DragonFly | 6.1 | -//! Illumos | ? | -//! Solaris | ? | -//! -//! As it is standardized we use it whereever possible, even when `getrandom` is -//! also available. NetBSD even warns that "Applications should avoid getrandom -//! and use getentropy(2) instead; getrandom may be removed from a later -//! release."[^1]. -//! -//! [^1]: - -pub fn fill_bytes(bytes: &mut [u8]) { - // GETENTROPY_MAX isn't defined yet on most platforms, but it's mandated - // to be at least 256, so just use that as limit. - for chunk in bytes.chunks_mut(256) { - let r = unsafe { libc::getentropy(chunk.as_mut_ptr().cast(), chunk.len()) }; - assert_ne!(r, -1, "failed to generate random data"); - } -} diff --git a/library/std/src/sys/random/unix_legacy.rs b/library/std/src/sys/random/unix_legacy.rs index dd6be43c173..587068b0d66 100644 --- a/library/std/src/sys/random/unix_legacy.rs +++ b/library/std/src/sys/random/unix_legacy.rs @@ -3,8 +3,8 @@ //! Before `getentropy` was standardized in 2024, UNIX didn't have a standardized //! way of getting random data, so systems just followed the precedent set by //! Linux and exposed random devices at `/dev/random` and `/dev/urandom`. Thus, -//! for the few systems that do not support `getentropy` yet, we just read from -//! the file. +//! for the few systems that support neither `arc4random_buf` nor `getentropy` +//! yet, we just read from the file. use crate::fs::File; use crate::io::Read; From a21ff017f4f119381088c38e6966ed1022ef40f6 Mon Sep 17 00:00:00 2001 From: joboet Date: Sun, 18 Aug 2024 15:08:39 +0200 Subject: [PATCH 08/11] miri: shim `CCRandomGenerateBytes` --- src/tools/miri/src/shims/unix/macos/foreign_items.rs | 10 ++++++++++ .../tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs | 7 +++++++ 2 files changed, 17 insertions(+) create mode 100644 src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs diff --git a/src/tools/miri/src/shims/unix/macos/foreign_items.rs b/src/tools/miri/src/shims/unix/macos/foreign_items.rs index 47b887dec94..95a41752059 100644 --- a/src/tools/miri/src/shims/unix/macos/foreign_items.rs +++ b/src/tools/miri/src/shims/unix/macos/foreign_items.rs @@ -78,6 +78,16 @@ fn emulate_foreign_item_inner( this.write_pointer(environ, dest)?; } + // Random data generation + "CCRandomGenerateBytes" => { + let [bytes, count] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + let bytes = this.read_pointer(bytes)?; + let count = this.read_target_usize(count)?; + let success = this.eval_libc_i32("kCCSuccess"); + this.gen_random(bytes, count)?; + this.write_int(success, dest)?; + } + // Time related shims "mach_absolute_time" => { let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; diff --git a/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs b/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs new file mode 100644 index 00000000000..d06e8622ee8 --- /dev/null +++ b/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs @@ -0,0 +1,7 @@ +//@only-target-apple: this directly tests apple-only functions + +fn main() { + let mut bytes = [0u8; 24]; + let ret = unsafe { libc::CCRandomGenerateBytes(bytes.as_mut_ptr().cast(), bytes.len()) }; + assert_eq!(ret, libc::kCCSuccess); +} From e94dd9b71268531efeecff00a7400935dd289383 Mon Sep 17 00:00:00 2001 From: joboet Date: Sun, 22 Sep 2024 16:33:58 +0200 Subject: [PATCH 09/11] random: add tracking issue, address other comments --- library/core/src/lib.rs | 2 +- library/core/src/random.rs | 4 ++-- library/std/src/lib.rs | 2 +- library/std/src/random.rs | 25 +++++++++++++++---------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index b4393889e75..d9b03c97072 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -394,7 +394,7 @@ pub mod assert_matches { #[unstable(feature = "core_pattern_types", issue = "123646")] pub mod pat; pub mod pin; -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] pub mod random; #[unstable(feature = "new_range_api", issue = "125687")] pub mod range; diff --git a/library/core/src/random.rs b/library/core/src/random.rs index 9e0d0d0c58b..051fe260863 100644 --- a/library/core/src/random.rs +++ b/library/core/src/random.rs @@ -4,7 +4,7 @@ //! given [`RandomSource`]. /// A source of randomness. -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] pub trait RandomSource { /// Fills `bytes` with random bytes. fn fill_bytes(&mut self, bytes: &mut [u8]); @@ -17,7 +17,7 @@ pub trait RandomSource { /// distribution, so a value of 1 is just as likely as [`i32::MAX`]. By using /// modulo operations, some of the resulting values can become more likely than /// others. Use audited crates when in doubt. -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] pub trait Random: Sized { /// Generates a random value. fn random(source: &mut (impl RandomSource + ?Sized)) -> Self; diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index ce05d816604..e6b2f15a7a6 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -597,7 +597,7 @@ #[unstable(feature = "anonymous_pipe", issue = "127154")] pub mod pipe; pub mod process; -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] pub mod random; pub mod sync; pub mod time; diff --git a/library/std/src/random.rs b/library/std/src/random.rs index 48b55a55b2f..ecbf02eee84 100644 --- a/library/std/src/random.rs +++ b/library/std/src/random.rs @@ -3,7 +3,7 @@ //! The [`Random`] trait allows generating a random value for a type using a //! given [`RandomSource`]. -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] pub use core::random::*; use crate::sys::random as sys; @@ -15,9 +15,9 @@ /// documentation below for the specific guarantees your target provides. /// /// The high quality of randomness provided by this source means it can be quite -/// slow. If you need a large quantity of random numbers and security is not a -/// concern, consider using an alternative random number generator (potentially -/// seeded from this one). +/// slow on some targets. If you need a large quantity of random numbers and +/// security is not a concern, consider using an alternative random number +/// generator (potentially seeded from this one). /// /// # Underlying sources /// @@ -26,9 +26,9 @@ /// Linux | [`getrandom`] or [`/dev/urandom`] after polling `/dev/random` /// Windows | [`ProcessPrng`](https://learn.microsoft.com/en-us/windows/win32/seccng/processprng) /// Apple | `CCRandomGenerateBytes` -/// DragonFly | [`arc4random_buf`](https://man.dragonflybsd.org/?command=arc4random§ion=ANY) +/// DragonFly | [`arc4random_buf`](https://man.dragonflybsd.org/?command=arc4random) /// ESP-IDF | [`esp_fill_random`](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/random.html#_CPPv415esp_fill_randomPv6size_t) -/// FreeBSD | [`arc4random_buf`](https://man.freebsd.org/cgi/man.cgi?query=arc4random&apropos=0&sektion=0&manpath=FreeBSD+15.0-CURRENT&arch=default&format=html) +/// FreeBSD | [`arc4random_buf`](https://man.freebsd.org/cgi/man.cgi?query=arc4random) /// Fuchsia | [`cprng_draw`](https://fuchsia.dev/reference/syscalls/cprng_draw) /// Haiku | `arc4random_buf` /// Illumos | [`arc4random_buf`](https://www.illumos.org/man/3C/arc4random) @@ -48,15 +48,19 @@ /// WASI | [`random_get`](https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md#-random_getbuf-pointeru8-buf_len-size---result-errno) /// ZKVM | `sys_rand` /// -/// **Disclaimer:** The sources used might change over time. +/// Note that the sources used might change over time. +/// +/// Consult the documentation for the underlying operations on your supported +/// targets to determine whether they provide any particular desired properties, +/// such as support for reseeding on VM fork operations. /// /// [`getrandom`]: https://www.man7.org/linux/man-pages/man2/getrandom.2.html /// [`/dev/urandom`]: https://www.man7.org/linux/man-pages/man4/random.4.html #[derive(Default, Debug, Clone, Copy)] -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] pub struct DefaultRandomSource; -#[unstable(feature = "random", issue = "none")] +#[unstable(feature = "random", issue = "130703")] impl RandomSource for DefaultRandomSource { fn fill_bytes(&mut self, bytes: &mut [u8]) { sys::fill_bytes(bytes) @@ -83,7 +87,7 @@ fn fill_bytes(&mut self, bytes: &mut [u8]) { /// /// use std::random::random; /// -/// let bits = random::(); +/// let bits: u128 = random(); /// let g1 = (bits >> 96) as u32; /// let g2 = (bits >> 80) as u16; /// let g3 = (0x4000 | (bits >> 64) & 0x0fff) as u16; @@ -94,6 +98,7 @@ fn fill_bytes(&mut self, bytes: &mut [u8]) { /// ``` /// /// [version 4/variant 1 UUID]: https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_(random) +#[unstable(feature = "random", issue = "130703")] pub fn random() -> T { T::random(&mut DefaultRandomSource) } From 3ff09a05c81f2c6b9eed71e6e8cfe0cf2c8d4f51 Mon Sep 17 00:00:00 2001 From: joboet Date: Sun, 22 Sep 2024 18:35:11 +0200 Subject: [PATCH 10/11] update miri test --- .../miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs b/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs index d06e8622ee8..afcc02bc839 100644 --- a/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs +++ b/src/tools/miri/tests/pass-dep/libc/ccrandomgeneratebytes_apple.rs @@ -1,4 +1,4 @@ -//@only-target-apple: this directly tests apple-only functions +//@only-target: apple # This directly tests apple-only functions fn main() { let mut bytes = [0u8; 24]; From b0c2c9372ad043935e9b92e46b6adddbeb34cefa Mon Sep 17 00:00:00 2001 From: joboet Date: Mon, 23 Sep 2024 13:37:52 +0200 Subject: [PATCH 11/11] readd @tgross35 and @joboet to the review rotation --- triagebot.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/triagebot.toml b/triagebot.toml index a6a0b02d7b7..5d80b9e656e 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -923,10 +923,8 @@ contributing_url = "https://rustc-dev-guide.rust-lang.org/getting-started.html" users_on_vacation = [ "fmease", "jhpratt", - "joboet", "jyn514", "oli-obk", - "tgross35", ] [assign.adhoc_groups]