mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Separate doctest collection from running

Browse Source
This commit is contained in:
Noah Lev 2024-05-31 00:31:26 -07:00 committed by Guillaume Gomez
parent 16db1a1bd0
commit 85499ebf13
3 changed files with 236 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
src/librustdoc/doctest.rs
View File

@ -8,7 +8,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_data_structures::sync::Lrc;
use rustc_errors::emitter::stderr_destination;
use rustc_errors::{ColorConfig, ErrorGuaranteed, FatalError};
use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_hir::CRATE_HIR_ID;
use rustc_interface::interface;
use rustc_parse::new_parser_from_source_str;
@ -19,10 +19,9 @@ use rustc_session::parse::ParseSess;
use rustc_span::edition::Edition;
use rustc_span::source_map::SourceMap;
use rustc_span::symbol::sym;
use rustc_span::{BytePos, FileName, Pos, Span, DUMMY_SP};
use rustc_span::FileName;
use rustc_target::spec::{Target, TargetTriple};
use std::env;
use std::fs::File;
use std::io::{self, Write};
use std::panic;
@ -38,7 +37,8 @@ use crate::config::Options as RustdocOptions;
use crate::html::markdown::{ErrorCodes, Ignore, LangString};
use crate::lint::init_lints;
use self::rust::HirCollector;
use self::markdown::MdDoctest;
use self::rust::{HirCollector, RustDoctest};
/// Options that apply to all doctests in a crate or Markdown file (for `rustdoc foo.md`).
#[derive(Clone, Default)]
@ -182,29 +182,19 @@ pub(crate) fn run(
let mut collector = Collector::new(
tcx.crate_name(LOCAL_CRATE).to_string(),
options,
false,
opts,
Some(compiler.sess.psess.clone_source_map()),
None,
enable_per_target_ignores,
file_path,
);
let mut hir_collector = HirCollector {
sess: &compiler.sess,
collector: &mut collector,
map: tcx.hir(),
codes: ErrorCodes::from(
compiler.sess.opts.unstable_features.is_nightly_build(),
),
let hir_collector = HirCollector::new(
&compiler.sess,
tcx.hir(),
ErrorCodes::from(compiler.sess.opts.unstable_features.is_nightly_build()),
enable_per_target_ignores,
tcx,
};
hir_collector.visit_testable(
"".to_string(),
CRATE_DEF_ID,
tcx.hir().span(CRATE_HIR_ID),
|this| tcx.hir().walk_toplevel_module(this),
);
let tests = hir_collector.collect_crate();
tests.into_iter().for_each(|t| collector.add_test(ScrapedDoctest::Rust(t)));
collector
});
@ -985,6 +975,12 @@ impl IndividualTestOptions {
}
}
/// A doctest scraped from the code, ready to be turned into a runnable test.
enum ScrapedDoctest {
Rust(RustDoctest),
Markdown(MdDoctest),
}
pub(crate) trait DoctestVisitor {
fn visit_test(&mut self, test: String, config: LangString, line: usize);
fn get_line(&self) -> usize {
@ -996,36 +992,9 @@ pub(crate) trait DoctestVisitor {
pub(crate) struct Collector {
pub(crate) tests: Vec<test::TestDescAndFn>,
// The name of the test displayed to the user, separated by `::`.
//
// In tests from Rust source, this is the path to the item
// e.g., `["std", "vec", "Vec", "push"]`.
//
// In tests from a markdown file, this is the titles of all headers (h1~h6)
// of the sections that contain the code block, e.g., if the markdown file is
// written as:
//
// ``````markdown
// # Title
//
// ## Subtitle
//
// ```rust
// assert!(true);
// ```
// ``````
//
// the `names` vector of that test will be `["Title", "Subtitle"]`.
names: Vec<String>,
rustdoc_options: RustdocOptions,
use_headers: bool,
enable_per_target_ignores: bool,
crate_name: String,
opts: GlobalTestOptions,
position: Span,
source_map: Option<Lrc<SourceMap>>,
filename: Option<PathBuf>,
visited_tests: FxHashMap<(String, usize), usize>,
unused_extern_reports: Arc<Mutex<Vec<UnusedExterns>>>,
compiling_test_count: AtomicUsize,
@ -1036,24 +1005,14 @@ impl Collector {
pub(crate) fn new(
crate_name: String,
rustdoc_options: RustdocOptions,
use_headers: bool,
opts: GlobalTestOptions,
source_map: Option<Lrc<SourceMap>>,
filename: Option<PathBuf>,
enable_per_target_ignores: bool,
arg_file: PathBuf,
) -> Collector {
Collector {
tests: Vec::new(),
names: Vec::new(),
rustdoc_options,
use_headers,
enable_per_target_ignores,
crate_name,
opts,
position: DUMMY_SP,
source_map,
filename,
visited_tests: FxHashMap::default(),
unused_extern_reports: Default::default(),
compiling_test_count: AtomicUsize::new(0),
@ -1061,8 +1020,8 @@ impl Collector {
}
}
fn generate_name(&self, line: usize, filename: &FileName) -> String {
let mut item_path = self.names.join("::");
fn generate_name(&self, filename: &FileName, line: usize, logical_path: &[String]) -> String {
let mut item_path = logical_path.join("::");
item_path.retain(|c| c != ' ');
if !item_path.is_empty() {
item_path.push(' ');
@ -1070,40 +1029,24 @@ impl Collector {
format!("{} - {item_path}(line {line})", filename.prefer_local())
}
pub(crate) fn set_position(&mut self, position: Span) {
self.position = position;
}
fn get_filename(&self) -> FileName {
if let Some(ref source_map) = self.source_map {
let filename = source_map.span_to_filename(self.position);
if let FileName::Real(ref filename) = filename
&& let Ok(cur_dir) = env::current_dir()
&& let Some(local_path) = filename.local_path()
&& let Ok(path) = local_path.strip_prefix(&cur_dir)
{
return path.to_owned().into();
fn add_test(&mut self, test: ScrapedDoctest) {
let (filename, line, logical_path, langstr, text) = match test {
ScrapedDoctest::Rust(RustDoctest { filename, line, logical_path, langstr, text }) => {
(filename, line, logical_path, langstr, text)
}
filename
} else if let Some(ref filename) = self.filename {
filename.clone().into()
} else {
FileName::Custom("input".to_owned())
}
}
}
ScrapedDoctest::Markdown(MdDoctest { filename, line, logical_path, langstr, text }) => {
(filename, line, logical_path, langstr, text)
}
};
impl DoctestVisitor for Collector {
fn visit_test(&mut self, test: String, config: LangString, line: usize) {
let filename = self.get_filename();
let name = self.generate_name(line, &filename);
let name = self.generate_name(&filename, line, &logical_path);
let crate_name = self.crate_name.clone();
let opts = self.opts.clone();
let edition = config.edition.unwrap_or(self.rustdoc_options.edition);
let edition = langstr.edition.unwrap_or(self.rustdoc_options.edition);
let target_str = self.rustdoc_options.target.to_string();
let unused_externs = self.unused_extern_reports.clone();
let no_run = config.no_run || self.rustdoc_options.no_run;
if !config.compile_fail {
let no_run = langstr.no_run || self.rustdoc_options.no_run;
if !langstr.compile_fail {
self.compiling_test_count.fetch_add(1, Ordering::SeqCst);
}
@ -1140,11 +1083,11 @@ impl DoctestVisitor for Collector {
let rustdoc_test_options =
IndividualTestOptions::new(&self.rustdoc_options, &self.arg_file, test_id);
debug!("creating test {name}: {test}");
debug!("creating test {name}: {text}");
self.tests.push(test::TestDescAndFn {
desc: test::TestDesc {
name: test::DynTestName(name),
ignore: match config.ignore {
ignore: match langstr.ignore {
Ignore::All => true,
Ignore::None => false,
Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)),
@ -1157,7 +1100,7 @@ impl DoctestVisitor for Collector {
end_col: 0,
// compiler failures are test failures
should_panic: test::ShouldPanic::No,
compile_fail: config.compile_fail,
compile_fail: langstr.compile_fail,
no_run,
test_type: test::TestType::DocTest,
},
@ -1166,11 +1109,11 @@ impl DoctestVisitor for Collector {
unused_externs.lock().unwrap().push(uext);
};
let res = run_test(
&test,
&text,
&crate_name,
line,
rustdoc_test_options,
config,
langstr,
no_run,
&opts,
edition,
@ -1233,59 +1176,6 @@ impl DoctestVisitor for Collector {
})),
});
}
fn get_line(&self) -> usize {
if let Some(ref source_map) = self.source_map {
let line = self.position.lo().to_usize();
let line = source_map.lookup_char_pos(BytePos(line as u32)).line;
if line > 0 { line - 1 } else { line }
} else {
0
}
}
fn visit_header(&mut self, name: &str, level: u32) {
if self.use_headers {
// We use these headings as test names, so it's good if
// they're valid identifiers.
let name = name
.chars()
.enumerate()
.map(|(i, c)| {
if (i == 0 && rustc_lexer::is_id_start(c))
|| (i != 0 && rustc_lexer::is_id_continue(c))
{
c
} else {
'_'
}
})
.collect::<String>();
// Here we try to efficiently assemble the header titles into the
// test name in the form of `h1::h2::h3::h4::h5::h6`.
//
// Suppose that originally `self.names` contains `[h1, h2, h3]`...
let level = level as usize;
if level <= self.names.len() {
// ... Consider `level == 2`. All headers in the lower levels
// are irrelevant in this new level. So we should reset
// `self.names` to contain headers until <h2>, and replace that
// slot with the new name: `[h1, name]`.
self.names.truncate(level);
self.names[level - 1] = name;
} else {
// ... On the other hand, consider `level == 5`. This means we
// need to extend `self.names` to contain five headers. We fill
// in the missing level (<h4>) with `_`. Thus `self.names` will
// become `[h1, h2, h3, "_", name]`.
if level - 1 > self.names.len() {
self.names.resize(level - 1, "_".to_owned());
}
self.names.push(name);
}
}
}
}
#[cfg(test)] // used in tests

110
src/librustdoc/doctest/markdown.rs
View File

@ -2,12 +2,84 @@
use std::fs::read_to_string;
use rustc_span::DUMMY_SP;
use rustc_span::FileName;
use tempfile::tempdir;
use super::{generate_args_file, Collector, GlobalTestOptions};
use super::{generate_args_file, Collector, DoctestVisitor, GlobalTestOptions, ScrapedDoctest};
use crate::config::Options;
use crate::html::markdown::{find_testable_code, ErrorCodes};
use crate::html::markdown::{find_testable_code, ErrorCodes, LangString};
pub(super) struct MdDoctest {
pub(super) filename: FileName,
pub(super) line: usize,
pub(super) logical_path: Vec<String>,
pub(super) langstr: LangString,
pub(super) text: String,
}
struct MdCollector {
tests: Vec<MdDoctest>,
cur_path: Vec<String>,
filename: FileName,
}
impl DoctestVisitor for MdCollector {
fn visit_test(&mut self, test: String, config: LangString, line: usize) {
let filename = self.filename.clone();
self.tests.push(MdDoctest {
filename,
line,
logical_path: self.cur_path.clone(),
langstr: config,
text: test,
});
}
fn get_line(&self) -> usize {
0
}
fn visit_header(&mut self, name: &str, level: u32) {
// We use these headings as test names, so it's good if
// they're valid identifiers.
let name = name
.chars()
.enumerate()
.map(|(i, c)| {
if (i == 0 && rustc_lexer::is_id_start(c))
|| (i != 0 && rustc_lexer::is_id_continue(c))
{
c
} else {
'_'
}
})
.collect::<String>();
// Here we try to efficiently assemble the header titles into the
// test name in the form of `h1::h2::h3::h4::h5::h6`.
//
// Suppose that originally `self.cur_path` contains `[h1, h2, h3]`...
let level = level as usize;
if level <= self.cur_path.len() {
// ... Consider `level == 2`. All headers in the lower levels
// are irrelevant in this new level. So we should reset
// `self.names` to contain headers until <h2>, and replace that
// slot with the new name: `[h1, name]`.
self.cur_path.truncate(level);
self.cur_path[level - 1] = name;
} else {
// ... On the other hand, consider `level == 5`. This means we
// need to extend `self.names` to contain five headers. We fill
// in the missing level (<h4>) with `_`. Thus `self.names` will
// become `[h1, h2, h3, "_", name]`.
if level - 1 > self.cur_path.len() {
self.cur_path.resize(level - 1, "_".to_owned());
}
self.cur_path.push(name);
}
}
}
/// Runs any tests/code examples in the markdown file `input`.
pub(crate) fn test(options: Options) -> Result<(), String> {
@ -27,21 +99,29 @@ pub(crate) fn test(options: Options) -> Result<(), String> {
let file_path = temp_dir.path().join("rustdoc-cfgs");
generate_args_file(&file_path, &options)?;
let mut collector = Collector::new(
options.input.filestem().to_string(),
options.clone(),
true,
opts,
None,
options.input.opt_path().map(ToOwned::to_owned),
options.enable_per_target_ignores,
file_path,
);
collector.set_position(DUMMY_SP);
let mut md_collector = MdCollector {
tests: vec![],
cur_path: vec![],
filename: options
.input
.opt_path()
.map(ToOwned::to_owned)
.map(FileName::from)
.unwrap_or(FileName::Custom("input".to_owned())),
};
let codes = ErrorCodes::from(options.unstable_features.is_nightly_build());
find_testable_code(&input_str, &mut collector, codes, options.enable_per_target_ignores, None);
find_testable_code(
&input_str,
&mut md_collector,
codes,
options.enable_per_target_ignores,
None,
);
let mut collector =
Collector::new(options.input.filestem().to_string(), options.clone(), opts, file_path);
md_collector.tests.into_iter().for_each(|t| collector.add_test(ScrapedDoctest::Markdown(t)));
crate::doctest::run_tests(options.test_args, options.nocapture, collector.tests);
Ok(())
}

131
src/librustdoc/doctest/rust.rs
View File

@ -1,29 +1,108 @@
//! Doctest functionality used only for doctests in `.rs` source files.
use rustc_data_structures::fx::FxHashSet;
use rustc_hir::def_id::LocalDefId;
use rustc_hir::{self as hir, intravisit};
use std::env;
use rustc_data_structures::{fx::FxHashSet, sync::Lrc};
use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
use rustc_hir::{self as hir, intravisit, CRATE_HIR_ID};
use rustc_middle::hir::map::Map;
use rustc_middle::hir::nested_filter;
use rustc_middle::ty::TyCtxt;
use rustc_resolve::rustdoc::span_of_fragments;
use rustc_session::Session;
use rustc_span::{Span, DUMMY_SP};
use rustc_span::source_map::SourceMap;
use rustc_span::{BytePos, FileName, Pos, Span, DUMMY_SP};
use super::Collector;
use super::DoctestVisitor;
use crate::clean::{types::AttributesExt, Attributes};
use crate::html::markdown::{self, ErrorCodes};
use crate::html::markdown::{self, ErrorCodes, LangString};
pub(super) struct HirCollector<'a, 'hir, 'tcx> {
pub(super) sess: &'a Session,
pub(super) collector: &'a mut Collector,
pub(super) map: Map<'hir>,
pub(super) codes: ErrorCodes,
pub(super) tcx: TyCtxt<'tcx>,
pub(super) struct RustDoctest {
pub(super) filename: FileName,
pub(super) line: usize,
pub(super) logical_path: Vec<String>,
pub(super) langstr: LangString,
pub(super) text: String,
}
impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> {
pub(super) fn visit_testable<F: FnOnce(&mut Self)>(
struct RustCollector {
source_map: Lrc<SourceMap>,
tests: Vec<RustDoctest>,
cur_path: Vec<String>,
position: Span,
}
impl RustCollector {
fn get_filename(&self) -> FileName {
let filename = self.source_map.span_to_filename(self.position);
if let FileName::Real(ref filename) = filename
&& let Ok(cur_dir) = env::current_dir()
&& let Some(local_path) = filename.local_path()
&& let Ok(path) = local_path.strip_prefix(&cur_dir)
{
return path.to_owned().into();
}
filename
}
}
impl DoctestVisitor for RustCollector {
fn visit_test(&mut self, test: String, config: LangString, line: usize) {
self.tests.push(RustDoctest {
filename: self.get_filename(),
line,
logical_path: self.cur_path.clone(),
langstr: config,
text: test,
});
}
fn get_line(&self) -> usize {
let line = self.position.lo().to_usize();
let line = self.source_map.lookup_char_pos(BytePos(line as u32)).line;
if line > 0 { line - 1 } else { line }
}
fn visit_header(&mut self, _name: &str, _level: u32) {}
}
pub(super) struct HirCollector<'a, 'tcx> {
sess: &'a Session,
map: Map<'tcx>,
codes: ErrorCodes,
tcx: TyCtxt<'tcx>,
enable_per_target_ignores: bool,
collector: RustCollector,
}
impl<'a, 'tcx> HirCollector<'a, 'tcx> {
pub fn new(
sess: &'a Session,
map: Map<'tcx>,
codes: ErrorCodes,
enable_per_target_ignores: bool,
tcx: TyCtxt<'tcx>,
) -> Self {
let collector = RustCollector {
source_map: sess.psess.clone_source_map(),
cur_path: vec![],
position: DUMMY_SP,
tests: vec![],
};
Self { sess, map, codes, enable_per_target_ignores, tcx, collector }
}
pub fn collect_crate(mut self) -> Vec<RustDoctest> {
let tcx = self.tcx;
self.visit_testable("".to_string(), CRATE_DEF_ID, tcx.hir().span(CRATE_HIR_ID), |this| {
tcx.hir().walk_toplevel_module(this)
});
self.collector.tests
}
}
impl<'a, 'tcx> HirCollector<'a, 'tcx> {
fn visit_testable<F: FnOnce(&mut Self)>(
&mut self,
name: String,
def_id: LocalDefId,
@ -39,7 +118,7 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> {
let has_name = !name.is_empty();
if has_name {
self.collector.names.push(name);
self.collector.cur_path.push(name);
}
// The collapse-docs pass won't combine sugared/raw doc attributes, or included files with
@ -52,12 +131,12 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> {
.find(|attr| attr.doc_str().is_some())
.map(|attr| attr.span.ctxt().outer_expn().expansion_cause().unwrap_or(attr.span))
.unwrap_or(DUMMY_SP);
self.collector.set_position(span);
self.collector.position = span;
markdown::find_testable_code(
&doc,
self.collector,
&mut self.collector,
self.codes,
self.collector.enable_per_target_ignores,
self.enable_per_target_ignores,
Some(&crate::html::markdown::ExtraInfo::new(
self.tcx,
def_id.to_def_id(),
@ -69,19 +148,19 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> {
nested(self);
if has_name {
self.collector.names.pop();
self.collector.cur_path.pop();
}
}
}
impl<'a, 'hir, 'tcx> intravisit::Visitor<'hir> for HirCollector<'a, 'hir, 'tcx> {
impl<'a, 'tcx> intravisit::Visitor<'tcx> for HirCollector<'a, 'tcx> {
type NestedFilter = nested_filter::All;
fn nested_visit_map(&mut self) -> Self::Map {
self.map
}
fn visit_item(&mut self, item: &'hir hir::Item<'_>) {
fn visit_item(&mut self, item: &'tcx hir::Item<'_>) {
let name = match &item.kind {
hir::ItemKind::Impl(impl_) => {
rustc_hir_pretty::id_to_string(&self.map, impl_.self_ty.hir_id)
@ -94,31 +173,31 @@ impl<'a, 'hir, 'tcx> intravisit::Visitor<'hir> for HirCollector<'a, 'hir, 'tcx>
});
}
fn visit_trait_item(&mut self, item: &'hir hir::TraitItem<'_>) {
fn visit_trait_item(&mut self, item: &'tcx hir::TraitItem<'_>) {
self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| {
intravisit::walk_trait_item(this, item);
});
}
fn visit_impl_item(&mut self, item: &'hir hir::ImplItem<'_>) {
fn visit_impl_item(&mut self, item: &'tcx hir::ImplItem<'_>) {
self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| {
intravisit::walk_impl_item(this, item);
});
}
fn visit_foreign_item(&mut self, item: &'hir hir::ForeignItem<'_>) {
fn visit_foreign_item(&mut self, item: &'tcx hir::ForeignItem<'_>) {
self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| {
intravisit::walk_foreign_item(this, item);
});
}
fn visit_variant(&mut self, v: &'hir hir::Variant<'_>) {
fn visit_variant(&mut self, v: &'tcx hir::Variant<'_>) {
self.visit_testable(v.ident.to_string(), v.def_id, v.span, |this| {
intravisit::walk_variant(this, v);
});
}
fn visit_field_def(&mut self, f: &'hir hir::FieldDef<'_>) {
fn visit_field_def(&mut self, f: &'tcx hir::FieldDef<'_>) {
self.visit_testable(f.ident.to_string(), f.def_id, f.span, |this| {
intravisit::walk_field_def(this, f);
});