Auto merge of #85652 - ehuss:linkchecker-perf, r=Mark-Simulacrum

Optimize linkchecker and add report.

This makes three changes to the linkchecker:

* Adds a report displayed after it finishes.
* Improves the performance by caching all filesystem access. The linkchecker can take over a minute to run on some systems, and this should make it about 2-3 times faster.
* Added a few tests.
This commit is contained in:
bors 2021-05-26 23:13:15 +00:00
commit 86ac0b4147
18 changed files with 519 additions and 241 deletions

View File

@ -14,18 +14,18 @@
//! A few exceptions are allowed as there's known bugs in rustdoc, but this
//! should catch the majority of "broken link" cases.
use std::collections::hash_map::Entry;
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::env;
use std::fs;
use std::io::ErrorKind;
use std::path::{Component, Path, PathBuf};
use std::rc::Rc;
use std::time::Instant;
use once_cell::sync::Lazy;
use regex::Regex;
use crate::Redirect::*;
// Add linkcheck exceptions here
// If at all possible you should use intra-doc links to avoid linkcheck issues. These
// are cases where that does not work
@ -87,33 +87,64 @@ macro_rules! t {
}
fn main() {
let docs = env::args_os().nth(1).unwrap();
let docs = env::args_os().nth(1).expect("doc path should be first argument");
let docs = env::current_dir().unwrap().join(docs);
let mut errors = false;
walk(&mut HashMap::new(), &docs, &docs, &mut errors);
if errors {
panic!("found some broken links");
let mut checker = Checker { root: docs.clone(), cache: HashMap::new() };
let mut report = Report {
errors: 0,
start: Instant::now(),
html_files: 0,
html_redirects: 0,
links_checked: 0,
links_ignored_external: 0,
links_ignored_exception: 0,
intra_doc_exceptions: 0,
};
checker.walk(&docs, &mut report);
report.report();
if report.errors != 0 {
println!("found some broken links");
std::process::exit(1);
}
}
#[derive(Debug)]
pub enum LoadError {
IOError(std::io::Error),
BrokenRedirect(PathBuf, std::io::Error),
IsRedirect,
struct Checker {
root: PathBuf,
cache: Cache,
}
enum Redirect {
SkipRedirect,
FromRedirect(bool),
struct Report {
errors: u32,
start: Instant,
html_files: u32,
html_redirects: u32,
links_checked: u32,
links_ignored_external: u32,
links_ignored_exception: u32,
intra_doc_exceptions: u32,
}
struct FileEntry {
source: Rc<String>,
ids: HashSet<String>,
/// A cache entry.
enum FileEntry {
/// An HTML file.
///
/// This includes the contents of the HTML file, and an optional set of
/// HTML IDs. The IDs are used for checking fragments. The are computed
/// as-needed. The source is discarded (replaced with an empty string)
/// after the file has been checked, to conserve on memory.
HtmlFile { source: Rc<String>, ids: RefCell<HashSet<String>> },
/// This file is an HTML redirect to the given local path.
Redirect { target: PathBuf },
/// This is not an HTML file.
OtherFile,
/// This is a directory.
Dir,
/// The file doesn't exist.
Missing,
}
type Cache = HashMap<PathBuf, FileEntry>;
/// A cache to speed up file access.
type Cache = HashMap<String, FileEntry>;
fn small_url_encode(s: &str) -> String {
s.replace("<", "%3C")
@ -130,38 +161,273 @@ fn small_url_encode(s: &str) -> String {
.replace("\"", "%22")
}
impl FileEntry {
fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
if self.ids.is_empty() {
with_attrs_in_source(contents, " id", |fragment, i, _| {
let frag = fragment.trim_start_matches("#").to_owned();
let encoded = small_url_encode(&frag);
if !self.ids.insert(frag) {
*errors = true;
println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
}
// Just in case, we also add the encoded id.
self.ids.insert(encoded);
});
}
}
}
fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
impl Checker {
/// Primary entry point for walking the filesystem to find HTML files to check.
fn walk(&mut self, dir: &Path, report: &mut Report) {
for entry in t!(dir.read_dir()).map(|e| t!(e)) {
let path = entry.path();
let kind = t!(entry.file_type());
if kind.is_dir() {
walk(cache, root, &path, errors);
self.walk(&path, report);
} else {
let pretty_path = check(cache, root, &path, errors);
if let Some(pretty_path) = pretty_path {
let entry = cache.get_mut(&pretty_path).unwrap();
self.check(&path, report);
}
}
}
/// Checks a single file.
fn check(&mut self, file: &Path, report: &mut Report) {
let (pretty_path, entry) = self.load_file(file, report);
let source = match entry {
FileEntry::Missing => panic!("missing file {:?} while walking", file),
FileEntry::Dir => unreachable!("never with `check` path"),
FileEntry::OtherFile => return,
FileEntry::Redirect { .. } => return,
FileEntry::HtmlFile { source, ids } => {
parse_ids(&mut ids.borrow_mut(), &pretty_path, source, report);
source.clone()
}
};
// Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
with_attrs_in_source(&source, " href", |url, i, base| {
// Ignore external URLs
if url.starts_with("http:")
|| url.starts_with("https:")
|| url.starts_with("javascript:")
|| url.starts_with("ftp:")
|| url.starts_with("irc:")
|| url.starts_with("data:")
{
report.links_ignored_external += 1;
return;
}
report.links_checked += 1;
let (url, fragment) = match url.split_once('#') {
None => (url, None),
Some((url, fragment)) => (url, Some(fragment)),
};
// NB: the `splitn` always succeeds, even if the delimiter is not present.
let url = url.splitn(2, '?').next().unwrap();
// Once we've plucked out the URL, parse it using our base url and
// then try to extract a file path.
let mut path = file.to_path_buf();
if !base.is_empty() || !url.is_empty() {
path.pop();
for part in Path::new(base).join(url).components() {
match part {
Component::Prefix(_) | Component::RootDir => {
// Avoid absolute paths as they make the docs not
// relocatable by making assumptions on where the docs
// are hosted relative to the site root.
report.errors += 1;
println!(
"{}:{}: absolute path - {}",
pretty_path,
i + 1,
Path::new(base).join(url).display()
);
return;
}
Component::CurDir => {}
Component::ParentDir => {
path.pop();
}
Component::Normal(s) => {
path.push(s);
}
}
}
}
let (target_pretty_path, target_entry) = self.load_file(&path, report);
let (target_source, target_ids) = match target_entry {
FileEntry::Missing => {
if is_exception(file, &target_pretty_path) {
report.links_ignored_exception += 1;
} else {
report.errors += 1;
println!(
"{}:{}: broken link - `{}`",
pretty_path,
i + 1,
target_pretty_path
);
}
return;
}
FileEntry::Dir => {
// Links to directories show as directory listings when viewing
// the docs offline so it's best to avoid them.
report.errors += 1;
println!(
"{}:{}: directory link to `{}` \
(directory links should use index.html instead)",
pretty_path,
i + 1,
target_pretty_path
);
return;
}
FileEntry::OtherFile => return,
FileEntry::Redirect { target } => {
let t = target.clone();
drop(target);
let (target, redir_entry) = self.load_file(&t, report);
match redir_entry {
FileEntry::Missing => {
report.errors += 1;
println!(
"{}:{}: broken redirect from `{}` to `{}`",
pretty_path,
i + 1,
target_pretty_path,
target
);
return;
}
FileEntry::Redirect { target } => {
// Redirect to a redirect, this link checker
// currently doesn't support this, since it would
// require cycle checking, etc.
report.errors += 1;
println!(
"{}:{}: redirect from `{}` to `{}` \
which is also a redirect (not supported)",
pretty_path,
i + 1,
target_pretty_path,
target.display()
);
return;
}
FileEntry::Dir => {
report.errors += 1;
println!(
"{}:{}: redirect from `{}` to `{}` \
which is a directory \
(directory links should use index.html instead)",
pretty_path,
i + 1,
target_pretty_path,
target
);
return;
}
FileEntry::OtherFile => return,
FileEntry::HtmlFile { source, ids } => (source, ids),
}
}
FileEntry::HtmlFile { source, ids } => (source, ids),
};
// Alright, if we've found an HTML file for the target link. If
// this is a fragment link, also check that the `id` exists.
if let Some(ref fragment) = fragment {
// Fragments like `#1-6` are most likely line numbers to be
// interpreted by javascript, so we're ignoring these
if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
return;
}
// These appear to be broken in mdbook right now?
if fragment.starts_with('-') {
return;
}
parse_ids(&mut target_ids.borrow_mut(), &pretty_path, target_source, report);
if target_ids.borrow().contains(*fragment) {
return;
}
if is_exception(file, &format!("#{}", fragment)) {
report.links_ignored_exception += 1;
} else {
report.errors += 1;
print!("{}:{}: broken link fragment ", pretty_path, i + 1);
println!("`#{}` pointing to `{}`", fragment, pretty_path);
};
}
});
// Search for intra-doc links that rustdoc didn't warn about
// FIXME(#77199, 77200) Rustdoc should just warn about these directly.
// NOTE: only looks at one line at a time; in practice this should find most links
for (i, line) in source.lines().enumerate() {
for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
if is_intra_doc_exception(file, &broken_link[1]) {
report.intra_doc_exceptions += 1;
} else {
report.errors += 1;
print!("{}:{}: broken intra-doc link - ", pretty_path, i + 1);
println!("{}", &broken_link[0]);
}
}
}
// we don't need the source anymore,
// so drop to reduce memory-usage
entry.source = Rc::new(String::new());
match self.cache.get_mut(&pretty_path).unwrap() {
FileEntry::HtmlFile { source, .. } => *source = Rc::new(String::new()),
_ => unreachable!("must be html file"),
}
}
/// Load a file from disk, or from the cache if available.
fn load_file(&mut self, file: &Path, report: &mut Report) -> (String, &FileEntry) {
let pretty_path =
file.strip_prefix(&self.root).unwrap_or(&file).to_str().unwrap().to_string();
let entry =
self.cache.entry(pretty_path.clone()).or_insert_with(|| match fs::metadata(file) {
Ok(metadata) if metadata.is_dir() => FileEntry::Dir,
Ok(_) => {
if file.extension().and_then(|s| s.to_str()) != Some("html") {
FileEntry::OtherFile
} else {
report.html_files += 1;
load_html_file(file, report)
}
}
Err(e) if e.kind() == ErrorKind::NotFound => FileEntry::Missing,
Err(e) => {
panic!("unexpected read error for {}: {}", file.display(), e);
}
});
(pretty_path, entry)
}
}
impl Report {
fn report(&self) {
println!("checked links in: {:.1}s", self.start.elapsed().as_secs_f64());
println!("number of HTML files scanned: {}", self.html_files);
println!("number of HTML redirects found: {}", self.html_redirects);
println!("number of links checked: {}", self.links_checked);
println!("number of links ignored due to external: {}", self.links_ignored_external);
println!("number of links ignored due to exceptions: {}", self.links_ignored_exception);
println!("number of intra doc links ignored: {}", self.intra_doc_exceptions);
println!("errors found: {}", self.errors);
}
}
fn load_html_file(file: &Path, report: &mut Report) -> FileEntry {
let source = match fs::read_to_string(file) {
Ok(s) => Rc::new(s),
Err(err) => {
// This usually should not fail since `metadata` was already
// called successfully on this file.
panic!("unexpected read error for {}: {}", file.display(), err);
}
};
match maybe_redirect(&source) {
Some(target) => {
report.html_redirects += 1;
let target = file.parent().unwrap().join(target);
FileEntry::Redirect { target }
}
None => FileEntry::HtmlFile { source: source.clone(), ids: RefCell::new(HashSet::new()) },
}
}
@ -191,198 +457,8 @@ fn is_exception(file: &Path, link: &str) -> bool {
}
}
fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
// Ignore non-HTML files.
if file.extension().and_then(|s| s.to_str()) != Some("html") {
return None;
}
let res = load_file(cache, root, file, SkipRedirect);
let (pretty_file, contents) = match res {
Ok(res) => res,
Err(_) => return None,
};
{
cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
}
// Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
with_attrs_in_source(&contents, " href", |url, i, base| {
// Ignore external URLs
if url.starts_with("http:")
|| url.starts_with("https:")
|| url.starts_with("javascript:")
|| url.starts_with("ftp:")
|| url.starts_with("irc:")
|| url.starts_with("data:")
{
return;
}
let (url, fragment) = match url.split_once('#') {
None => (url, None),
Some((url, fragment)) => (url, Some(fragment)),
};
// NB: the `splitn` always succeeds, even if the delimiter is not present.
let url = url.splitn(2, '?').next().unwrap();
// Once we've plucked out the URL, parse it using our base url and
// then try to extract a file path.
let mut path = file.to_path_buf();
if !base.is_empty() || !url.is_empty() {
path.pop();
for part in Path::new(base).join(url).components() {
match part {
Component::Prefix(_) | Component::RootDir => {
// Avoid absolute paths as they make the docs not
// relocatable by making assumptions on where the docs
// are hosted relative to the site root.
*errors = true;
println!(
"{}:{}: absolute path - {}",
pretty_file.display(),
i + 1,
Path::new(base).join(url).display()
);
return;
}
Component::CurDir => {}
Component::ParentDir => {
path.pop();
}
Component::Normal(s) => {
path.push(s);
}
}
}
}
// Alright, if we've found a file name then this file had better
// exist! If it doesn't then we register and print an error.
if path.exists() {
if path.is_dir() {
// Links to directories show as directory listings when viewing
// the docs offline so it's best to avoid them.
*errors = true;
let pretty_path = path.strip_prefix(root).unwrap_or(&path);
println!(
"{}:{}: directory link - {}",
pretty_file.display(),
i + 1,
pretty_path.display()
);
return;
}
if let Some(extension) = path.extension() {
// Ignore none HTML files.
if extension != "html" {
return;
}
}
let res = load_file(cache, root, &path, FromRedirect(false));
let (pretty_path, contents) = match res {
Ok(res) => res,
Err(LoadError::IOError(err)) => {
panic!("error loading {}: {}", path.display(), err);
}
Err(LoadError::BrokenRedirect(target, _)) => {
*errors = true;
println!(
"{}:{}: broken redirect to {}",
pretty_file.display(),
i + 1,
target.display()
);
return;
}
Err(LoadError::IsRedirect) => unreachable!(),
};
if let Some(ref fragment) = fragment {
// Fragments like `#1-6` are most likely line numbers to be
// interpreted by javascript, so we're ignoring these
if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
return;
}
// These appear to be broken in mdbook right now?
if fragment.starts_with('-') {
return;
}
let entry = &mut cache.get_mut(&pretty_path).unwrap();
entry.parse_ids(&pretty_path, &contents, errors);
if !entry.ids.contains(*fragment) && !is_exception(file, &format!("#{}", fragment))
{
*errors = true;
print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
};
}
} else {
let pretty_path = path.strip_prefix(root).unwrap_or(&path);
if !is_exception(file, pretty_path.to_str().unwrap()) {
*errors = true;
print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
println!("{}", pretty_path.display());
}
}
});
// Search for intra-doc links that rustdoc didn't warn about
// FIXME(#77199, 77200) Rustdoc should just warn about these directly.
// NOTE: only looks at one line at a time; in practice this should find most links
for (i, line) in contents.lines().enumerate() {
for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
if !is_intra_doc_exception(file, &broken_link[1]) {
*errors = true;
print!("{}:{}: broken intra-doc link - ", pretty_file.display(), i + 1);
println!("{}", &broken_link[0]);
}
}
}
Some(pretty_file)
}
fn load_file(
cache: &mut Cache,
root: &Path,
file: &Path,
redirect: Redirect,
) -> Result<(PathBuf, Rc<String>), LoadError> {
let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));
let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
Entry::Occupied(entry) => (None, entry.get().source.clone()),
Entry::Vacant(entry) => {
let contents = match fs::read_to_string(file) {
Ok(s) => Rc::new(s),
Err(err) => {
return Err(if let FromRedirect(true) = redirect {
LoadError::BrokenRedirect(file.to_path_buf(), err)
} else {
LoadError::IOError(err)
});
}
};
let maybe = maybe_redirect(&contents);
if maybe.is_some() {
if let SkipRedirect = redirect {
return Err(LoadError::IsRedirect);
}
} else {
entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
}
(maybe, contents)
}
};
match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
None => Ok((pretty_file, contents)),
}
}
/// If the given HTML file contents is an HTML redirect, this returns the
/// destination path given in the redirect.
fn maybe_redirect(source: &str) -> Option<String> {
const REDIRECT: &str = "<p>Redirecting to <a href=";
@ -396,9 +472,9 @@ fn maybe_redirect(source: &str) -> Option<String> {
})
}
fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(source: &str, attr: &str, mut f: F) {
let mut base = "";
for (i, mut line) in contents.lines().enumerate() {
for (i, mut line) in source.lines().enumerate() {
while let Some(j) = line.find(attr) {
let rest = &line[j + attr.len()..];
// The base tag should always be the first link in the document so
@ -437,3 +513,18 @@ fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str,
}
}
}
fn parse_ids(ids: &mut HashSet<String>, file: &str, source: &str, report: &mut Report) {
if ids.is_empty() {
with_attrs_in_source(source, " id", |fragment, i, _| {
let frag = fragment.trim_start_matches("#").to_owned();
let encoded = small_url_encode(&frag);
if !ids.insert(frag) {
report.errors += 1;
println!("{}:{}: id is not unique: `{}`", file, i, fragment);
}
// Just in case, we also add the encoded id.
ids.insert(encoded);
});
}
}

View File

@ -0,0 +1,5 @@
<html>
<body>
<a href="bar.html">test</a>
</body>
</html>

View File

@ -0,0 +1,5 @@
<html>
<body>
<a href="#somefrag">test</a>
</body>
</html>

View File

@ -0,0 +1,4 @@
<html>
<body>
</body>
</html>

View File

@ -0,0 +1,5 @@
<html>
<body>
<a href="../bar.html#somefrag">test</a>
</body>
</html>

View File

@ -0,0 +1,5 @@
<html>
<body>
<a href="redir-bad.html">bad redir</a>
</body>
</html>

View File

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="refresh" content="0;URL=sometarget">
</head>
<body>
<p>Redirecting to <a href="sometarget">sometarget</a>...</p>
<script>location.replace("sometarget" + location.search + location.hash);</script>
</body>
</html>

View File

@ -0,0 +1,77 @@
use std::path::Path;
use std::process::{Command, ExitStatus};
fn run(dirname: &str) -> (ExitStatus, String, String) {
let output = Command::new(env!("CARGO_BIN_EXE_linkchecker"))
.current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"))
.arg(dirname)
.output()
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
(output.status, stdout, stderr)
}
fn broken_test(dirname: &str, expected: &str) {
let (status, stdout, stderr) = run(dirname);
assert!(!status.success());
if !stdout.contains(expected) {
panic!(
"stdout did not contain expected text: {}\n\
--- stdout:\n\
{}\n\
--- stderr:\n\
{}\n",
expected, stdout, stderr
);
}
}
fn valid_test(dirname: &str) {
let (status, stdout, stderr) = run(dirname);
if !status.success() {
panic!(
"test did not succeed as expected\n\
--- stdout:\n\
{}\n\
--- stderr:\n\
{}\n",
stdout, stderr
);
}
}
#[test]
fn valid() {
valid_test("valid/inner");
}
#[test]
fn basic_broken() {
broken_test("basic_broken", "bar.html");
}
#[test]
fn broken_fragment_local() {
broken_test("broken_fragment_local", "#somefrag");
}
#[test]
fn broken_fragment_remote() {
broken_test("broken_fragment_remote/inner", "#somefrag");
}
#[test]
fn broken_redir() {
broken_test("broken_redir", "sometarget");
}
#[test]
fn directory_link() {
broken_test("directory_link", "somedir");
}
#[test]
fn redirect_loop() {
broken_test("redirect_loop", "redir-bad.html");
}

View File

@ -0,0 +1,5 @@
<html>
<body>
<a href="somedir">dir link</a>
</body>
</html>

View File

@ -0,0 +1,4 @@
<html>
<body>
</body>
</html>

View File

@ -0,0 +1,5 @@
<html>
<body>
<a href="redir-bad.html">loop link</a>
</body>
</html>

View File

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="refresh" content="0;URL=redir-bad.html">
</head>
<body>
<p>Redirecting to <a href="redir-bad.html">redir-bad.html</a>...</p>
<script>location.replace("redir-bad.html" + location.search + location.hash);</script>
</body>
</html>

View File

@ -0,0 +1,7 @@
<html>
<body>
<h2 id="barfrag">Bar</h2>
</body>
</html>

View File

@ -0,0 +1,14 @@
<html>
<body>
<a href="#localfrag">test local frag</a>
<a href="../outer.html">remote link</a>
<a href="../outer.html#somefrag">remote link with fragment</a>
<a href="bar.html">this book</a>
<a href="bar.html#barfrag">this book with fragment</a>
<a href="https://example.com/doesnotexist">external links not validated</a>
<a href="redir.html#redirfrag">Redirect</a>
<h2 id="localfrag">Local</h2>
</body>
</html>

View File

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="refresh" content="0;URL=xxx">
</head>
<body>
<p>Redirecting to <a href="xxx">xxx</a>...</p>
<script>location.replace("xxx" + location.search + location.hash);</script>
These files are skipped, but probably shouldn't be.
</body>
</html>

View File

@ -0,0 +1,5 @@
<html>
<body>
<h2 id="redirfrag">Redir</h2>
</body>
</html>

View File

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="refresh" content="0;URL=redir-target.html">
</head>
<body>
<p>Redirecting to <a href="redir-target.html">redir-target.html</a>...</p>
<script>location.replace("redir-target.html" + location.search + location.hash);</script>
</body>
</html>

View File

@ -0,0 +1,5 @@
<html>
<body>
<a id="somefrag"></a>
</body>
</html>