From e9cafdb48e2dc44f825f93b6f7286ce8acf15f08 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Mon, 7 Oct 2024 16:37:35 -0700 Subject: [PATCH] Add test linking This adds the ability to link rust-lang/rust tests to reference rule annotations. Rules now have a link that pops up a list of tests that are associated with that rule. Additionally, there is a new appendix that lists the number of rules in each chapter, how many tests are associated, and various summaries. This requires a local checkout of rust-lang/rust which is pointed to by the `SPEC_RUST_ROOT` environment variable. --- .github/workflows/main.yml | 14 ++- README.md | 4 + book.toml | 1 + docs/authoring.md | 12 ++ mdbook-spec/Cargo.lock | 31 +++++- mdbook-spec/Cargo.toml | 1 + mdbook-spec/src/lib.rs | 60 +++++++++- mdbook-spec/src/rules.rs | 26 ++++- mdbook-spec/src/test_links.rs | 203 ++++++++++++++++++++++++++++++++++ src/SUMMARY.md | 1 + src/test-summary.md | 5 + theme/reference.css | 30 +++++ theme/reference.js | 24 ++++ 13 files changed, 403 insertions(+), 9 deletions(-) create mode 100644 mdbook-spec/src/test_links.rs create mode 100644 src/test-summary.md create mode 100644 theme/reference.js diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0dfd2da3a..4db2538ce 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,6 +35,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master + - name: Checkout rust-lang/rust + uses: actions/checkout@master + with: + repository: rust-lang/rust + path: rust - name: Update rustup run: rustup self update - name: Install Rust @@ -52,16 +57,17 @@ jobs: rustup --version rustc -Vv mdbook --version - - name: Verify the book builds - env: - SPEC_DENY_WARNINGS: 1 - run: mdbook build - name: Style checks working-directory: style-check run: cargo run --locked -- ../src - name: Style fmt working-directory: style-check run: cargo fmt --check + - name: Verify the book builds + env: + SPEC_DENY_WARNINGS: 1 + SPEC_RUST_ROOT: ${{ github.workspace }}/rust + run: mdbook build - name: Check for broken links run: | curl -sSLo linkcheck.sh \ diff --git a/README.md b/README.md index af6310abc..0c7f3c496 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,7 @@ The published site at (or local docs usin ### `SPEC_DENY_WARNINGS` The `SPEC_DENY_WARNINGS=1` environment variable will turn all warnings generated by `mdbook-spec` to errors. This is used in CI to ensure that there aren't any problems with the book content. + +### `SPEC_RUST_ROOT` + +The `SPEC_RUST_ROOT` can be used to point to the directory of a checkout of . This is used by the test-linking feature so that it can find tests linked to reference rules. If this is not set, then the tests won't be linked. diff --git a/book.toml b/book.toml index 404b8cca8..18a99728b 100644 --- a/book.toml +++ b/book.toml @@ -5,6 +5,7 @@ author = "The Rust Project Developers" [output.html] additional-css = ["theme/reference.css"] +additional-js = ["theme/reference.js"] git-repository-url = "https://github.com/rust-lang/reference/" edit-url-template = "https://github.com/rust-lang/reference/edit/master/{path}" smart-punctuation = true diff --git a/docs/authoring.md b/docs/authoring.md index cd5eaa752..fa32fc281 100644 --- a/docs/authoring.md +++ b/docs/authoring.md @@ -99,6 +99,18 @@ When assigning rules to new paragraphs, or when modifying rule names, use the fo * Target specific admonitions should typically be named by the least specific target property to which they apply (e.g. if a rule affects all x86 CPUs, the rule name should include `x86` rather than separately listing `i586`, `i686` and `x86_64`, and if a rule applies to all ELF platforms, it should be named `elf` rather than listing every ELF OS). * Use an appropriately descriptive, but short, name if the language does not provide one. +#### Test rule annotations + +Tests in can be linked to rules in the reference. The rule will include a link to the tests, and there is also an appendix which tracks how the rules are currently linked. + +Tests in the `tests` directory can be annotated with the `//@ reference: x.y.z` header to link it to a rule. The header can be specified multiple times if a single file covers multiple rules. + +You *should* when possible make sure every rule has a test associated with it. This is beneficial for reviewers to see the behavior and readers who may want to see examples of particular behaviors. When adding new rules, you should wait until the reference side is approved before submitting a PR to `rust-lang/rust` (to avoid churn if we decide on different names). + +Prefixed rule names should not be used in tests. That is, do not use something like `asm.rules` when there are specific rules like `asm.rules.reg-not-input`. + +We are not expecting 100% coverage at any time. Although it would be nice, it is unrealistic due to the sequence things are developed, and resources available. + ### Standard library links You should link to the standard library without specifying a URL in a fashion similar to [rustdoc intra-doc links][intra]. Some examples: diff --git a/mdbook-spec/Cargo.lock b/mdbook-spec/Cargo.lock index ff835b409..b101e142f 100644 --- a/mdbook-spec/Cargo.lock +++ b/mdbook-spec/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -412,6 +412,7 @@ dependencies = [ "semver", "serde_json", "tempfile", + "walkdir", ] [[package]] @@ -597,6 +598,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.23" @@ -764,6 +774,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -834,6 +854,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/mdbook-spec/Cargo.toml b/mdbook-spec/Cargo.toml index 703322cb0..c9a6e31af 100644 --- a/mdbook-spec/Cargo.toml +++ b/mdbook-spec/Cargo.toml @@ -19,3 +19,4 @@ regex = "1.9.4" semver = "1.0.21" serde_json = "1.0.113" tempfile = "3.10.1" +walkdir = "2.5.0" diff --git a/mdbook-spec/src/lib.rs b/mdbook-spec/src/lib.rs index add3b25e8..e55935023 100644 --- a/mdbook-spec/src/lib.rs +++ b/mdbook-spec/src/lib.rs @@ -1,7 +1,7 @@ #![deny(rust_2018_idioms, unused_lifetimes)] use crate::rules::Rules; -use anyhow::Result; +use anyhow::{bail, Context, Result}; use mdbook::book::{Book, Chapter}; use mdbook::errors::Error; use mdbook::preprocess::{CmdPreprocessor, Preprocessor, PreprocessorContext}; @@ -10,9 +10,11 @@ use once_cell::sync::Lazy; use regex::{Captures, Regex}; use semver::{Version, VersionReq}; use std::io; +use std::path::PathBuf; mod rules; mod std_links; +mod test_links; /// The Regex for the syntax for blockquotes that have a specific CSS class, /// like `> [!WARNING]`. @@ -47,12 +49,37 @@ pub struct Spec { /// Whether or not warnings should be errors (set by SPEC_DENY_WARNINGS /// environment variable). deny_warnings: bool, + /// Path to the rust-lang/rust git repository (set by SPEC_RUST_ROOT + /// environment variable). + rust_root: Option, + /// The git ref that can be used in a URL to the rust-lang/rust repository. + git_ref: String, } impl Spec { fn new() -> Result { let deny_warnings = std::env::var("SPEC_DENY_WARNINGS").as_deref() == Ok("1"); - Ok(Spec { deny_warnings }) + let rust_root = std::env::var_os("SPEC_RUST_ROOT").map(PathBuf::from); + if deny_warnings && rust_root.is_none() { + bail!("SPEC_RUST_ROOT environment variable must be set"); + } + let git_ref = match git_ref(&rust_root) { + Ok(s) => s, + Err(e) => { + if deny_warnings { + eprintln!("error: {e:?}"); + std::process::exit(1); + } else { + eprintln!("warning: {e:?}"); + "master".into() + } + } + }; + Ok(Spec { + deny_warnings, + rust_root, + git_ref, + }) } /// Generates link references to all rules on all pages, so you can easily @@ -115,6 +142,28 @@ fn to_initial_case(s: &str) -> String { format!("{first}{rest}") } +/// Determines the git ref used for linking to a particular branch/tag in GitHub. +fn git_ref(rust_root: &Option) -> Result { + let Some(rust_root) = rust_root else { + return Ok("master".into()); + }; + let channel = std::fs::read_to_string(rust_root.join("src/ci/channel")) + .context("failed to read src/ci/channel")?; + let git_ref = match channel.trim() { + // nightly/beta are branches, not stable references. Should be ok + // because we're not expecting those channels to be long-lived. + "nightly" => "master".into(), + "beta" => "beta".into(), + "stable" => { + let version = std::fs::read_to_string(rust_root.join("src/version")) + .context("|| failed to read src/version")?; + version.trim().into() + } + ch => bail!("unknown channel {ch}"), + }; + Ok(git_ref) +} + impl Preprocessor for Spec { fn name(&self) -> &str { "spec" @@ -122,6 +171,8 @@ impl Preprocessor for Spec { fn run(&self, _ctx: &PreprocessorContext, mut book: Book) -> Result { let rules = self.collect_rules(&book); + let tests = self.collect_tests(&rules); + let summary_table = test_links::make_summary_table(&book, &tests, &rules); book.for_each_mut(|item| { let BookItem::Chapter(ch) = item else { @@ -132,7 +183,10 @@ impl Preprocessor for Spec { } ch.content = self.admonitions(&ch); ch.content = self.auto_link_references(&ch, &rules); - ch.content = self.render_rule_definitions(&ch.content); + ch.content = self.render_rule_definitions(&ch.content, &tests); + if ch.name == "Test summary" { + ch.content = ch.content.replace("{{summary-table}}", &summary_table); + } }); // Final pass will resolve everything as a std link (or error if the diff --git a/mdbook-spec/src/rules.rs b/mdbook-spec/src/rules.rs index ac402b880..b477ab721 100644 --- a/mdbook-spec/src/rules.rs +++ b/mdbook-spec/src/rules.rs @@ -1,11 +1,13 @@ //! Handling for rule identifiers. +use crate::test_links::RuleToTests; use crate::Spec; use mdbook::book::Book; use mdbook::BookItem; use once_cell::sync::Lazy; use regex::{Captures, Regex}; use std::collections::{BTreeMap, HashSet}; +use std::fmt::Write; use std::path::PathBuf; /// The Regex for rules like `r[foo]`. @@ -76,13 +78,35 @@ impl Spec { /// Converts lines that start with `r[…]` into a "rule" which has special /// styling and can be linked to. - pub fn render_rule_definitions(&self, content: &str) -> String { + pub fn render_rule_definitions(&self, content: &str, tests: &RuleToTests) -> String { RULE_RE .replace_all(content, |caps: &Captures<'_>| { let rule_id = &caps[1]; + let mut test_html = String::new(); + if let Some(tests) = tests.get(rule_id) { + test_html = format!( + "\n\ +     \ + Tests\n\ +
\n\ + Tests with this rule: +
    "); + for test in tests { + writeln!( + test_html, + "
  • {test_path}
    • ", + test_path = test.path, + git_ref = self.git_ref + ) + .unwrap(); + } + + test_html.push_str("
"); + } format!( "
\ [{rule_id}]\ + {test_html}\
\n" ) }) diff --git a/mdbook-spec/src/test_links.rs b/mdbook-spec/src/test_links.rs new file mode 100644 index 000000000..8f847d58c --- /dev/null +++ b/mdbook-spec/src/test_links.rs @@ -0,0 +1,203 @@ +//! Handling for linking tests in rust's testsuite to rule identifiers. + +use crate::{Rules, Spec}; +use mdbook::book::{Book, BookItem}; +use std::collections::HashMap; +use std::fmt::Write; +use std::path::PathBuf; +use walkdir::WalkDir; + +/// Mapping of rule identifier to the tests that include that identifier. +pub type RuleToTests = HashMap>; +/// A test in rustc's test suite. +pub struct Test { + pub path: String, +} + +const TABLE_START: &str = " + + + + + + + + +"; + +/// Generates an HTML table summarizing the coverage of the testsuite. +pub fn make_summary_table(book: &Book, tests: &RuleToTests, rules: &Rules) -> String { + let ch_to_rules = invert_rule_map(rules); + + let mut table = String::from(TABLE_START); + let mut total_rules = 0; + let mut total_tests = 0; + let mut total_uncovered = 0; + + for (item_index, item) in book.iter().enumerate() { + let BookItem::Chapter(ch) = item else { + continue; + }; + let Some(ch_path) = &ch.path else { + continue; + }; + let level = ch + .number + .as_ref() + .map(|ch| ch.len() - 1) + .unwrap_or_default() as u32; + // Note: This path assumes that the summary chapter is in the root of + // the book. If instead it is in a subdirectory, then this needs to + // include relative `../` as needed. + let html_path = ch_path + .with_extension("html") + .to_str() + .unwrap() + .replace('\\', "/"); + let number = ch + .number + .as_ref() + .map(|n| n.to_string()) + .unwrap_or_default(); + let mut num_rules = 0; + let mut num_tests_str = String::from(""); + let mut uncovered_str = String::from(""); + let mut coverage_str = String::from(""); + if let Some(rules) = ch_to_rules.get(ch_path) { + num_rules = rules.len(); + total_rules += num_rules; + let num_tests = rules + .iter() + .map(|rule| tests.get(rule).map(|ts| ts.len()).unwrap_or_default()) + .sum::(); + total_tests += num_tests; + num_tests_str = num_tests.to_string(); + let uncovered_rules: Vec<_> = rules + .iter() + .filter(|rule| !tests.contains_key(rule.as_str())) + .collect(); + let uncovered = uncovered_rules.len(); + total_uncovered += uncovered; + coverage_str = fmt_pct(uncovered, num_rules); + if uncovered == 0 { + uncovered_str = String::from("0"); + } else { + uncovered_str = format!( + "
\n\ + \ + {uncovered}\n\ +
\n\ + Uncovered rules +
    "); + for uncovered_rule in uncovered_rules { + writeln!( + uncovered_str, + "
  • {uncovered_rule}
    • " + ) + .unwrap(); + } + uncovered_str.push_str("
"); + } + } + let indent = " ".repeat(level as usize * 6); + + writeln!( + table, + "\n\ + \n\ + \n\ + \n\ + \n\ + \n\ + ", + name = ch.name, + ) + .unwrap(); + } + + let total_coverage = fmt_pct(total_uncovered, total_rules); + writeln!( + table, + "\n\ + \n\ + \n\ + \n\ + \n\ + \n\ + " + ) + .unwrap(); + table.push_str("
RulesTestsUncovered RulesCoverage
{indent}{number} {name}{num_rules}{num_tests_str}{uncovered_str}{coverage_str}
Total:{total_rules}{total_tests}{total_uncovered}{total_coverage}
\n"); + table +} + +/// Formats a float as a percentage string. +fn fmt_pct(uncovered: usize, total: usize) -> String { + let pct = ((total - uncovered) as f32 / total as f32) * 100.0; + // Round up to tenths of a percent. + let x = (pct * 10.0).ceil() / 10.0; + format!("{x:.1}%") +} + +/// Inverts the rule map so that it is chapter path to set of rules in that +/// chapter. +fn invert_rule_map(rules: &Rules) -> HashMap> { + let mut map: HashMap> = HashMap::new(); + for (rule, (_, path)) in &rules.def_paths { + map.entry(path.clone()).or_default().push(rule.clone()); + } + for value in map.values_mut() { + value.sort(); + } + map +} + +impl Spec { + /// Scans all tests in rust-lang/rust, and creates a mapping of a rule + /// identifier to the set of tests that include that identifier. + pub fn collect_tests(&self, rules: &Rules) -> RuleToTests { + let mut map = HashMap::new(); + let Some(rust_root) = &self.rust_root else { + return map; + }; + for entry in WalkDir::new(rust_root.join("tests")) { + let entry = entry.unwrap(); + let path = entry.path(); + let relative = path.strip_prefix(rust_root).unwrap_or_else(|_| { + panic!("expected root {rust_root:?} to be a prefix of {path:?}") + }); + if path.extension().unwrap_or_default() == "rs" { + let contents = std::fs::read_to_string(path).unwrap(); + for line in contents.lines() { + if let Some(id) = line.strip_prefix("//@ reference: ") { + if rules.interior_prefixes.contains(id) { + let instead: Vec<_> = rules + .def_paths + .keys() + .filter(|key| key.starts_with(&format!("{id}."))) + .collect(); + eprintln!( + "info: Interior prefix rule {id} found in {path:?}\n \ + Tests should not be annotated with prefixed rule names.\n \ + Use the rules from {instead:?} instead." + ); + } else if !rules.def_paths.contains_key(id) { + eprintln!( + "info: Orphaned rule identifier {id} found in {path:?}\n \ + Please update the test to use an existing rule name." + ); + } + let test = Test { + path: relative.to_str().unwrap().replace('\\', "/"), + }; + map.entry(id.to_string()).or_default().push(test); + } + } + } + } + for tests in map.values_mut() { + tests.sort_by(|a, b| a.path.cmp(&b.path)); + } + map + } +} diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 2b17bf45d..91f343b8d 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -132,4 +132,5 @@ - [Appendices](appendices.md) - [Macro Follow-Set Ambiguity Formal Specification](macro-ambiguity.md) - [Influences](influences.md) + - [Test summary](test-summary.md) - [Glossary](glossary.md) diff --git a/src/test-summary.md b/src/test-summary.md new file mode 100644 index 000000000..e4e3e7491 --- /dev/null +++ b/src/test-summary.md @@ -0,0 +1,5 @@ +# Test summary + +The following is a summary of the total tests that are linked to individual rule identifiers within the reference. + +{{summary-table}} diff --git a/theme/reference.css b/theme/reference.css index cbc7aca8a..58be91816 100644 --- a/theme/reference.css +++ b/theme/reference.css @@ -200,3 +200,33 @@ dfn { .history > blockquote { background: #f7c0eb; } + +/* Provides a anchor container for positioning popups. */ +.popup-container { + position: relative; +} +/* In the test summary page, a convenience class for toggling visibility. */ +.popup-hidden { + display: none; +} +/* In the test summary page, the styling for the uncovered rule popup. */ +.uncovered-rules-popup { + position: absolute; + left: -250px; + width: 400px; + background: var(--bg); + border-radius: 4px; + border: 1px solid; + z-index: 1000; + padding: 1rem; +} + +/* The popup that shows when viewing tests for a specific rule. */ +.tests-popup { + color: var(--fg); + background: var(--bg); + border-radius: 4px; + border: 1px solid; + z-index: 1000; + padding: 1rem; +} diff --git a/theme/reference.js b/theme/reference.js new file mode 100644 index 000000000..44a237034 --- /dev/null +++ b/theme/reference.js @@ -0,0 +1,24 @@ +/* On the test summary page, toggles the popup for the uncovered tests. */ +function spec_toggle_uncovered(item_index) { + let el = document.getElementById(`uncovered-${item_index}`); + const currently_hidden = el.classList.contains('popup-hidden'); + const all = document.querySelectorAll('.uncovered-rules-popup'); + all.forEach(element => { + element.classList.add('popup-hidden'); + }); + if (currently_hidden) { + el.classList.remove('popup-hidden'); + } +} + +function spec_toggle_tests(rule_id) { + let el = document.getElementById(`tests-${rule_id}`); + const currently_hidden = el.classList.contains('popup-hidden'); + const all = document.querySelectorAll('.tests-popup'); + all.forEach(element => { + element.classList.add('popup-hidden'); + }); + if (currently_hidden) { + el.classList.remove('popup-hidden'); + } +}