auto merge of #12416 : alexcrichton/rust/highlight, r=huonw

This adds simple syntax highlighting based off libsyntax's lexer to be sure to stay up to date with rust's grammar. Some of the highlighting is a bit ad-hoc, but it definitely seems to get the job done! This currently doesn't highlight rustdoc-rendered function signatures and structs that are emitted to each page because the colors already signify what's clickable and I think we'd have to figure out a different scheme before colorizing them. This does, however, colorize all code examples and source code. Closes #11393
2014-02-23 03:36:56 -08:00 · 2014-02-23 03:36:56 -08:00 · 8786405047
commit 8786405047
parent 551da06157 ad9e26dab3
8 changed files with 239 additions and 26 deletions
--- a/src/librustdoc/html/highlight.rs
+++ b/src/librustdoc/html/highlight.rs
@ -0,0 +1,174 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Basic html highlighting functionality
+//!
+//! This module uses libsyntax's lexer to provide token-based highlighting for
+//! the HTML documentation generated by rustdoc.
+
+use std::str;
+use std::io;
+
+use syntax::parse;
+use syntax::parse::lexer;
+use syntax::diagnostic;
+use syntax::codemap::{BytePos, Span};
+
+use html::escape::Escape;
+
+use t = syntax::parse::token;
+
+/// Highlights some source code, returning the HTML output.
+pub fn highlight(src: &str) -> ~str {
+    let sess = parse::new_parse_sess();
+    let handler = diagnostic::mk_handler();
+    let span_handler = diagnostic::mk_span_handler(handler, sess.cm);
+    let fm = parse::string_to_filemap(sess, src.to_owned(), ~"<stdin>");
+
+    let mut out = io::MemWriter::new();
+    doit(sess,
+         lexer::new_string_reader(span_handler, fm),
+         &mut out).unwrap();
+    str::from_utf8_lossy(out.unwrap()).into_owned()
+}
+
+/// Exhausts the `lexer` writing the output into `out`.
+///
+/// The general structure for this method is to iterate over each token,
+/// possibly giving it an HTML span with a class specifying what flavor of token
+/// it's used. All source code emission is done as slices from the source map,
+/// not from the tokens themselves, in order to stay true to the original
+/// source.
+fn doit(sess: @parse::ParseSess, lexer: lexer::StringReader,
+        out: &mut Writer) -> io::IoResult<()> {
+    use syntax::parse::lexer::Reader;
+
+    try!(write!(out, "<pre class='rust'>\n"));
+    let mut last = BytePos(0);
+    let mut is_attribute = false;
+    let mut is_macro = false;
+    loop {
+        let next = lexer.next_token();
+        let test = if next.tok == t::EOF {lexer.pos.get()} else {next.sp.lo};
+
+        // The lexer consumes all whitespace and non-doc-comments when iterating
+        // between tokens. If this token isn't directly adjacent to our last
+        // token, then we need to emit the whitespace/comment.
+        //
+        // If the gap has any '/' characters then we consider the whole thing a
+        // comment. This will classify some whitespace as a comment, but that
+        // doesn't matter too much for syntax highlighting purposes.
+        if test > last {
+            let snip = sess.cm.span_to_snippet(Span {
+                lo: last,
+                hi: test,
+                expn_info: None,
+            }).unwrap();
+            if snip.contains("/") {
+                try!(write!(out, "<span class='comment'>{}</span>",
+                              Escape(snip)));
+            } else {
+                try!(write!(out, "{}", Escape(snip)));
+            }
+        }
+        last = next.sp.hi;
+        if next.tok == t::EOF { break }
+
+        let klass = match next.tok {
+            // If this '&' token is directly adjacent to another token, assume
+            // that it's the address-of operator instead of the and-operator.
+            // This allows us to give all pointers their own class (~ and @ are
+            // below).
+            t::BINOP(t::AND) if lexer.peek().sp.lo == next.sp.hi => "kw-2",
+            t::AT | t::TILDE => "kw-2",
+
+            // consider this as part of a macro invocation if there was a
+            // leading identifier
+            t::NOT if is_macro => { is_macro = false; "macro" }
+
+            // operators
+            t::EQ | t::LT | t::LE | t::EQEQ | t::NE | t::GE | t::GT |
+                t::ANDAND | t::OROR | t::NOT | t::BINOP(..) | t::RARROW |
+                t::BINOPEQ(..) | t::FAT_ARROW => "op",
+
+            // miscellaneous, no highlighting
+            t::DOT | t::DOTDOT | t::DOTDOTDOT | t::COMMA | t::SEMI |
+                t::COLON | t::MOD_SEP | t::LARROW | t::DARROW | t::LPAREN |
+                t::RPAREN | t::LBRACKET | t::LBRACE | t::RBRACE |
+                t::DOLLAR => "",
+
+            // This is the start of an attribute. We're going to want to
+            // continue highlighting it as an attribute until the ending ']' is
+            // seen, so skip out early. Down below we terminate the attribute
+            // span when we see the ']'.
+            t::POUND => {
+                is_attribute = true;
+                try!(write!(out, r"<span class='attribute'>\#"));
+                continue
+            }
+            t::RBRACKET => {
+                if is_attribute {
+                    is_attribute = false;
+                    try!(write!(out, "]</span>"));
+                    continue
+                } else {
+                    ""
+                }
+            }
+
+            // text literals
+            t::LIT_CHAR(..) | t::LIT_STR(..) | t::LIT_STR_RAW(..) => "string",
+
+            // number literals
+            t::LIT_INT(..) | t::LIT_UINT(..) | t::LIT_INT_UNSUFFIXED(..) |
+                t::LIT_FLOAT(..) | t::LIT_FLOAT_UNSUFFIXED(..) => "number",
+
+            // keywords are also included in the identifier set
+            t::IDENT(ident, _is_mod_sep) => {
+                match t::get_ident(ident).get() {
+                    "ref" | "mut" => "kw-2",
+
+                    "self" => "self",
+                    "false" | "true" => "boolval",
+
+                    "Option" | "Result" => "prelude-ty",
+                    "Some" | "None" | "Ok" | "Err" => "prelude-val",
+
+                    _ if t::is_any_keyword(&next.tok) => "kw",
+                    _ => {
+                        if lexer.peek().tok == t::NOT {
+                            is_macro = true;
+                            "macro"
+                        } else {
+                            "ident"
+                        }
+                    }
+                }
+            }
+
+            t::LIFETIME(..) => "lifetime",
+            t::DOC_COMMENT(..) => "doccomment",
+            t::UNDERSCORE | t::EOF | t::INTERPOLATED(..) => "",
+        };
+
+        // as mentioned above, use the original source code instead of
+        // stringifying this token
+        let snip = sess.cm.span_to_snippet(next.sp).unwrap();
+        if klass == "" {
+            try!(write!(out, "{}", Escape(snip)));
+        } else {
+            try!(write!(out, "<span class='{}'>{}</span>", klass,
+                          Escape(snip)));
+        }
+    }
+
+    write!(out, "</pre>\n")
+}
+
--- a/src/librustdoc/html/markdown.rs
+++ b/src/librustdoc/html/markdown.rs
@ -35,6 +35,8 @@ use std::str;
 use std::intrinsics;
 use std::vec;

+use html::highlight;
+
 /// A unit struct which has the `fmt::Show` trait implemented. When
 /// formatted, this struct will emit the HTML corresponding to the rendered
 /// version of the contained markdown string.
@ -95,6 +97,7 @@ extern {
    fn sd_markdown_free(md: *sd_markdown);

    fn bufnew(unit: libc::size_t) -> *buf;
+    fn bufputs(b: *buf, c: *libc::c_char);
    fn bufrelease(b: *buf);

 }
@ -127,7 +130,27 @@ pub fn render(w: &mut io::Writer, s: &str) -> fmt::Result {
                    asize: text.len() as libc::size_t,
                    unit: 0,
                };
-                (my_opaque.dfltblk)(ob, &buf, lang, opaque);
+                let rendered = if lang.is_null() {
+                    false
+                } else {
+                    vec::raw::buf_as_slice((*lang).data,
+                                           (*lang).size as uint, |rlang| {
+                        let rlang = str::from_utf8(rlang).unwrap();
+                        if rlang.contains("notrust") {
+                            (my_opaque.dfltblk)(ob, &buf, lang, opaque);
+                            true
+                        } else {
+                            false
+                        }
+                    })
+                };
+
+                if !rendered {
+                    let output = highlight::highlight(text).to_c_str();
+                    output.with_ref(|r| {
+                        bufputs(ob, r)
+                    })
+                }
            })
        }
    }
@ -181,7 +204,8 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
                vec::raw::buf_as_slice((*lang).data,
                                       (*lang).size as uint, |lang| {
                    let s = str::from_utf8(lang).unwrap();
-                    (s.contains("should_fail"), s.contains("ignore"))
+                    (s.contains("should_fail"), s.contains("ignore") ||
+                                                s.contains("notrust"))
                })
            };
            if ignore { return }
--- a/src/librustdoc/html/render.rs
+++ b/src/librustdoc/html/render.rs
@ -50,10 +50,10 @@ use syntax::parse::token::InternedString;
 use clean;
 use doctree;
 use fold::DocFolder;
-use html::escape::Escape;
 use html::format::{VisSpace, Method, PuritySpace};
 use html::layout;
 use html::markdown::Markdown;
+use html::highlight;

 /// Major driving force in all rustdoc rendering. This contains information
 /// about where in the tree-like hierarchy rendering is occurring and controls
@ -1091,7 +1091,8 @@ fn item_module(w: &mut Writer, cx: &Context,

 fn item_function(w: &mut Writer, it: &clean::Item,
                 f: &clean::Function) -> fmt::Result {
-    try!(write!(w, "<pre class='fn'>{vis}{purity}fn {name}{generics}{decl}</pre>",
+    try!(write!(w, "<pre class='rust fn'>{vis}{purity}fn \
+                    {name}{generics}{decl}</pre>",
           vis = VisSpace(it.visibility),
           purity = PuritySpace(f.purity),
           name = it.name.get_ref().as_slice(),
@ -1112,7 +1113,7 @@ fn item_trait(w: &mut Writer, it: &clean::Item,
    }

    // Output the trait definition
-    try!(write!(w, "<pre class='trait'>{}trait {}{}{} ",
+    try!(write!(w, "<pre class='rust trait'>{}trait {}{}{} ",
                  VisSpace(it.visibility),
                  it.name.get_ref().as_slice(),
                  t.generics,
@ -1231,7 +1232,7 @@ fn render_method(w: &mut Writer, meth: &clean::Item) -> fmt::Result {

 fn item_struct(w: &mut Writer, it: &clean::Item,
               s: &clean::Struct) -> fmt::Result {
-    try!(write!(w, "<pre class='struct'>"));
+    try!(write!(w, "<pre class='rust struct'>"));
    try!(render_struct(w, it, Some(&s.generics), s.struct_type, s.fields,
                         s.fields_stripped, "", true));
    try!(write!(w, "</pre>"));
@ -1255,7 +1256,7 @@ fn item_struct(w: &mut Writer, it: &clean::Item,
 }

 fn item_enum(w: &mut Writer, it: &clean::Item, e: &clean::Enum) -> fmt::Result {
-    try!(write!(w, "<pre class='enum'>{}enum {}{}",
+    try!(write!(w, "<pre class='rust enum'>{}enum {}{}",
                  VisSpace(it.visibility),
                  it.name.get_ref().as_slice(),
                  e.generics));
@ -1532,7 +1533,7 @@ fn render_impl(w: &mut Writer, i: &clean::Impl,

 fn item_typedef(w: &mut Writer, it: &clean::Item,
                t: &clean::Typedef) -> fmt::Result {
-    try!(write!(w, "<pre class='typedef'>type {}{} = {};</pre>",
+    try!(write!(w, "<pre class='rust typedef'>type {}{} = {};</pre>",
                  it.name.get_ref().as_slice(),
                  t.generics,
                  t.type_));
@ -1625,9 +1626,7 @@ impl<'a> fmt::Show for Source<'a> {
            try!(write!(fmt.buf, "<span id='{0:u}'>{0:1$u}</span>\n", i, cols));
        }
        try!(write!(fmt.buf, "</pre>"));
-        try!(write!(fmt.buf, "<pre class='rust'>"));
-        try!(write!(fmt.buf, "{}", Escape(s.as_slice())));
-        try!(write!(fmt.buf, "</pre>"));
+        try!(write!(fmt.buf, "{}", highlight::highlight(s.as_slice())));
        Ok(())
    }
 }
--- a/src/librustdoc/html/static/main.css
+++ b/src/librustdoc/html/static/main.css
@ -303,3 +303,18 @@ a {
 .stability.Locked { border-color: #0084B6; color: #00668c; }

 :target { background: #FDFFD3; }
+
+pre.rust .kw { color: #cc782f; }
+pre.rust .kw-2 { color: #3bbb33; }
+pre.rust .prelude-ty { color: #3bbb33; }
+pre.rust .number { color: #c13928; }
+pre.rust .self { color: #c13928; }
+pre.rust .boolval { color: #c13928; }
+pre.rust .prelude-val { color: #c13928; }
+pre.rust .op { color: #cc782f; }
+pre.rust .comment { color: #533add; }
+pre.rust .doccomment { color: #d343d0; }
+pre.rust .macro { color: #d343d0; }
+pre.rust .string { color: #c13928; }
+pre.rust .lifetime { color: #d343d0; }
+pre.rust .attribute { color: #d343d0 !important; }
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@ -38,6 +38,7 @@ pub mod core;
 pub mod doctree;
 pub mod fold;
 pub mod html {
+    pub mod highlight;
    pub mod escape;
    pub mod format;
    pub mod layout;
--- a/src/libstd/fmt/mod.rs
+++ b/src/libstd/fmt/mod.rs
@ -82,7 +82,7 @@ function, but the `format!` macro is a syntax extension which allows it to
 leverage named parameters. Named parameters are listed at the end of the
 argument list and have the syntax:

-```ignore
+```notrust
 identifier '=' expression
 ```

@ -107,7 +107,7 @@ and if all references to one argument do not provide a type, then the format `?`
 is used (the type's rust-representation is printed). For example, this is an
 invalid format string:

-```ignore
+```notrust
 {0:d} {0:s}
 ```

@ -123,7 +123,7 @@ must have the type `uint`. Although a `uint` can be printed with `{:u}`, it is
 illegal to reference an argument as such. For example, this is another invalid
 format string:

-```ignore
+```notrust
 {:.*s} {0:u}
 ```

@ -232,7 +232,7 @@ fn main() {
 There are a number of related macros in the `format!` family. The ones that are
 currently implemented are:

-```rust,ignore
+```ignore
 format!      // described above
 write!       // first argument is a &mut io::Writer, the destination
 writeln!     // same as write but appends a newline
@ -276,7 +276,7 @@ references information on the stack. Under the hood, all of
 the related macros are implemented in terms of this. First
 off, some example usage is:

-```rust,ignore
+```ignore
 use std::fmt;

 # fn lol<T>() -> T { fail!() }
@ -334,7 +334,7 @@ This example is the equivalent of `{0:s}` essentially.
 The select method is a switch over a `&str` parameter, and the parameter *must*
 be of the type `&str`. An example of the syntax is:

-```ignore
+```notrust
 {0, select, male{...} female{...} other{...}}
 ```

@ -353,7 +353,7 @@ The plural method is a switch statement over a `uint` parameter, and the
 parameter *must* be a `uint`. A plural method in its full glory can be specified
 as:

-```ignore
+```notrust
 {0, plural, offset=1 =1{...} two{...} many{...} other{...}}
 ```

@ -381,7 +381,7 @@ should not be too alien. Arguments are formatted with python-like syntax,
 meaning that arguments are surrounded by `{}` instead of the C-like `%`. The
 actual grammar for the formatting syntax is:

-```ignore
+```notrust
 format_string := <text> [ format <text> ] *
 format := '{' [ argument ] [ ':' format_spec ] [ ',' function_spec ] '}'
 argument := integer | identifier
@ -896,10 +896,10 @@ impl<'a> Formatter<'a> {
    ///
    /// # Arguments
    ///
-    ///     * is_positive - whether the original integer was positive or not.
-    ///     * prefix - if the '#' character (FlagAlternate) is provided, this
-    ///       is the prefix to put in front of the number.
-    ///     * buf - the byte array that the number has been formatted into
+    /// * is_positive - whether the original integer was positive or not.
+    /// * prefix - if the '#' character (FlagAlternate) is provided, this
+    ///   is the prefix to put in front of the number.
+    /// * buf - the byte array that the number has been formatted into
    ///
    /// This function will correctly account for the flags provided as well as
    /// the minimum width. It will not take precision into account.
--- a/src/libstd/num/mod.rs
+++ b/src/libstd/num/mod.rs
@ -53,7 +53,7 @@ pub trait Zero: Add<Self, Self> {
    ///
    /// # Laws
    ///
-    /// ~~~ignore
+    /// ~~~notrust
    /// a + 0 = a       ∀ a ∈ Self
    /// 0 + a = a       ∀ a ∈ Self
    /// ~~~
@ -79,7 +79,7 @@ pub trait One: Mul<Self, Self> {
    ///
    /// # Laws
    ///
-    /// ~~~ignore
+    /// ~~~notrust
    /// a * 1 = a       ∀ a ∈ Self
    /// 1 * a = a       ∀ a ∈ Self
    /// ~~~
--- a/src/test/run-make/rustdoc-hidden-line/verify.sh
+++ b/src/test/run-make/rustdoc-hidden-line/verify.sh
@ -3,6 +3,6 @@
 file="$1/doc/foo/fn.foo.html"

 grep -v 'invisible' $file &&
-grep '#\[deriving(Eq)\] // Bar' $file
+grep '#.*\[.*deriving.*(.*Eq.*).*\].*//.*Bar' $file

 exit $?