Extract libeditor

Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Aleksey Kladov <aleksey.kladov@gmail.com>"]
 license = "MIT OR Apache-2.0"
 
 [workspace]
-members = [ "tools", "cli" ]
+members = [ "tools", "cli", "libeditor" ]
 
 [dependencies]
 unicode-xid = "0.1.0"

cli/src/main.rs

@@ -52,7 +52,7 @@ fn main() -> Result<()> {
 fn parse() -> Result<String> {
     let text = read_stdin()?;
     let start = Instant::now();
-    let file = libsyntax2::parse(text);
+    let file = libsyntax2::parse(&text);
     eprintln!("elapsed {:?}", start.elapsed());
     let tree = libsyntax2::utils::dump_tree(&file);
     Ok(tree)
@@ -74,7 +74,7 @@ fn render_test(file: &Path, line: usize) -> Result<(String, String)> {
         None => bail!("No test found at line {} at {}", line, file.display()),
         Some((_start_line, test)) => test,
     };
-    let file = libsyntax2::parse(test.text.clone());
+    let file = libsyntax2::parse(&test.text);
     let tree = libsyntax2::utils::dump_tree(&file);
     Ok((test.text, tree))
 }

code/native/Cargo.toml

@@ -16,4 +16,4 @@ neon-build = "0.2.0"
 
 [dependencies]
 neon = "0.2.0"
-libsyntax2 = { path = "../../" }
+libeditor = { path = "../../libeditor" }

code/native/src/lib.rs

@@ -1,42 +1,20 @@
 #[macro_use]
 extern crate neon;
-extern crate libsyntax2;
+extern crate libeditor;
 
-use libsyntax2::{
-    TextRange,
-    File,
-    utils::dump_tree,
-    SyntaxKind::*,
-    algo,
-};
 use neon::prelude::*;
 
 pub struct Wrapper {
-    inner: File,
+    inner: libeditor::File,
 }
 
-impl Wrapper {
-    fn highlight(&self) -> Vec<(TextRange, &'static str)> {
-        let mut res = Vec::new();
-        let syntax = self.inner.syntax();
-        for node in algo::walk::preorder(syntax.as_ref()) {
-            if node.kind() == ERROR {
-                res.push((node.range(), "error"))
-            }
-        }
-        res
-    }
-}
-
-
-
 declare_types! {
     /// A class for generating greeting strings.
     pub class RustFile for Wrapper {
         init(mut cx) {
             let text = cx.argument::<JsString>(0)?.value();
             Ok(Wrapper {
-                inner: File::parse(&text)
+                inner: libeditor::File::new(&text)
             })
         }
 
@@ -45,7 +23,7 @@ fn highlight(&self) -> Vec<(TextRange, &'static str)> {
             let tree = {
                 let guard = cx.lock();
                 let wrapper = this.borrow(&guard);
-                dump_tree(&wrapper.inner.syntax())
+                wrapper.inner.syntax_tree()
             };
             Ok(cx.string(tree.as_str()).upcast())
         }
@@ -55,15 +33,15 @@ fn highlight(&self) -> Vec<(TextRange, &'static str)> {
             let highlights = {
                 let guard = cx.lock();
                 let wrapper = this.borrow(&guard);
-                wrapper.highlight()
+                wrapper.inner.highlight()
             };
             let res = cx.empty_array();
-            for (i, (range, tag)) in highlights.into_iter().enumerate() {
-                let start: u32 = range.start().into();
-                let end: u32 = range.end().into();
+            for (i, hl) in highlights.into_iter().enumerate() {
+                let start: u32 = hl.range.start().into();
+                let end: u32 = hl.range.end().into();
                 let start = cx.number(start);
                 let end = cx.number(end);
-                let tag = cx.string(tag);
+                let tag = cx.string(hl.tag);
                 let hl = cx.empty_array();
                 hl.set(&mut cx, 0, start)?;
                 hl.set(&mut cx, 1, end)?;

libeditor/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "libeditor"
+version = "0.1.0"
+authors = ["Aleksey Kladov <aleksey.kladov@gmail.com>"]
+publish = false
+
+[dependencies]
+libsyntax2 = { path = "../" }
+text_unit = "0.1.2"

libeditor/src/lib.rs

@@ -0,0 +1,53 @@
+extern crate libsyntax2;
+extern crate text_unit;
+
+use libsyntax2::{
+    algo::walk,
+    SyntaxKind::*,
+};
+use text_unit::TextRange;
+
+pub struct File {
+    inner: libsyntax2::File
+}
+
+pub struct HighlightedRange {
+    pub range: TextRange,
+    pub tag: &'static str,
+}
+
+impl File {
+    pub fn new(text: &str) -> File {
+        File {
+            inner: libsyntax2::File::parse(text)
+        }
+    }
+
+    pub fn highlight(&self) -> Vec<HighlightedRange> {
+        let syntax = self.inner.syntax();
+        let mut res = Vec::new();
+        for node in walk::preorder(syntax.as_ref()) {
+            let tag = match node.kind() {
+                ERROR => "error",
+                COMMENT | DOC_COMMENT => "comment",
+                STRING | RAW_STRING | RAW_BYTE_STRING | BYTE_STRING => "string",
+                ATTR => "attribute",
+                NAME_REF => "text",
+                NAME => "function",
+                INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE => "literal",
+                LIFETIME => "parameter",
+                k if k.is_keyword() => "keyword",
+                _ => continue,
+            };
+            res.push(HighlightedRange {
+                range: node.range(),
+                tag
+            })
+        }
+        res
+    }
+
+    pub fn syntax_tree(&self) -> String {
+        ::libsyntax2::utils::dump_tree(&self.inner.syntax())
+    }
+}

src/algo/walk.rs

@@ -6,7 +6,6 @@ pub fn preorder<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item = SyntaxNodeR
         WalkEvent::Exit(_) => None,
     })
 }
-
 #[derive(Debug, Copy, Clone)]
 pub enum WalkEvent<'a> {
     Enter(SyntaxNodeRef<'a>),

src/ast.rs

@@ -9,7 +9,7 @@ pub struct File<R: TreeRoot = Arc<SyntaxRoot>> {
 impl File<Arc<SyntaxRoot>> {
     pub fn parse(text: &str) -> Self {
         File {
-            syntax: ::parse(text.to_owned()),
+            syntax: ::parse(text),
         }
     }
 }

src/lib.rs

@@ -45,7 +45,7 @@
     yellow::{SyntaxNode, SyntaxNodeRef, SyntaxRoot, TreeRoot},
 };
 
-pub fn parse(text: String) -> SyntaxNode {
+pub fn parse(text: &str) -> SyntaxNode {
     let tokens = tokenize(&text);
     parser_impl::parse::<yellow::GreenBuilder>(text, &tokens)
 }

src/parser_impl/event.rs

@@ -76,7 +76,7 @@ pub(crate) enum Event {
     },
 }
 
-pub(super) fn process(builder: &mut impl Sink, tokens: &[Token], events: Vec<Event>) {
+pub(super) fn process<'a>(builder: &mut impl Sink<'a>, tokens: &[Token], events: Vec<Event>) {
     let mut idx = 0;
 
     let mut holes = Vec::new();

src/parser_impl/mod.rs

@@ -14,10 +14,10 @@
 
 use SyntaxKind::{self, EOF, TOMBSTONE};
 
-pub(crate) trait Sink {
+pub(crate) trait Sink<'a> {
     type Tree;
 
-    fn new(text: String) -> Self;
+    fn new(text: &'a str) -> Self;
 
     fn leaf(&mut self, kind: SyntaxKind, len: TextUnit);
     fn start_internal(&mut self, kind: SyntaxKind);
@@ -27,9 +27,9 @@ pub(crate) trait Sink {
 }
 
 /// Parse a sequence of tokens into the representative node tree
-pub(crate) fn parse<S: Sink>(text: String, tokens: &[Token]) -> S::Tree {
+pub(crate) fn parse<'a, S: Sink<'a>>(text: &'a str, tokens: &[Token]) -> S::Tree {
     let events = {
-        let input = input::ParserInput::new(&text, tokens);
+        let input = input::ParserInput::new(text, tokens);
         let parser_impl = ParserImpl::new(&input);
         let mut parser_api = Parser(parser_impl);
         grammar::file(&mut parser_api);

src/syntax_kinds/generated.rs

@@ -169,6 +169,47 @@ pub enum SyntaxKind {
 use self::SyntaxKind::*;
 
 impl SyntaxKind {
+    pub fn is_keyword(self) -> bool {
+        match self {
+            | USE_KW
+            | FN_KW
+            | STRUCT_KW
+            | ENUM_KW
+            | TRAIT_KW
+            | IMPL_KW
+            | TRUE_KW
+            | FALSE_KW
+            | AS_KW
+            | EXTERN_KW
+            | CRATE_KW
+            | MOD_KW
+            | PUB_KW
+            | SELF_KW
+            | SUPER_KW
+            | IN_KW
+            | WHERE_KW
+            | FOR_KW
+            | LOOP_KW
+            | WHILE_KW
+            | IF_KW
+            | ELSE_KW
+            | MATCH_KW
+            | CONST_KW
+            | STATIC_KW
+            | MUT_KW
+            | UNSAFE_KW
+            | TYPE_KW
+            | REF_KW
+            | LET_KW
+            | MOVE_KW
+            | AUTO_KW
+            | DEFAULT_KW
+            | UNION_KW
+                => true,
+            _ => false
+        }
+    }
+
     pub(crate) fn info(self) -> &'static SyntaxInfo {
         match self {
             SEMI => &SyntaxInfo { name: "SEMI" },

src/syntax_kinds/generated.rs.tera

@@ -24,6 +24,16 @@ pub enum SyntaxKind {
 use self::SyntaxKind::*;
 
 impl SyntaxKind {
+    pub fn is_keyword(self) -> bool {
+        match self {
+{%- for kw in concat(a=keywords, b=contextual_keywords) %}
+            | {{kw | upper}}_KW
+{%- endfor %}
+                => true,
+            _ => false
+        }
+    }
+
     pub(crate) fn info(self) -> &'static SyntaxInfo {
         match self {
 {%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %}

src/yellow/builder.rs

@@ -4,20 +4,18 @@
     SyntaxKind, TextRange, TextUnit,
 };
 
-pub(crate) struct GreenBuilder {
-    text: String,
+pub(crate) struct GreenBuilder<'a> {
+    text: &'a str,
     stack: Vec<GreenNodeBuilder>,
     pos: TextUnit,
     root: Option<GreenNode>,
     errors: Vec<SyntaxError>,
 }
 
-impl GreenBuilder {}
-
-impl Sink for GreenBuilder {
+impl<'a> Sink<'a> for GreenBuilder<'a> {
     type Tree = SyntaxNode;
 
-    fn new(text: String) -> Self {
+    fn new(text: &'a str) -> Self {
         GreenBuilder {
             text,
             stack: Vec::new(),

tests/parser.rs

@@ -8,7 +8,7 @@
 #[test]
 fn parser_tests() {
     dir_tests(&["parser/inline", "parser/ok", "parser/err"], |text| {
-        let file = parse(text.to_string());
+        let file = parse(text);
         dump_tree(&file)
     })
 }