Lexer: basic comments

src/lexer/comments.rs

@@ -1,11 +1,36 @@
 use lexer::ptr::Ptr;
 
 use {SyntaxKind};
+use syntax_kinds::*;
 
 pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool {
+    if ptr.next_is('!') && ptr.nnext_is('/') {
+        ptr.bump();
+        ptr.bump();
+        bump_until_eol(ptr);
+        true
+    } else {
         false
+    }
 }
 
 pub(crate) fn scan_comment(ptr: &mut Ptr) -> Option<SyntaxKind> {
+    if ptr.next_is('/') {
+        bump_until_eol(ptr);
+        Some(COMMENT)
+    } else {
         None
+    }
+}
+
+
+fn bump_until_eol(ptr: &mut Ptr) {
+    loop {
+        if ptr.next_is('\n') || ptr.next_is('\r') && ptr.nnext_is('\n') {
+            return;
+        }
+        if ptr.bump().is_none() {
+            break;
+        }
+    }
 }

tests/data/lexer/0010_comments.rs

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+// hello
+//! World

tests/data/lexer/0010_comments.txt

@@ -0,0 +1,6 @@
+SHEBANG 19 "#!/usr/bin/env bash"
+WHITESPACE 1 "\n"
+COMMENT 8 "// hello"
+WHITESPACE 1 "\n"
+COMMENT 9 "//! World"
+WHITESPACE 1 "\n"

validation.md

@@ -6,3 +6,4 @@ Fixmes:
   base, and are in range
 * Validation for unclosed char literal
 * Strings are completely wrong: more tests and comparison with libsyntax.
+* Comment lexing is completely wrong