Use a single token for identifiers

2026-09-19 19:00:39 -07:00 · 2014-10-29 11:28:41 -04:00
parent 4dc682313f
commit a07e382617
4 changed files with 11 additions and 9 deletions
@@ -32,7 +32,7 @@ class LexerUnitTest < Minitest::Test

  def test_fancy_identifiers
    tokens = Lexer.new('hi five?').tokenize
-    assert_equal [[:id,'hi'], [:id, 'five'], [:question, '?'], [:end_of_string]], tokens
+    assert_equal [[:id,'hi'], [:id, 'five?'], [:end_of_string]], tokens
  end

  def test_whitespace