Separate ? and - into special tokens

2026-09-19 10:52:48 -07:00 · 2014-10-17 13:30:54 -04:00
parent 7843bcca8d
commit b4ea483c4e
4 changed files with 15 additions and 6 deletions
@@ -31,8 +31,8 @@ class LexerUnitTest < Minitest::Test
  end

  def test_fancy_identifiers
-    tokens = Lexer.new('hi! five?').tokenize
-    assert_equal [[:id,'hi!'], [:id, 'five?'], [:end_of_string]], tokens
+    tokens = Lexer.new('hi five?').tokenize
+    assert_equal [[:id,'hi'], [:id, 'five'], [:question, '?'], [:end_of_string]], tokens
  end

  def test_whitespace