Revert "Merge pull request #185 from ISSIntel/liquid-utf8"

This reverts commit c5dfcd29b0, reversing
changes made to f7d1e1d0c1.

lib/liquid.rb

@@ -20,15 +20,14 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 module Liquid
-  WordRegex                   = RUBY_VERSION < "1.9" ? '\w' : '[[:word:]]'
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
   FilterArgumentSeparator     = ':'
   VariableAttributeSeparator  = '.'
   TagStart                    = /\{\%/
   TagEnd                      = /\%\}/
-  VariableSignature           = /\(?[#{WordRegex}\-\.\[\]]\)?/o
+  VariableSignature           = /\(?[\w\-\.\[\]]\)?/
-  VariableSegment             = /[#{WordRegex}\-]/o
+  VariableSegment             = /[\w\-]/
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
   VariableIncompleteEnd       = /\}\}?/
@@ -39,7 +38,7 @@ module Liquid
   OtherFilterArgument         = /#{ArgumentSeparator}(?:#{StrictQuotedFragment})/o
   SpacelessFilter             = /^(?:'[^']+'|"[^"]+"|[^'"])*#{FilterSeparator}(?:#{StrictQuotedFragment})(?:#{FirstFilterArgument}(?:#{OtherFilterArgument})*)?/o
   Expression                  = /(?:#{QuotedFragment}(?:#{SpacelessFilter})*)/o
-  TagAttributes               = /(#{WordRegex}+)\s*\:\s*(#{QuotedFragment})/o
+  TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
   AnyStartingTag              = /\{\{|\{\%/
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/o
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/o

lib/liquid/block.rb

@@ -3,7 +3,7 @@ module Liquid
   class Block < Tag
     IsTag             = /^#{TagStart}/o
     IsVariable        = /^#{VariableStart}/o
-    FullToken         = /^#{TagStart}\s*(#{WordRegex}+)\s*(.*)?#{TagEnd}$/o
+    FullToken         = /^#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}$/o
     ContentOfVariable = /^#{VariableStart}(.*)#{VariableEnd}$/o
 
     def parse(tokens)

lib/liquid/htmltags.rb

@@ -1,6 +1,6 @@
 module Liquid
   class TableRow < Block
-    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)/o
+    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/capture.rb

@@ -12,7 +12,7 @@ module Liquid
   # in a sidebar or footer.
   #
   class Capture < Block
-    Syntax = /(#{WordRegex}+)/o
+    Syntax = /(\w+)/
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/for.rb

@@ -44,7 +44,7 @@ module Liquid
   # forloop.last:: Returns true if the item is the last item.
   #
   class For < Block                                             
-    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/ou
+    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
   
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/variable.rb

@@ -23,7 +23,7 @@ module Liquid
         if match[2].match(/#{FilterSeparator}\s*(.*)/o)
           filters = Regexp.last_match(1).scan(FilterParser)
           filters.each do |f|
-            if matches = f.match(/\s*(#{WordRegex}+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
+            if matches = f.match(/\s*(\w+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
               filtername = matches[1]
               filterargs = matches[2].to_s.scan(/(?:\A|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
               @filters << [filtername, filterargs]

test/liquid/assign_test.rb

@@ -12,13 +12,7 @@ class AssignTest < Test::Unit::TestCase
                            '{% assign foo = values %}.{{ foo[1] }}.',
                            'values' => %w{foo bar baz})
   end
-
+  
-  def test_assigned_utf8_variable
-    assert_template_result('.bar.',
-                           "{% assign foo\u6000 = values %}.{{ foo\u6000[1] }}.",
-                           'values' => %w{foo bar baz})
-  end
-
   def test_assign_with_filter
     assert_template_result('.bar.',
                            '{% assign foo = values | split: "," %}.{{ foo[1] }}.',

test/liquid/block_test.rb

@@ -51,14 +51,6 @@ class BlockTest < Test::Unit::TestCase
     end
   end
 
-  def test_with_custom_utf8_tag
-    Liquid::Template.register_tag("testtag\u6000", Block)
-
-    assert_nothing_thrown do
-      template = Liquid::Template.parse( "{% testtag\u6000 something\u6000 %} {% endtesttag\u6000 %}")
-    end
-  end
-
   private
     def block_types(nodelist)
       nodelist.collect { |node| node.class }

test/liquid/capture_test.rb

@@ -7,10 +7,6 @@ class CaptureTest < Test::Unit::TestCase
     assert_template_result("test string", "{% capture 'var' %}test string{% endcapture %}{{var}}", {})
   end
 
-  def test_captures_block_content_in_utf8_variable
-    assert_template_result("test string", "{% capture var\u6000 %}test string{% endcapture %}{{var\u6000}}", {})
-  end
-
   def test_capture_to_variable_from_outer_scope_if_existing
     template_source = <<-END_TEMPLATE
     {% assign var = '' %}

test/liquid/context_test.rb

@@ -98,11 +98,6 @@ class ContextTest < Test::Unit::TestCase
     assert_equal nil, @context['nil']
   end
 
-  def test_utf8_variables
-    @context["chinese\u6000variable"] = 'chinese'
-    assert_equal 'chinese', @context["chinese\u6000variable"]
-  end
-
   def test_variables_not_existing
     assert_equal nil, @context['does_not_exist']
   end

test/liquid/tags/for_tag_test.rb

@@ -25,11 +25,6 @@ HERE
     assert_template_result(expected,template,'array' => [1,2,3])
   end
 
-  def test_utf8_for
-    assigns = {"array\u6000chinese" => [1,2,3]}
-    assert_template_result('123', "{% for item\u6000chinese in array\u6000chinese %}{{ item\u6000chinese }}{% endfor %}", assigns)
-  end
-
   def test_for_reversed
     assigns = {'array' => [ 1, 2, 3] }
     assert_template_result('321','{%for item in array reversed %}{{item}}{%endfor%}',assigns)

test/liquid/tags/html_tag_test.rb

@@ -26,12 +26,6 @@ class HtmlTagTest < Test::Unit::TestCase
                            'numbers' => [])
   end
 
-  def test_utf8_html_table
-    assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td></tr>\n",
-                           "{% tablerow n\u6000 in numbers\u6000 %} {{n\u6000}} {% endtablerow %}",
-                           "numbers\u6000" => [1])
-  end
-
   def test_html_table_with_different_cols
     assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td><td class=\"col2\"> 2 </td><td class=\"col3\"> 3 </td><td class=\"col4\"> 4 </td><td class=\"col5\"> 5 </td></tr>\n<tr class=\"row2\"><td class=\"col1\"> 6 </td></tr>\n",
                            '{% tablerow n in numbers cols:5%} {{n}} {% endtablerow %}',

test/liquid/variable_test.rb

@@ -50,12 +50,6 @@ class VariableTest < Test::Unit::TestCase
     assert_equal [["things",["\"%Y, okay?\"","'the other one'"]]], var.filters
   end
 
-  def test_utf8_filters
-    var = Variable.new("foo | chinese\u6000filter: value\u6000")
-    assert_equal 'foo', var.name
-    assert_equal [["chinese\u6000filter",["value\u6000"]]], var.filters
-  end
-
   def test_filter_with_date_parameter
 
     var = Variable.new(%! '2006-06-06' | date: "%m/%d/%Y"!)