Compare commits

..
Author SHA1 Message Date
Josh Faigan f3270183a7 tests passing 2024-10-01 14:23:55 -04:00
Josh Faigan 3399981b89 snippet using inner context 2024-09-30 15:58:00 -04:00
Josh Faigan 404d71613c checkpoint 2024-09-30 11:25:48 -04:00
Josh Faigan 98a69c80ef working with default parsing for args 2024-09-27 12:29:03 -04:00
Guilherme Carreiro 30ea917a38 Demo 2024-09-20 13:02:02 +02:00
Guilherme Carreiro d7045f9d64 Add more unit tests 2024-09-20 11:28:00 +02:00
Guilherme Carreiro b397513f8b Introduce inline snippets prototype 2024-09-20 11:24:12 +02:00
Josh Faigan ffe48869be snippets 2024-09-19 12:03:43 -04:00
21 changed files with 479 additions and 258 deletions
+2 -2
View File
@@ -11,10 +11,10 @@ jobs:
strategy:
matrix:
entry:
- { ruby: 3.0, allowed-failure: false } # minimum supported
- { ruby: 2.7, allowed-failure: false } # minimum supported
- { ruby: 3.2, allowed-failure: false }
- { ruby: 3.3, allowed-failure: false } # latest
- { ruby: ruby-head, allowed-failure: false }
- { ruby: ruby-head, allowed-failure: true }
name: Test Ruby ${{ matrix.entry.ruby }}
steps:
- uses: actions/checkout@v3
-1
View File
@@ -6,4 +6,3 @@ pkg
.rvmrc
.bundle
.byebug_history
Gemfile.lock
+1 -1
View File
@@ -1 +1 @@
3.3.4
3.3.0
+4
View File
@@ -28,3 +28,7 @@ group :test do
gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'main'
end
end
group :development do
gem "webrick"
end
+77
View File
@@ -0,0 +1,77 @@
GIT
remote: https://github.com/Shopify/liquid-c.git
revision: 5a786af7284df55e013ea20551c4b688d02e8326
ref: main
specs:
liquid-c (4.2.0)
liquid (>= 5.0.1)
PATH
remote: .
specs:
liquid (5.6.0.alpha)
GEM
remote: https://rubygems.org/
specs:
ast (2.4.2)
base64 (0.2.0)
benchmark-ips (2.13.0)
json (2.7.2)
language_server-protocol (3.17.0.3)
memory_profiler (1.0.1)
minitest (5.22.3)
parallel (1.24.0)
parser (3.3.0.5)
ast (~> 2.4.1)
racc
racc (1.7.3)
rainbow (3.1.1)
rake (13.2.1)
regexp_parser (2.9.0)
rexml (3.2.6)
rubocop (1.61.0)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.30.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.31.2)
parser (>= 3.3.0.4)
rubocop-performance (1.19.1)
rubocop (>= 1.7.0, < 2.0)
rubocop-ast (>= 0.4.0)
rubocop-shopify (2.12.0)
rubocop (~> 1.44)
ruby-progressbar (1.13.0)
stackprof (0.2.26)
terminal-table (3.0.2)
unicode-display_width (>= 1.1.1, < 3)
unicode-display_width (2.5.0)
webrick (1.8.1)
PLATFORMS
ruby
DEPENDENCIES
base64
benchmark-ips
liquid!
liquid-c!
memory_profiler
minitest
rake (~> 13.0)
rubocop (~> 1.61.0)
rubocop-performance
rubocop-shopify (~> 2.12.0)
stackprof
terminal-table
webrick
BUNDLED WITH
2.5.7
-8
View File
@@ -81,14 +81,6 @@ namespace :benchmark do
task :strict do
ruby "./performance/benchmark.rb strict"
end
desc "Run unit benchmarks"
task :unit do
Dir["./performance/unit/*_benchmark.rb"].each do |file|
puts "🧪 Running #{file}"
ruby file
end
end
end
namespace :profile do
+59
View File
@@ -0,0 +1,59 @@
# frozen_string_literal: true
require 'bundler/inline'
gemfile(true) do
source "https://rubygems.org"
gem 'liquid'
end
require 'liquid'
class Parser
def initialize(template)
@template = template
end
def parse
@parsed_template = Liquid::Template.parse(@template)
end
def test_parse
document = @parsed_template.root
variables = []
if document.is_a?(Liquid::Document)
body = document.body
if body.is_a?(Liquid::BlockBody)
body.nodelist.each do |node|
next unless node.is_a?(Liquid::Variable)
puts node.inspect
variable_name = node.name.name
variables << variable_name
end
end
end
puts "Variables: #{variables}"
end
def render
@parsed_template.render
end
end
starter_template = "{{ foo }}"
starter_template_2 = "{{foo}}, {{bar}}"
starter_template_2_1 = "{{ foo }} and {{ bar }}"
starter_template_3 = "{% assign foo = 'bar' %}{{ foo }}"
# Let's start small here
template = <<~LIQUID
{% assign foo = 'bar' %}
{{ foo }}
LIQUID
parser = Parser.new(starter_template)
parser.parse
parser.test_parse
+68 -3
View File
@@ -1,6 +1,71 @@
<p>Hello world!</p>
<!DOCTYPE html>
<html lang="en">
<p>It is {{date}}</p>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Simple Code Editor</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.5/codemirror.min.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.5/theme/dracula.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.5/codemirror.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.5/mode/xml/xml.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.5/mode/htmlmixed/htmlmixed.min.js"></script>
<style>
.liquid, .CodeMirror {
position: fixed;
height: 100vh;
width: 50vw;
top: 0;
font-size: 24px;
}
.liquid {
left: 0;
}
.CodeMirror {
left: 50%;
}
.CodeMirror-hscrollbar {
overflow: hidden;
}
</style>
</head>
<body>
<div class="liquid">
{% snippet "main" %}
<p>Check out the <a href="/products">Products</a> screen </p>
{% # Snippet input %}
{% snippet "input" |type, name| %}
<div>
<label>{{ type | capitalize }}</label>
<input type={{ type }}>
</div>
{% endsnippet %}
{% snippet "league" %}
<h1>Welcome to the league of super evil</h1>
{% endsnippet %}
{% render "league" %}
{% render "input", type: "text" %}
{% render "input", type: "password" %}
{% endsnippet %}
{% render 'main' %}
</div>
<textarea id="code">
{% capture html %}{% render 'main' %}{% endcapture %}
{{ html | escape }}
</textarea>
<script>
const editorElement = document.querySelector('#code')
const editor = CodeMirror.fromTextArea(editorElement, {
lineNumbers: true,
mode: "htmlmixed",
theme: "dracula"
});
</script>
</body>
</html>
+2 -9
View File
@@ -246,17 +246,10 @@ module Liquid
end
def create_variable(token, parse_context)
if token.end_with?("}}")
i = 2
i = 3 if token[i] == "-"
parse_end = token.length - 3
parse_end -= 1 if token[parse_end] == "-"
markup_end = parse_end - i + 1
markup = markup_end <= 0 ? "" : token.byteslice(i, markup_end)
if token =~ ContentOfVariable
markup = Regexp.last_match(1)
return Variable.new(markup, parse_context)
end
BlockBody.raise_missing_variable_terminator(token, parse_context)
end
+1 -155
View File
@@ -1,9 +1,8 @@
# frozen_string_literal: true
require "strscan"
module Liquid
class Lexer1
class Lexer
SPECIALS = {
'|' => :pipe,
'.' => :dot,
@@ -59,157 +58,4 @@ module Liquid
@output << [:end_of_string]
end
end
class Lexer2
CLOSE_ROUND = [:close_round, ")"].freeze
CLOSE_SQUARE = [:close_square, "]"].freeze
COLON = [:colon, ":"].freeze
COMMA = [:comma, ","].freeze
COMPARISION_NOT_EQUAL = [:comparison, "!="].freeze
COMPARISON_CONTAINS = [:comparison, "contains"].freeze
COMPARISON_EQUAL = [:comparison, "=="].freeze
COMPARISON_GREATER_THAN = [:comparison, ">"].freeze
COMPARISON_GREATER_THAN_OR_EQUAL = [:comparison, ">="].freeze
COMPARISON_LESS_THAN = [:comparison, "<"].freeze
COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
CONTAINS = /contains(?=\s)/
DASH = [:dash, "-"].freeze
DOT = [:dot, "."].freeze
DOTDOT = [:dotdot, ".."].freeze
DOT_ORD = ".".ord
DOUBLE_STRING_LITERAL = /"[^\"]*"/
EOS = [:end_of_string].freeze
IDENTIFIER = /[a-zA-Z_][\w-]*\??/
NUMBER_LITERAL = /-?\d+(\.\d+)?/
OPEN_ROUND = [:open_round, "("].freeze
OPEN_SQUARE = [:open_square, "["].freeze
PIPE = [:pipe, "|"].freeze
QUESTION = [:question, "?"].freeze
RUBY_WHITESPACE = [" ", "\t", "\r", "\n", "\f"].freeze
SINGLE_STRING_LITERAL = /'[^\']*'/
WHITESPACE_OR_NOTHING = /\s*/
COMPARISON_JUMP_TABLE = [].tap do |table|
table["=".ord] = [].tap do |sub_table|
sub_table["=".ord] = COMPARISON_EQUAL
sub_table.freeze
end
table["!".ord] = [].tap do |sub_table|
sub_table["=".ord] = COMPARISION_NOT_EQUAL
sub_table.freeze
end
table["<".ord] = [].tap do |sub_table|
sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
sub_table.freeze
end
table[">".ord] = [].tap do |sub_table|
sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
sub_table.freeze
end
table.freeze
end
NEXT_MATCHER_JUMP_TABLE = [].tap do |table|
"a".upto("z") do |c|
table[c.ord] = [:id, IDENTIFIER].freeze
table[c.upcase.ord] = [:id, IDENTIFIER].freeze
end
table["_".ord] = [:id, IDENTIFIER].freeze
"0".upto("9") do |c|
table[c.ord] = [:number, NUMBER_LITERAL].freeze
end
table["-".ord] = [:number, NUMBER_LITERAL].freeze
table["'".ord] = [:string, SINGLE_STRING_LITERAL].freeze
table["\"".ord] = [:string, DOUBLE_STRING_LITERAL].freeze
table.freeze
end
SPECIAL_TABLE = [].tap do |table|
table["|".ord] = PIPE
table[".".ord] = DOT
table[":".ord] = COLON
table[",".ord] = COMMA
table["[".ord] = OPEN_SQUARE
table["]".ord] = CLOSE_SQUARE
table["(".ord] = OPEN_ROUND
table[")".ord] = CLOSE_ROUND
table["?".ord] = QUESTION
table["-".ord] = DASH
end
NUMBER_TABLE = [].tap do |table|
"0".upto("9") do |c|
table[c.ord] = true
end
table.freeze
end
def initialize(input)
@ss = StringScanner.new(input)
end
# rubocop:disable Metrics/BlockNesting
def tokenize
@output = []
until @ss.eos?
@ss.skip(WHITESPACE_OR_NOTHING)
break if @ss.eos?
peeked = @ss.peek_byte
if (special = SPECIAL_TABLE[peeked])
@ss.scan_byte
# Special case for ".."
if special == DOT && @ss.peek_byte == DOT_ORD
@ss.scan_byte
@output << DOTDOT
elsif special == DASH
# Special case for negative numbers
if NUMBER_TABLE[@ss.peek_byte]
@ss.pos -= 1
@output << [:number, @ss.scan(NUMBER_LITERAL)]
else
@output << special
end
else
@output << special
end
elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
@ss.scan_byte
if (found = sub_table[@ss.peek_byte])
@output << found
@ss.scan_byte
else
raise SyntaxError, "Unexpected character #{peeked.chr}"
end
else
type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
if type && (t = @ss.scan(pattern))
# Special case for "contains"
@output << if type == :id && t == "contains"
COMPARISON_CONTAINS
else
[type, t]
end
else
raise SyntaxError, "Unexpected character #{peeked.chr}"
end
end
end
# rubocop:enable Metrics/BlockNesting
@output << EOS
end
end
Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
end
+1
View File
@@ -4,6 +4,7 @@
tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: %{tag}"
assign: "Syntax Error in 'assign' - Valid syntax: assign [var] = [source]"
capture: "Syntax Error in 'capture' - Valid syntax: capture [var]"
snippet: "Syntax Error in 'snippet' - Valid syntax: snippet [quoted string]"
case: "Syntax Error in 'case' - Valid syntax: case [condition]"
case_invalid_when: "Syntax Error in tag 'case' - Valid when condition: {% when [condition] [or condition2...] %}"
case_invalid_else: "Syntax Error in tag 'case' - Valid else condition: {% else %} (no parameters) "
+1 -1
View File
@@ -53,7 +53,7 @@ module Liquid
str = consume
str << variable_lookups
when :open_square
str = consume.dup
str = consume
str << expression
str << consume(:close_square)
str << variable_lookups
+2
View File
@@ -19,6 +19,7 @@ require_relative "tags/comment"
require_relative "tags/raw"
require_relative "tags/render"
require_relative "tags/cycle"
require_relative "tags/snippet"
module Liquid
module Tags
@@ -42,6 +43,7 @@ module Liquid
'if' => If,
'echo' => Echo,
'tablerow' => TableRow,
'snippet' => Snippet,
}.freeze
end
end
+18
View File
@@ -66,6 +66,24 @@ module Liquid
template_name = @template_name_expr
raise ::ArgumentError unless template_name.is_a?(String)
# Inline snippets take precedence over external snippets
if (inline_snippet = context.registers[:inline_snippet][template_name])
inner_context = context.new_isolated_subcontext
snippet_body = inline_snippet[:body]
snippet_args = inline_snippet[:args]
# Validate and set the arguments in the inner context
@attributes.each do |key, value|
unless snippet_args.include?(key)
raise Liquid::ArgumentError, "Invalid argument `#{key}` for snippet `#{template_name}`"
end
inner_context[key] = context.evaluate(value)
end
return output << snippet_body.render(inner_context)
end
partial = PartialCache.load(
template_name,
context: context,
+51
View File
@@ -0,0 +1,51 @@
# frozen_string_literal: true
module Liquid
# @liquid_public_docs
# @liquid_type tag
# @liquid_category theme
# @liquid_name snippet
# @liquid_summary
# Creates a new inline snippet using a string value as the identifier.
# @liquid_description
# You can create inline snippets to make your Liquid code more modular.
# @liquid_syntax
# {% snippet "input" %}
# value
# {% endsnippet %}
class Snippet < Block
SYNTAX = /(#{QuotedString})(?:\s*\|\s*([\w\s,]+)\s*\|)?/o
def initialize(tag_name, markup, options)
super
if markup =~ SYNTAX
@to = Regexp.last_match(1)
args = Regexp.last_match(2)
@args = args ? args.split(/\s*,\s*/) : []
else
raise SyntaxError, options[:locale].t("errors.syntax.snippet")
end
end
def render(context)
context.registers[:inline_snippet] ||= {}
context.registers[:inline_snippet][snippet_id] = {
body: snippet_body,
args: @args,
}
''
end
private
def snippet_id
@to[1, @to.size - 2]
end
def snippet_body
body = @body
body
end
end
end
+1 -4
View File
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
s.license = "MIT"
# s.description = "A secure, non-evaling end user template engine with aesthetic markup."
s.required_ruby_version = ">= 3.0.0"
s.required_ruby_version = ">= 2.7.0"
s.required_rubygems_version = ">= 1.3.7"
s.metadata['allowed_push_host'] = 'https://rubygems.org'
@@ -28,9 +28,6 @@ Gem::Specification.new do |s|
s.require_path = "lib"
s.add_dependency("strscan")
s.add_dependency("bigdecimal")
s.add_development_dependency('rake', '~> 13.0')
s.add_development_dependency('minitest')
end
-1
View File
@@ -3,7 +3,6 @@
require 'benchmark/ips'
require_relative 'theme_runner'
RubyVM::YJIT.enable if defined?(RubyVM::YJIT)
Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
profiler = ThemeRunner.new
-62
View File
@@ -1,62 +0,0 @@
# frozen_string_literal: true
require "benchmark/ips"
# benchmark liquid lexing
require 'liquid'
RubyVM::YJIT.enable
EXPRESSIONS = [
"foo[1..2].baz",
"12.0",
"foo.bar.based",
"21 - 62",
"foo.bar.baz",
"foo > 12",
"foo < 12",
"foo <= 12",
"foo >= 12",
"foo <> 12",
"foo == 12",
"foo != 12",
"foo contains 12",
"foo contains 'bar'",
"foo != 'bar'",
"'foo' contains 'bar'",
'234089',
"foo | default: -1",
]
EXPRESSIONS.each do |expr|
lexer_1_result = Liquid::Lexer1.new(expr).tokenize
lexer_2_result = Liquid::Lexer2.new(expr).tokenize
next if lexer_1_result == lexer_2_result
warn "Lexer1 and Lexer2 results are different for expression: #{expr}"
warn "expected: #{lexer_1_result}"
warn "got: #{lexer_2_result}"
abort
end
Benchmark.ips do |x|
x.config(time: 10, warmup: 5)
x.report("Liquid::Lexer1#tokenize") do
EXPRESSIONS.each do |expr|
l = Liquid::Lexer1.new(expr)
l.tokenize
end
end
x.report("Liquid::Lexer2#tokenize") do
EXPRESSIONS.each do |expr|
l = Liquid::Lexer2.new(expr)
l.tokenize
end
end
x.compare!
end
+1 -1
View File
@@ -32,7 +32,7 @@ class TestDrop < Liquid::Drop
attr_reader :value
def registers
"{#{@value.inspect}=>#{@context.registers[@value].inspect}}"
{ @value => @context.registers[@value] }
end
end
+190
View File
@@ -0,0 +1,190 @@
# frozen_string_literal: true
require 'test_helper'
class SnippetTest < Minitest::Test
include Liquid
def test_valid_inline_snippet
template = <<~LIQUID.strip
{% snippet "input" %}
Hey
{% endsnippet %}
LIQUID
expected = ''
assert_template_result(expected, template)
end
def test_invalid_inline_snippet
template = <<~LIQUID.strip
{% snippet input %}
Hey
{% endsnippet %}
LIQUID
expected = "Syntax Error in 'snippet' - Valid syntax: snippet [quoted string]"
assert_match_syntax_error(expected, template)
end
def test_render_inline_snippet
template = <<~LIQUID.strip
{% snippet "hey" %}
Hey
{% endsnippet %}
{%- render "hey" -%}
LIQUID
expected = <<~OUTPUT
Hey
OUTPUT
assert_template_result(expected, template)
end
def test_render_multiple_inline_snippets
template = <<~LIQUID.strip
{% snippet "input" %}
<input />
{% endsnippet %}
{% snippet "banner" %}
<marquee direction="up" height="100px">
Welcome to my store!
</marquee>
{% endsnippet %}
{%- render "input" -%}
{%- render "banner" -%}
LIQUID
expected = <<~OUTPUT
<input />
<marquee direction="up" height="100px">
Welcome to my store!
</marquee>
OUTPUT
assert_template_result(expected, template)
end
def test_render_inline_snippet_with_argument
template = <<~LIQUID.strip
{% snippet "input" |type| %}
<input type="{{ type }}" />
{% endsnippet %}
{%- render "input", type: "text" -%}
LIQUID
expected = <<~OUTPUT
<input type="text" />
OUTPUT
assert_template_result(expected, template)
end
def test_render_inline_snippet_with_multiple_arguments
template = <<~LIQUID.strip
{% snippet "input" |type, value| %}
<input type="{{ type }}" value="{{ value }}" />
{% endsnippet %}
{%- render "input", type: "text", value: "Hello" -%}
LIQUID
expected = <<~OUTPUT
<input type="text" value="Hello" />
OUTPUT
assert_template_result(expected, template)
end
def test_render_inline_snippets_using_same_argument_name
template = <<~LIQUID.strip
{% snippet "input" |type| %}
<input type="{{ type }}" />
{% endsnippet %}
{% snippet "inputs" |type, value| %}
<input type="{{ type }}" value="{{ value }}" />
{% endsnippet %}
{%- render "input", type: "text" -%}
{%- render "inputs", type: "password", value: "pass" -%}
LIQUID
expected = <<~OUTPUT
<input type="text" />
<input type="password" value="pass" />
OUTPUT
assert_template_result(expected, template)
end
def test_render_inline_snippet_empty_string_when_missing_argument
template = <<~LIQUID.strip
{% snippet "input" |type| %}
<input type="{{ type }}" value="{{ value }}" />
{% endsnippet %}
{%- render "input", type: "text" -%}
LIQUID
expected = <<~OUTPUT
<input type="text" value="" />
OUTPUT
assert_template_result(expected, template)
end
def test_render_inline_snippet_shouldnt_leak_context
template = <<~LIQUID.strip
{% snippet "input" |type, value| %}
<input type="{{ type }}" value="{{ value }}" />
{% endsnippet %}
{%- render "input", type: "text", value: "Hello" -%}
{{ type }}
{{ value }}
LIQUID
expected = <<~OUTPUT
<input type="text" value="Hello" />
OUTPUT
assert_template_result(expected, template)
end
def test_render_multiple_inline_snippets_without_leaking_context
template = <<~LIQUID.strip
{% snippet "input" |type| %}
<input type="{{ type }}" />
{% endsnippet %}
{% snippet "no_leak" %}
<input type="{{ type }}" />
{% endsnippet %}
{%- render "input", type: "text" -%}
{%- render "no_leak" -%}
LIQUID
expected = <<~OUTPUT
<input type="text" />
<input type="" />
OUTPUT
assert_template_result(expected, template)
end
end
-10
View File
@@ -50,14 +50,4 @@ class LexerUnitTest < Minitest::Test
Lexer.new("%").tokenize
end
end
def test_negative_numbers
tokens = Lexer.new("foo | default: -1").tokenize
assert_equal([[:id, 'foo'], [:pipe, '|'], [:id, 'default'], [:colon, ":"], [:number, '-1'], [:end_of_string]], tokens)
end
def test_greater_than_two_digits
tokens = Lexer.new("foo > 12").tokenize
assert_equal([[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]], tokens)
end
end