Initial checkin

Rakefile

@@ -9,11 +9,11 @@ require 'rubygems/package_task'
 
 task :default => [:compile, :test]
 
-task :ragel do
+task :gen do
-  sh "find . -name '*.rl' | xargs ragel -C -G2"
+  sh "leg -oext/liquid/liquid_context.c ext/liquid/liquid_context.leg"
 end
 
-task :compile => [:ragel, :liquid_ext]
+task :compile => [:gen, :liquid_ext]
 
 extension = "liquid_ext"
 ext = "ext/liquid"
@@ -21,7 +21,7 @@ ext_so = "#{ext}/#{extension}.#{RbConfig::CONFIG['DLEXT']}"
 ext_files = FileList[
   "#{ext}/*.c",
   "#{ext}/*.h",
-  "#{ext}/*.rl",
+  "#{ext}/*.leg",
   "#{ext}/extconf.rb",
   "#{ext}/Makefile",
   "lib"
@@ -32,7 +32,7 @@ task "lib" do
 end
 
 desc "Builds just the #{extension} extension"
-task extension.to_sym => [:ragel, "#{ext}/Makefile", ext_so ]
+task extension.to_sym => [:gen, "#{ext}/Makefile", ext_so ]
 
 file "#{ext}/Makefile" => ["#{ext}/extconf.rb"] do
   Dir.chdir(ext) do ruby "extconf.rb" end
@@ -45,7 +45,7 @@ file ext_so => ext_files do
   cp ext_so, "lib"
 end
 
-Rake::TestTask.new(:test => [:ragel, 'liquid_ext']) do |t|
+Rake::TestTask.new(:test => [:gen, 'liquid_ext']) do |t|
   t.libs << '.' << 'lib' << 'test'
   t.test_files = FileList['test/liquid/**/*_test.rb']
   t.verbose = false

ext/liquid/Makefile

@@ -115,8 +115,8 @@ extout_prefix =
 target_prefix = 
 LOCAL_LIBS = 
 LIBS =  -lc  -lpthread -ldl -lobjc 
-SRCS = liquid_ext.c
+SRCS = liquid_context.c liquid_ext.c
-OBJS = liquid_ext.o
+OBJS = liquid_context.o liquid_ext.o
 TARGET = liquid_ext
 DLLIB = $(TARGET).bundle
 EXTSTATIC = 

ext/liquid/liquid_context.c

@@ -0,0 +1,338 @@
+/* A recursive-descent parser generated by peg 0.1.9 */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#define YYRULECOUNT 1
+ 
+#include <ruby.h>
+
+#define YYSTYPE VALUE
+#define YY_CTX_LOCAL
+#define YY_CTX_MEMBERS VALUE rb_tokens; char *p; int p_len;
+#define YY_DEBUG
+
+#define YY_INPUT(buf, result, max_size) { \
+    result = ctx->p_len;  \
+    if (result>0 || EOF == ctx->p[0]) { \
+      if (max_size < result) { result = max_size; } \
+      strncpy(buf, ctx->p, result); \
+      buf[result] = '\0'; \
+      yyprintf((stderr, "\nREFILLING %d chars now:<%s>", result, buf));      \
+      ctx->p += result; ctx->p_len -= result; \
+      yyprintf((stderr, "\nREFILLING DONE size left: %d <%s>", ctx->p_len, ctx->p));      \
+    } \
+  }
+
+#ifndef YY_LOCAL
+#define YY_LOCAL(T)	static T
+#endif
+#ifndef YY_ACTION
+#define YY_ACTION(T)	static T
+#endif
+#ifndef YY_RULE
+#define YY_RULE(T)	static T
+#endif
+#ifndef YY_PARSE
+#define YY_PARSE(T)	T
+#endif
+#ifndef YYPARSE
+#define YYPARSE		yyparse
+#endif
+#ifndef YYPARSEFROM
+#define YYPARSEFROM	yyparsefrom
+#endif
+#ifndef YY_INPUT
+#define YY_INPUT(buf, result, max_size)			\
+  {							\
+    int yyc= getchar();					\
+    result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1);	\
+    yyprintf((stderr, "<%c>", yyc));			\
+  }
+#endif
+#ifndef YY_BEGIN
+#define YY_BEGIN	( ctx->begin= ctx->pos, 1)
+#endif
+#ifndef YY_END
+#define YY_END		( ctx->end= ctx->pos, 1)
+#endif
+#ifdef YY_DEBUG
+# define yyprintf(args)	fprintf args
+#else
+# define yyprintf(args)
+#endif
+#ifndef YYSTYPE
+#define YYSTYPE	int
+#endif
+
+#ifndef YY_PART
+
+typedef struct _yycontext yycontext;
+typedef void (*yyaction)(yycontext *ctx, char *yytext, int yyleng);
+typedef struct _yythunk { int begin, end;  yyaction  action;  struct _yythunk *next; } yythunk;
+
+struct _yycontext {
+  char     *buf;
+  int       buflen;
+  int       pos;
+  int       limit;
+  char     *text;
+  int       textlen;
+  int       begin;
+  int       end;
+  int       textmax;
+  yythunk  *thunks;
+  int       thunkslen;
+  int       thunkpos;
+  YYSTYPE   yy;
+  YYSTYPE  *val;
+  YYSTYPE  *vals;
+  int       valslen;
+#ifdef YY_CTX_MEMBERS
+  YY_CTX_MEMBERS
+#endif
+};
+
+#ifdef YY_CTX_LOCAL
+#define YY_CTX_PARAM_	yycontext *yyctx,
+#define YY_CTX_PARAM	yycontext *yyctx
+#define YY_CTX_ARG_	yyctx,
+#define YY_CTX_ARG	yyctx
+#else
+#define YY_CTX_PARAM_
+#define YY_CTX_PARAM
+#define YY_CTX_ARG_
+#define YY_CTX_ARG
+yycontext yyctx0;
+yycontext *yyctx= &yyctx0;
+#endif
+
+YY_LOCAL(int) yyrefill(yycontext *ctx)
+{
+  int yyn;
+  while (ctx->buflen - ctx->pos < 512)
+    {
+      ctx->buflen *= 2;
+      ctx->buf= (char *)realloc(ctx->buf, ctx->buflen);
+    }
+  YY_INPUT((ctx->buf + ctx->pos), yyn, (ctx->buflen - ctx->pos));
+  if (!yyn) return 0;
+  ctx->limit += yyn;
+  return 1;
+}
+
+YY_LOCAL(int) yymatchDot(yycontext *ctx)
+{
+  if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
+  ++ctx->pos;
+  return 1;
+}
+
+YY_LOCAL(int) yymatchChar(yycontext *ctx, int c)
+{
+  if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
+  if ((unsigned char)ctx->buf[ctx->pos] == c)
+    {
+      ++ctx->pos;
+      yyprintf((stderr, "  ok   yymatchChar(ctx, %c) @ %s\n", c, ctx->buf+ctx->pos));
+      return 1;
+    }
+  yyprintf((stderr, "  fail yymatchChar(ctx, %c) @ %s\n", c, ctx->buf+ctx->pos));
+  return 0;
+}
+
+YY_LOCAL(int) yymatchString(yycontext *ctx, char *s)
+{
+  int yysav= ctx->pos;
+  while (*s)
+    {
+      if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
+      if (ctx->buf[ctx->pos] != *s)
+        {
+          ctx->pos= yysav;
+          return 0;
+        }
+      ++s;
+      ++ctx->pos;
+    }
+  return 1;
+}
+
+YY_LOCAL(int) yymatchClass(yycontext *ctx, unsigned char *bits)
+{
+  int c;
+  if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
+  c= (unsigned char)ctx->buf[ctx->pos];
+  if (bits[c >> 3] & (1 << (c & 7)))
+    {
+      ++ctx->pos;
+      yyprintf((stderr, "  ok   yymatchClass @ %s\n", ctx->buf+ctx->pos));
+      return 1;
+    }
+  yyprintf((stderr, "  fail yymatchClass @ %s\n", ctx->buf+ctx->pos));
+  return 0;
+}
+
+YY_LOCAL(void) yyDo(yycontext *ctx, yyaction action, int begin, int end)
+{
+  while (ctx->thunkpos >= ctx->thunkslen)
+    {
+      ctx->thunkslen *= 2;
+      ctx->thunks= (yythunk *)realloc(ctx->thunks, sizeof(yythunk) * ctx->thunkslen);
+    }
+  ctx->thunks[ctx->thunkpos].begin=  begin;
+  ctx->thunks[ctx->thunkpos].end=    end;
+  ctx->thunks[ctx->thunkpos].action= action;
+  ++ctx->thunkpos;
+}
+
+YY_LOCAL(int) yyText(yycontext *ctx, int begin, int end)
+{
+  int yyleng= end - begin;
+  if (yyleng <= 0)
+    yyleng= 0;
+  else
+    {
+      while (ctx->textlen < (yyleng + 1))
+	{
+	  ctx->textlen *= 2;
+	  ctx->text= (char *)realloc(ctx->text, ctx->textlen);
+	}
+      memcpy(ctx->text, ctx->buf + begin, yyleng);
+    }
+  ctx->text[yyleng]= '\0';
+  return yyleng;
+}
+
+YY_LOCAL(void) yyDone(yycontext *ctx)
+{
+  int pos;
+  for (pos= 0;  pos < ctx->thunkpos;  ++pos)
+    {
+      yythunk *thunk= &ctx->thunks[pos];
+      int yyleng= thunk->end ? yyText(ctx, thunk->begin, thunk->end) : thunk->begin;
+      yyprintf((stderr, "DO [%d] %p %s\n", pos, thunk->action, ctx->text));
+      thunk->action(ctx, ctx->text, yyleng);
+    }
+  ctx->thunkpos= 0;
+}
+
+YY_LOCAL(void) yyCommit(yycontext *ctx)
+{
+  if ((ctx->limit -= ctx->pos))
+    {
+      memmove(ctx->buf, ctx->buf + ctx->pos, ctx->limit);
+    }
+  ctx->begin -= ctx->pos;
+  ctx->end -= ctx->pos;
+  ctx->pos= ctx->thunkpos= 0;
+}
+
+YY_LOCAL(int) yyAccept(yycontext *ctx, int tp0)
+{
+  if (tp0)
+    {
+      fprintf(stderr, "accept denied at %d\n", tp0);
+      return 0;
+    }
+  else
+    {
+      yyDone(ctx);
+      yyCommit(ctx);
+    }
+  return 1;
+}
+
+YY_LOCAL(void) yyPush(yycontext *ctx, char *text, int count)  { ctx->val += count; }
+YY_LOCAL(void) yyPop(yycontext *ctx, char *text, int count)   { ctx->val -= count; }
+YY_LOCAL(void) yySet(yycontext *ctx, char *text, int count)   { ctx->val[count]= ctx->yy; }
+
+#endif /* YY_PART */
+
+#define	YYACCEPT	yyAccept(ctx, yythunkpos0)
+
+YY_RULE(int) yy_grammar(yycontext *ctx); /* 1 */
+
+YY_ACTION(void) yy_1_grammar(yycontext *ctx, char *yytext, int yyleng)
+{
+#define yy ctx->yy
+#define yypos ctx->pos
+#define yythunkpos ctx->thunkpos
+  yyprintf((stderr, "do yy_1_grammar\n"));
+   ;
+#undef yythunkpos
+#undef yypos
+#undef yy
+}
+
+YY_RULE(int) yy_grammar(yycontext *ctx)
+{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
+  yyprintf((stderr, "%s\n", "grammar"));  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l1;  if (!yymatchDot(ctx)) goto l1;
+  l2:;	
+  {  int yypos3= ctx->pos, yythunkpos3= ctx->thunkpos;  if (!yymatchDot(ctx)) goto l3;  goto l2;
+  l3:;	  ctx->pos= yypos3; ctx->thunkpos= yythunkpos3;
+  }  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l1;  yyDo(ctx, yy_1_grammar, ctx->begin, ctx->end);
+  yyprintf((stderr, "  ok   %s @ %s\n", "grammar", ctx->buf+ctx->pos));
+  return 1;
+  l1:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
+  yyprintf((stderr, "  fail %s @ %s\n", "grammar", ctx->buf+ctx->pos));
+  return 0;
+}
+
+#ifndef YY_PART
+
+typedef int (*yyrule)(yycontext *ctx);
+
+YY_PARSE(int) YYPARSEFROM(YY_CTX_PARAM_ yyrule yystart)
+{
+  int yyok;
+  if (!yyctx->buflen)
+    {
+      yyctx->buflen= 1024;
+      yyctx->buf= (char *)malloc(yyctx->buflen);
+      yyctx->textlen= 1024;
+      yyctx->text= (char *)malloc(yyctx->textlen);
+      yyctx->thunkslen= 32;
+      yyctx->thunks= (yythunk *)malloc(sizeof(yythunk) * yyctx->thunkslen);
+      yyctx->valslen= 32;
+      yyctx->vals= (YYSTYPE *)malloc(sizeof(YYSTYPE) * yyctx->valslen);
+      yyctx->begin= yyctx->end= yyctx->pos= yyctx->limit= yyctx->thunkpos= 0;
+    }
+  yyctx->begin= yyctx->end= yyctx->pos;
+  yyctx->thunkpos= 0;
+  yyctx->val= yyctx->vals;
+  yyok= yystart(yyctx);
+  if (yyok) yyDone(yyctx);
+  yyCommit(yyctx);
+  return yyok;
+}
+
+YY_PARSE(int) YYPARSE(YY_CTX_PARAM)
+{
+  return YYPARSEFROM(YY_CTX_ARG_ yy_grammar);
+}
+
+#endif
+ 
+
+VALUE liquid_context_parse_impl(VALUE self, VALUE text) {
+  char *p; 
+  int len; 
+  yycontext ctx;  
+
+  memset(&ctx, 0, sizeof(yycontext));
+  ctx.p = RSTRING_PTR(text);  
+  ctx.p_len = (int) RSTRING_LEN(text);
+  ctx.rb_tokens = rb_ary_new();
+
+  yyprintf((stderr, "About to start: %s %d\n", ctx.p, ctx.p_len));
+  
+  //while(yyparse(&ctx))
+  //  ;
+  
+  yyparse(&ctx);
+  yyparse(&ctx);
+  yyparse(&ctx);
+
+  return ctx.rb_tokens;
+}

ext/liquid/liquid_context.leg

@@ -0,0 +1,47 @@
+
+%{ 
+#include <ruby.h>
+
+#define YYSTYPE VALUE
+#define YY_CTX_LOCAL
+#define YY_CTX_MEMBERS VALUE rb_tokens; char *p; int p_len;
+#define YY_DEBUG
+
+#define YY_INPUT(buf, result, max_size) { \
+    result = ctx->p_len;  \
+    if (result>0 || EOF == ctx->p[0]) { \
+      if (max_size < result) { result = max_size; } \
+      strncpy(buf, ctx->p, result); \
+      buf[result] = '\0'; \
+      yyprintf((stderr, "\nREFILLING %d chars now:<%s>", result, buf));      \
+      ctx->p += result; ctx->p_len -= result; \
+      yyprintf((stderr, "\nREFILLING DONE size left: %d <%s>", ctx->p_len, ctx->p));      \
+    } \
+  }
+%}
+
+grammar  = <.+> { }
+
+%% 
+
+VALUE liquid_context_parse_impl(VALUE self, VALUE text) {
+  char *p; 
+  int len; 
+  yycontext ctx;  
+
+  memset(&ctx, 0, sizeof(yycontext));
+  ctx.p = RSTRING_PTR(text);  
+  ctx.p_len = (int) RSTRING_LEN(text);
+  ctx.rb_tokens = rb_ary_new();
+
+  yyprintf((stderr, "About to start: %s %d\n", ctx.p, ctx.p_len));
+  
+  //while(yyparse(&ctx))
+  //  ;
+  
+  yyparse(&ctx);
+  yyparse(&ctx);
+  yyparse(&ctx);
+
+  return ctx.rb_tokens;
+}

ext/liquid/liquid_ext.rl

@@ -1,159 +0,0 @@
-/*
-  Parser for context#[] method. Generated through ragel from parser.rl 
-  Only modify parser.rl. Run rake ragel afterwards to generate this file. 
-*/
-
-#include <ruby.h>
-
-%%{
-  machine fsm;
-
-  action mark {
-    mark = p;
-  }
-
-  action lookup {
-    EMIT("lookup", Qnil)
-  }
-
-  action call {
-    EMIT("call", Qnil)
-  }
-  action range {
-    EMIT("range", Qnil)
-  }
-
-  constants = ( "true" | "false" | "nil" | "null" );
-
-  # strings 
-  string = "\"" any* "\"" | "'" any* "'";
-
-  # nothingness
-  nil = "nil" | "null" ;
-
-  # numbers
-  integer = ('+'|'-')? digit+;
-  float = ('+'|'-')? digit+ '.' digit+;
-
-  # simple values
-  primitive = (
-
-    integer       >mark %{ 
-      EMIT("id", rb_funcall(rb_cObject, rb_intern("Integer"), 1, rb_str_new(mark, p - mark))); 
-    } |
-
-    float         >mark %{ 
-      EMIT("id", rb_funcall(rb_cObject, rb_intern("Float"), 1, rb_str_new(mark, p - mark))) 
-    } |
-
-    nil           %{ EMIT("id", Qnil) } | 
-    "true"        %{ EMIT("id", Qtrue) } |
-    "false"       %{ EMIT("id", Qfalse) } |
-    
-    string        >mark %{ EMIT("id", rb_str_new(mark + 1, p - mark - 2)) } 
-
-  );
-
-  entity = (
-    ((alpha [A-Za-z0-9_\-]*) - (constants)) >mark %{ 
-      EMIT("id", rb_str_new(mark, p - mark))
-      EMIT("lookup", Qnil) 
-    }
-  );
-
-  # Because of recursion we cannot immediatly resolve the content of this in 
-  # the current grammar. We simply re-invoke the parser here to descend into 
-  # the substring
-  recur = (
-    (any+ - ']') >mark %{
-      VALUE body = rb_str_new(mark, p - mark);
-      liquid_context_parse_impl(body, tokens);
-    }
-  );
-
-  expr = (
-    entity |
-    primitive |    
-    "(" (primitive | entity) ".." (primitive | entity) <: ")"  %range |
-    "[" recur "]" %lookup    
-  );
-
-  hash_accessors  = (
-    "[" recur "]" %call |
-
-    ".first"  %{ 
-      EMIT("buildin", rb_str_new2("first"))
-    } |
-
-    ".last"  %{ 
-      EMIT("buildin", rb_str_new2("last"))
-    } |
-
-    ".size"  %{ 
-      EMIT("buildin", rb_str_new2("size"))      
-    } |
-
-    "." ((alpha [A-Za-z0-9_\-]*) - ("first"|"last"|"size"))  >mark  %{ 
-      EMIT("id", rb_str_new(mark, p - mark))
-      EMIT("call", Qnil) 
-    }
-  );
-
-  main := ( 
-    
-    expr <: (hash_accessors)*
-
-  );
-    
-}%%
-
-%% write data nofinal;
-
-// def self.emit(sym, data, tokens) 
-//   puts "emitting: #{sym} -> #{data.inspect}" if $VERBOSE
-//   tokens.push [sym, data]
-// end
-
-#define EMIT(sym, data) rb_ary_push(tokens, rb_ary_new3(2, ID2SYM(rb_intern(sym)), data)); 
-
-
-void liquid_context_parse_impl(VALUE text, VALUE tokens)
-{
-  char *p;
-  char *pe;
-  char *eof;
-  char *mark;
-  int cs, res = 0;
-
-  if (RSTRING_LEN(text) <= 0) {
-    return;
-  }
-  
-  mark = p = RSTRING_PTR(text); 
-  eof = pe = RSTRING_PTR(text) + RSTRING_LEN(text);    
-
-  %% write init;
-  %% write exec;
-}
-
-VALUE liquid_context_parse(VALUE self, VALUE text) {
-  VALUE tokens;
-
-  //printf("text: %s\n", RSTRING_PTR(text));
-  
-  //Check_Type(text, T_STRING);
-
-  tokens = rb_ary_new();
-  liquid_context_parse_impl(text, tokens);
-  return tokens;
-}
-
-static VALUE rb_Liquid;
-static VALUE rb_Parser;
-
-void Init_liquid_ext()
-{
-  rb_Liquid = rb_define_module("Liquid");
-  rb_Parser = rb_define_class_under(rb_Liquid, "Parser", rb_cObject);
-  rb_define_singleton_method(rb_Parser, "parse", liquid_context_parse, 1);  
-}

ext/liquid/mkmf.log

@@ -1,22 +0,0 @@
-have_library: checking for main() in -lc... -------------------- yes
-
-"/usr/bin/gcc-4.2 -o conftest -I/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include/ruby-1.9.1/x86_64-darwin12.0.0 -I/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include/ruby-1.9.1/ruby/backward -I/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include/ruby-1.9.1 -I. -I'/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include'  -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE  -I'/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include'   -O3 -ggdb -Wextra -Wno-unused-parameter -Wno-parentheses -Wno-long-long -Wno-missing-field-initializers -Wpointer-arith -Wwrite-strings -Wdeclaration-after-statement -Wshorten-64-to-32 -Wimplicit-function-declaration  -pipe conftest.c  -L. -L/Users/tobi/.rbenv/versions/1.9.3-p194-perf/lib -L. -L'/Users/tobi/.rbenv/versions/1.9.3-p194-perf/lib'  -L/usr/local/lib     -lruby-static  -lpthread -ldl -lobjc "
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main() {return 0;}
-/* end */
-
-"/usr/bin/gcc-4.2 -o conftest -I/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include/ruby-1.9.1/x86_64-darwin12.0.0 -I/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include/ruby-1.9.1/ruby/backward -I/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include/ruby-1.9.1 -I. -I'/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include'  -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE  -I'/Users/tobi/.rbenv/versions/1.9.3-p194-perf/include'   -O3 -ggdb -Wextra -Wno-unused-parameter -Wno-parentheses -Wno-long-long -Wno-missing-field-initializers -Wpointer-arith -Wwrite-strings -Wdeclaration-after-statement -Wshorten-64-to-32 -Wimplicit-function-declaration  -pipe conftest.c  -L. -L/Users/tobi/.rbenv/versions/1.9.3-p194-perf/lib -L. -L'/Users/tobi/.rbenv/versions/1.9.3-p194-perf/lib'  -L/usr/local/lib     -lruby-static -lc  -lpthread -ldl -lobjc "
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: /*top*/
-4: int main() {return 0;}
-5: int t() { void ((*volatile p)()); p = (void ((*)()))main; return 0; }
-/* end */
-
---------------------
-