From 5a33b4b84a9568003cba2f1640818cbf1f500a3e Mon Sep 17 00:00:00 2001 From: harttle Date: Mon, 13 Jun 2016 18:38:43 +0800 Subject: [PATCH] tokenizer --- .gitignore | 3 +++ .travis.yml | 7 +++++++ package.json | 36 ++++++++++++++++++++++++++++++++++++ test/tokenizer.js | 43 +++++++++++++++++++++++++++++++++++++++++++ tokenizer.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+) create mode 100644 .travis.yml create mode 100644 package.json create mode 100644 test/tokenizer.js create mode 100644 tokenizer.js diff --git a/.gitignore b/.gitignore index 5148e527a..7bd4e7cbc 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ jspm_packages # Optional REPL history .node_repl_history + +# vim +.*.swp diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..cf6045554 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: node_js +node_js: + - "4" +before_script: + - npm install -g mocha +after_script: + - NODE_ENV=test ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage diff --git a/package.json b/package.json new file mode 100644 index 000000000..647c7d687 --- /dev/null +++ b/package.json @@ -0,0 +1,36 @@ +{ + "name": "shopify-liquid", + "version": "1.0.0", + "description": "A Shopify Liquid Implementation in Node.js", + "main": "index.js", + "scripts": { + "test": "mocha" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/harttle/shopify-liquid.git" + }, + "keywords": [ + "Shopify", + "Liquid" + ], + "author": "Harttle", + "license": "MIT", + "bugs": { + "url": "https://github.com/harttle/shopify-liquid/issues" + }, + "homepage": "https://github.com/harttle/shopify-liquid#readme", + "dependencies": { + "bluebird": "^3.4.0", + "debug": "^2.2.0", + "lodash": "^4.13.1" + }, + "devDependencies": { + "chai": "^3.5.0", + "chai-as-promised": "^5.3.0", + "coveralls": "^2.11.9", + "istanbul": "^0.4.3", + "mocha": "^2.5.3", + "sinon": "^1.17.4" + } +} diff --git a/test/tokenizer.js b/test/tokenizer.js new file mode 100644 index 000000000..8ce69bc49 --- /dev/null +++ b/test/tokenizer.js @@ -0,0 +1,43 @@ +var chai = require("chai"); +var should = chai.should(); +chai.use(require("chai-as-promised")); + +var tokenizer = require('../tokenizer.js'); + +describe('tokenizer', function() { + it('should handle plain HTML', function() { + var html = '

Lorem Ipsum

'; + var tokens = tokenizer.parse(html); + + tokens.length.should.equal(1); + tokens[0].value.should.equal(html); + tokens[0].type.should.equal('html'); + }); + it('should handle tag syntax', function(){ + var html = '

{% for p in a[1]%}

'; + var tokens = tokenizer.parse(html); + + tokens.length.should.equal(3); + tokens[1].type.should.equal('tag'); + tokens[1].value.should.equal('for p in a[1]'); + }); + it('should handle output syntax', function(){ + var html = '

{{foo | date: "%Y-%m-%d"}}

'; + var tokens = tokenizer.parse(html); + + tokens.length.should.equal(3); + tokens[1].type.should.equal('output'); + tokens[1].value.should.equal('foo | date: "%Y-%m-%d"'); + }); + it('should handle successive output and tags', function(){ + var html = '{{foo}}{{bar}}{%foo%}{%bar%}'; + var tokens = tokenizer.parse(html); + + tokens.length.should.equal(4); + tokens[0].type.should.equal('output'); + tokens[3].type.should.equal('tag'); + + tokens[1].value.should.equal('bar'); + tokens[2].value.should.equal('foo'); + }); +}); diff --git a/tokenizer.js b/tokenizer.js new file mode 100644 index 000000000..e4ea0e013 --- /dev/null +++ b/tokenizer.js @@ -0,0 +1,47 @@ + +function parse(html){ + var tokens = []; + if(!html) return tokens; + + var syntax = /({%(.*?)%})|({{(.*?)}})/g; + var result, htmlFragment; + var idx = 0; + + while ((result = syntax.exec(html)) !== null) { + if(result.index > idx){ + htmlFragment = html.slice(idx, result.index); + tokens.push({ + type: 'html', + raw: htmlFragment, + value: htmlFragment + }); + } + if(result[1]){ + tokens.push({ + type: 'tag', + raw: result[1], + value: result[2].trim() + }); + } + else{ + tokens.push({ + type: 'output', + raw: result[3], + value: result[4].trim() + }); + } + idx = syntax.lastIndex; + } + + if(html.length > idx){ + htmlFragment = html.slice(idx, html.length); + tokens.push({ + type: 'html', + raw: htmlFragment, + value: htmlFragment + }); + } + return tokens; +} + +exports.parse = parse;