tokenizer

2026-09-15 04:10:40 -07:00 · 2016-06-13 18:38:43 +08:00
parent afb256e5bd
commit 5a33b4b84a
5 changed files with 136 additions and 0 deletions
@@ -35,3 +35,6 @@ jspm_packages

 # Optional REPL history
 .node_repl_history
+
+# vim
+.*.swp
@@ -0,0 +1,7 @@
+language: node_js
+node_js:
+  - "4"
+before_script:
+  - npm install -g mocha
+after_script: 
+  - NODE_ENV=test ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage
@@ -0,0 +1,36 @@
+{
+  "name": "shopify-liquid",
+  "version": "1.0.0",
+  "description": "A Shopify Liquid Implementation in Node.js",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/harttle/shopify-liquid.git"
+  },
+  "keywords": [
+    "Shopify",
+    "Liquid"
+  ],
+  "author": "Harttle",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/harttle/shopify-liquid/issues"
+  },
+  "homepage": "https://github.com/harttle/shopify-liquid#readme",
+  "dependencies": {
+    "bluebird": "^3.4.0",
+    "debug": "^2.2.0",
+    "lodash": "^4.13.1"
+  },
+  "devDependencies": {
+    "chai": "^3.5.0",
+    "chai-as-promised": "^5.3.0",
+    "coveralls": "^2.11.9",
+    "istanbul": "^0.4.3",
+    "mocha": "^2.5.3",
+    "sinon": "^1.17.4"
+  }
+}
@@ -0,0 +1,43 @@
+var chai = require("chai");
+var should = chai.should();
+chai.use(require("chai-as-promised"));
+
+var tokenizer = require('../tokenizer.js');
+
+describe('tokenizer', function() {
+    it('should handle plain HTML', function() {
+        var html = '<html><body><p>Lorem Ipsum</p></body></html>';
+        var tokens = tokenizer.parse(html);
+
+        tokens.length.should.equal(1);
+        tokens[0].value.should.equal(html);
+        tokens[0].type.should.equal('html');
+    });
+    it('should handle tag syntax', function(){
+        var html = '<p>{% for p in a[1]%}</p>';
+        var tokens = tokenizer.parse(html);
+
+        tokens.length.should.equal(3);
+        tokens[1].type.should.equal('tag');
+        tokens[1].value.should.equal('for p in a[1]');
+    });
+    it('should handle output syntax', function(){
+        var html = '<p>{{foo | date: "%Y-%m-%d"}}</p>';
+        var tokens = tokenizer.parse(html);
+
+        tokens.length.should.equal(3);
+        tokens[1].type.should.equal('output');
+        tokens[1].value.should.equal('foo | date: "%Y-%m-%d"');
+    });
+    it('should handle successive output and tags', function(){
+        var html = '{{foo}}{{bar}}{%foo%}{%bar%}';
+        var tokens = tokenizer.parse(html);
+
+        tokens.length.should.equal(4);
+        tokens[0].type.should.equal('output');
+        tokens[3].type.should.equal('tag');
+
+        tokens[1].value.should.equal('bar');
+        tokens[2].value.should.equal('foo');
+    });
+});
@@ -0,0 +1,47 @@
+
+function parse(html){
+    var tokens = [];
+    if(!html) return tokens;
+
+    var syntax = /({%(.*?)%})|({{(.*?)}})/g;
+    var result, htmlFragment;
+    var idx = 0;
+
+    while ((result = syntax.exec(html)) !== null) {
+        if(result.index > idx){
+            htmlFragment = html.slice(idx, result.index);
+            tokens.push({
+                type: 'html',
+                raw: htmlFragment,
+                value: htmlFragment
+            });
+        }
+        if(result[1]){
+            tokens.push({
+                type: 'tag',
+                raw: result[1],
+                value: result[2].trim()
+            });
+        }
+        else{
+            tokens.push({
+                type: 'output',
+                raw: result[3],
+                value: result[4].trim()
+            });
+        }
+        idx = syntax.lastIndex;
+    }
+
+    if(html.length > idx){
+        htmlFragment = html.slice(idx, html.length);
+        tokens.push({
+            type: 'html',
+            raw: htmlFragment,
+            value: htmlFragment
+        });
+    }
+    return tokens;
+}
+
+exports.parse = parse;