From b916e665bb420c862350030660843e51d9d77138 Mon Sep 17 00:00:00 2001 From: Daniel Lohse Date: Wed, 4 Jul 2018 00:19:07 +0200 Subject: [PATCH 1/4] Move and rename delimiter and comment consts to the top This also adds these consts to the lexer struct when calling the constructor. --- lex.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lex.go b/lex.go index ab3a6fd..68b1b89 100644 --- a/lex.go +++ b/lex.go @@ -127,6 +127,13 @@ var key = map[string]itemType{ const eof = -1 +const ( + defaultLeftDelim = "{{" + defaultRightDelim = "}}" + leftComment = "{*" + rightComment = "*}" +) + // stateFn represents the state of the scanner as a function that returns the next state. type stateFn func(*lexer) stateFn @@ -142,6 +149,8 @@ type lexer struct { items chan item // channel of scanned items parenDepth int // nesting depth of ( ) exprs lastType itemType + leftDelim string + rightDelim string } // next returns the next rune in the input. @@ -227,11 +236,12 @@ func (l *lexer) drain() { // lex creates a new scanner for the input string. func lex(name, input string) *lexer { - l := &lexer{ - name: name, - input: input, - items: make(chan item), + name: name, + input: input, + items: make(chan item), + leftDelim: defaultLeftDelim, + rightDelim: defaultRightDelim, } go l.run() return l @@ -245,13 +255,6 @@ func (l *lexer) run() { close(l.items) } -const ( - leftDelim = "{{" - rightDelim = "}}" - leftComment = "{*" - rightComment = "*}" -) - // state functions func lexText(l *lexer) stateFn { for { From f0b53fe7cfc99c37eb64054bad5b2d6a631e4735 Mon Sep 17 00:00:00 2001 From: Daniel Lohse Date: Wed, 4 Jul 2018 00:21:28 +0200 Subject: [PATCH 2/4] Add boolean run to the lexer constructor to enable delayed lexing In order to be able to set custom delimiters it's necessary to explicitly tell it to start the lexing process. This change is backwards compatible, albeit it's also internal because the lexer isn't exported. --- lex.go | 16 ++++++++++------ lex_test.go | 9 +++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/lex.go b/lex.go index 68b1b89..e88ad55 100644 --- a/lex.go +++ b/lex.go @@ -235,7 +235,7 @@ func (l *lexer) drain() { } // lex creates a new scanner for the input string. -func lex(name, input string) *lexer { +func lex(name, input string, run bool) *lexer { l := &lexer{ name: name, input: input, @@ -243,16 +243,20 @@ func lex(name, input string) *lexer { leftDelim: defaultLeftDelim, rightDelim: defaultRightDelim, } - go l.run() + if run { + l.run() + } return l } // run runs the state machine for the lexer. func (l *lexer) run() { - for l.state = lexText; l.state != nil; { - l.state = l.state(l) - } - close(l.items) + go func() { + for l.state = lexText; l.state != nil; { + l.state = l.state(l) + } + close(l.items) + }() } // state functions diff --git a/lex_test.go b/lex_test.go index 14b80e3..0540859 100644 --- a/lex_test.go +++ b/lex_test.go @@ -16,8 +16,8 @@ package jet import "testing" -func lexerTestCase(t *testing.T, input string, items ...itemType) { - lexer := lex("test.flowRender", input) +func lexerTestCaseCustomLexer(t *testing.T, lexer *lexer, input string, items ...itemType) { + t.Helper() for i := 0; i < len(items); i++ { item := lexer.nextItem() @@ -36,6 +36,11 @@ func lexerTestCase(t *testing.T, input string, items ...itemType) { } } +func lexerTestCase(t *testing.T, input string, items ...itemType) { + lexer := lex("test.flowRender", input, true) + lexerTestCaseCustomLexer(t, lexer, input, items...) +} + func TestLexer(t *testing.T) { lexerTestCase(t, `{{}}`, itemLeftDelim, itemRightDelim) lexerTestCase(t, `{{ line }}`, itemLeftDelim, itemIdentifier, itemRightDelim) From e71a709425486ac17650374c75220ad7cd9ef7f3 Mon Sep 17 00:00:00 2001 From: Daniel Lohse Date: Wed, 4 Jul 2018 00:25:03 +0200 Subject: [PATCH 3/4] Allow lexing using custom delimiters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This includes unit tests – the same ones that were already there, just with different delimiters. --- lex.go | 21 +++++++++++++++------ lex_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/lex.go b/lex.go index e88ad55..09f9dbe 100644 --- a/lex.go +++ b/lex.go @@ -153,6 +153,15 @@ type lexer struct { rightDelim string } +func (l *lexer) setDelimiters(leftDelim, rightDelim string) { + if leftDelim != "" { + l.leftDelim = leftDelim + } + if rightDelim != "" { + l.rightDelim = rightDelim + } +} + // next returns the next rune in the input. func (l *lexer) next() rune { if int(l.pos) >= len(l.input) { @@ -262,12 +271,12 @@ func (l *lexer) run() { // state functions func lexText(l *lexer) stateFn { for { - if i := strings.IndexByte(l.input[l.pos:], '{'); i == -1 { + if i := strings.IndexByte(l.input[l.pos:], l.leftDelim[0]); i == -1 { l.pos = Pos(len(l.input)) break } else { l.pos += Pos(i) - if strings.HasPrefix(l.input[l.pos:], leftDelim) { + if strings.HasPrefix(l.input[l.pos:], l.leftDelim) { if l.pos > l.start { l.emit(itemText) } @@ -293,7 +302,7 @@ func lexText(l *lexer) stateFn { } func lexLeftDelim(l *lexer) stateFn { - l.pos += Pos(len(leftDelim)) + l.pos += Pos(len(l.leftDelim)) l.emit(itemLeftDelim) l.parenDepth = 0 return lexInsideAction @@ -313,7 +322,7 @@ func lexComment(l *lexer) stateFn { // lexRightDelim scans the right delimiter, which is known to be present. func lexRightDelim(l *lexer) stateFn { - l.pos += Pos(len(rightDelim)) + l.pos += Pos(len(l.rightDelim)) l.emit(itemRightDelim) return lexText } @@ -323,7 +332,7 @@ func lexInsideAction(l *lexer) stateFn { // Either number, quoted string, or identifier. // Spaces separate arguments; runs of spaces turn into itemSpace. // Pipe symbols separate and are emitted. - if strings.HasPrefix(l.input[l.pos:], rightDelim) { + if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { if l.parenDepth == 0 { return lexRightDelim } @@ -553,7 +562,7 @@ func (l *lexer) atTerminator() bool { // Does r start the delimiter? This can be ambiguous (with delim=="//", $x/2 will // succeed but should fail) but only in extremely rare cases caused by willfully // bad choice of delimiter. - if rd, _ := utf8.DecodeRuneInString(rightDelim); rd == r { + if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r { return true } return false diff --git a/lex_test.go b/lex_test.go index 0540859..d41ea8c 100644 --- a/lex_test.go +++ b/lex_test.go @@ -41,6 +41,13 @@ func lexerTestCase(t *testing.T, input string, items ...itemType) { lexerTestCaseCustomLexer(t, lexer, input, items...) } +func lexerTestCaseCustomDelimiters(t *testing.T, leftDelim, rightDelim, input string, items ...itemType) { + lexer := lex("test.customDelimiters", input, false) + lexer.setDelimiters(leftDelim, rightDelim) + lexer.run() + lexerTestCaseCustomLexer(t, lexer, input, items...) +} + func TestLexer(t *testing.T) { lexerTestCase(t, `{{}}`, itemLeftDelim, itemRightDelim) lexerTestCase(t, `{{ line }}`, itemLeftDelim, itemIdentifier, itemRightDelim) @@ -58,7 +65,25 @@ func TestLexer(t *testing.T) { lexerTestCase(t, `{{.Ex!1}}`, itemLeftDelim, itemField, itemNot, itemNumber, itemRightDelim) lexerTestCase(t, `{{.Ex==1}}`, itemLeftDelim, itemField, itemEquals, itemNumber, itemRightDelim) lexerTestCase(t, `{{.Ex&&1}}`, itemLeftDelim, itemField, itemAnd, itemNumber, itemRightDelim) +} +func TestCustomDelimiters(t *testing.T) { + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[]]`, itemLeftDelim, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ line ]]`, itemLeftDelim, itemIdentifier, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ . ]]`, itemLeftDelim, itemIdentifier, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ .Field ]]`, itemLeftDelim, itemField, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ "value" ]]`, itemLeftDelim, itemString, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ call: value ]]`, itemLeftDelim, itemIdentifier, itemColon, itemIdentifier, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex+1]]`, itemLeftDelim, itemField, itemAdd, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex-1]]`, itemLeftDelim, itemField, itemMinus, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex*1]]`, itemLeftDelim, itemField, itemMul, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex/1]]`, itemLeftDelim, itemField, itemDiv, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex%1]]`, itemLeftDelim, itemField, itemMod, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex=1]]`, itemLeftDelim, itemField, itemAssign, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[Ex:=1]]`, itemLeftDelim, itemIdentifier, itemAssign, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex!1]]`, itemLeftDelim, itemField, itemNot, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex==1]]`, itemLeftDelim, itemField, itemEquals, itemNumber, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex&&1]]`, itemLeftDelim, itemField, itemAnd, itemNumber, itemRightDelim) } func TestLexNegatives(t *testing.T) { @@ -73,4 +98,5 @@ func TestLexNegatives(t *testing.T) { func TestLexer_Bug35(t *testing.T) { lexerTestCase(t, `{{if x>y}}blahblah...{{end}}`, itemLeftDelim, itemIf, itemIdentifier, itemGreat, itemIdentifier, itemRightDelim, itemText, itemLeftDelim, itemEnd, itemRightDelim) + lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[if x>y]]blahblah...[[end]]`, itemLeftDelim, itemIf, itemIdentifier, itemGreat, itemIdentifier, itemRightDelim, itemText, itemLeftDelim, itemEnd, itemRightDelim) } From 4e11fa6f12ce4c3a27ae4224c32a463e7dd8cfcb Mon Sep 17 00:00:00 2001 From: Daniel Lohse Date: Wed, 4 Jul 2018 00:26:24 +0200 Subject: [PATCH 4/4] Enable a Set to have custom delimiters This includes an integration test. --- parse.go | 5 ++++- parse_test.go | 26 +++++++++++++++++++------- template.go | 9 +++++++++ testData/custom_delimiters.jet | 21 +++++++++++++++++++++ 4 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 testData/custom_delimiters.jet diff --git a/parse.go b/parse.go index 07174ef..4b0c1a3 100644 --- a/parse.go +++ b/parse.go @@ -167,7 +167,10 @@ func (s *Set) parse(name, text string) (t *Template, err error) { defer t.recover(&err) t.ParseName = t.Name - t.startParse(lex(t.Name, text)) + lexer := lex(t.Name, text, false) + lexer.setDelimiters(s.leftDelim, s.rightDelim) + lexer.run() + t.startParse(lexer) t.parseTemplate() t.stopParse() diff --git a/parse_test.go b/parse_test.go index d3c68a6..fb83ad4 100644 --- a/parse_test.go +++ b/parse_test.go @@ -26,10 +26,15 @@ var parseSet = NewSet(nil, "./testData") type ParserTestCase struct { *testing.T + set *Set } func (t ParserTestCase) ExpectPrintName(name, input, output string) { - template, err := parseSet.parse(name, input) + set := parseSet + if t.set != nil { + set = t.set + } + template, err := set.parse(name, input) if err != nil { t.Errorf("%q %s", input, err.Error()) return @@ -60,36 +65,43 @@ func (t ParserTestCase) ExpectPrintSame(input string) { } func TestParseTemplateAndImport(t *testing.T) { - p := ParserTestCase{t} + p := ParserTestCase{T: t} p.TestPrintFile("extends.jet") p.TestPrintFile("imports.jet") } func TestParseTemplateControl(t *testing.T) { - p := ParserTestCase{t} + p := ParserTestCase{T: t} p.TestPrintFile("if.jet") p.TestPrintFile("range.jet") } func TestParseTemplateExpressions(t *testing.T) { - p := ParserTestCase{t} + p := ParserTestCase{T: t} p.TestPrintFile("simple_expression.jet") p.TestPrintFile("additive_expression.jet") p.TestPrintFile("multiplicative_expression.jet") } func TestParseTemplateBlockYield(t *testing.T) { - p := ParserTestCase{t} + p := ParserTestCase{T: t} p.TestPrintFile("block_yield.jet") p.TestPrintFile("new_block_yield.jet") } func TestParseTemplateIndexSliceExpression(t *testing.T) { - p := ParserTestCase{t} + p := ParserTestCase{T: t} p.TestPrintFile("index_slice_expression.jet") } func TestParseTemplateAssignment(t *testing.T) { - p := ParserTestCase{t} + p := ParserTestCase{T: t} p.TestPrintFile("assignment.jet") } + +func TestParseTemplateWithCustomDelimiters(t *testing.T) { + set := NewSet(nil, "./testData") + set.Delims("[[", "]]") + p := ParserTestCase{T: t, set: set} + p.TestPrintFile("custom_delimiters.jet") +} diff --git a/template.go b/template.go index ed91657..8afaaf5 100644 --- a/template.go +++ b/template.go @@ -41,6 +41,8 @@ type Set struct { gmx *sync.RWMutex // global variables map mutex defaultExtensions []string developmentMode bool + leftDelim string + rightDelim string } // SetDevelopmentMode set's development mode on/off, in development mode template will be recompiled on every run @@ -111,6 +113,13 @@ func (s *Set) AddGopathPath(path string) { } } +// Delims sets the delimiters to the specified strings. Parsed templates will +// inherit the settings. Not setting them leaves them at the default: {{ or }}. +func (s *Set) Delims(left, right string) { + s.leftDelim = left + s.rightDelim = right +} + // resolveName try to resolve a template name, the steps as follow // 1. try provided path // 2. try provided path+defaultExtensions diff --git a/testData/custom_delimiters.jet b/testData/custom_delimiters.jet new file mode 100644 index 0000000..46d326a --- /dev/null +++ b/testData/custom_delimiters.jet @@ -0,0 +1,21 @@ +[[ . ]] +[[ singleValue ]] +[[ nil ]] +[[ "" ]] +[[ val.Field ]] +[[ url: "" ]] +[[ url: "","" |pipe ]] +[[ url("") |pipe |pipe ]] +[[ url("","").Field |pipe ]] +[[ url("","").Method("") |pipe ]] +=== +{{.}} +{{singleValue}} +{{nil}} +{{""}} +{{val.Field}} +{{url:""}} +{{url:"", "" | pipe}} +{{url("") | pipe | pipe}} +{{url("", "").Field | pipe}} +{{url("", "").Method("") | pipe}}