Add ability to use custom delimiters

lex.go

@@ -127,6 +127,13 @@ var key = map[string]itemType{
 
 const eof = -1
 
+const (
+	defaultLeftDelim  = "{{"
+	defaultRightDelim = "}}"
+	leftComment       = "{*"
+	rightComment      = "*}"
+)
+
 // stateFn represents the state of the scanner as a function that returns the next state.
 type stateFn func(*lexer) stateFn
 
@@ -142,6 +149,17 @@ type lexer struct {
 	items      chan item // channel of scanned items
 	parenDepth int       // nesting depth of ( ) exprs
 	lastType   itemType
+	leftDelim  string
+	rightDelim string
+}
+
+func (l *lexer) setDelimiters(leftDelim, rightDelim string) {
+	if leftDelim != "" {
+		l.leftDelim = leftDelim
+	}
+	if rightDelim != "" {
+		l.rightDelim = rightDelim
+	}
 }
 
 // next returns the next rune in the input.
@@ -226,41 +244,39 @@ func (l *lexer) drain() {
 }
 
 // lex creates a new scanner for the input string.
-func lex(name, input string) *lexer {
-
+func lex(name, input string, run bool) *lexer {
 	l := &lexer{
-		name:  name,
-		input: input,
-		items: make(chan item),
+		name:       name,
+		input:      input,
+		items:      make(chan item),
+		leftDelim:  defaultLeftDelim,
+		rightDelim: defaultRightDelim,
+	}
+	if run {
+		l.run()
 	}
-	go l.run()
 	return l
 }
 
 // run runs the state machine for the lexer.
 func (l *lexer) run() {
-	for l.state = lexText; l.state != nil; {
-		l.state = l.state(l)
-	}
-	close(l.items)
+	go func() {
+		for l.state = lexText; l.state != nil; {
+			l.state = l.state(l)
+		}
+		close(l.items)
+	}()
 }
 
-const (
-	leftDelim    = "{{"
-	rightDelim   = "}}"
-	leftComment  = "{*"
-	rightComment = "*}"
-)
-
 // state functions
 func lexText(l *lexer) stateFn {
 	for {
-		if i := strings.IndexByte(l.input[l.pos:], '{'); i == -1 {
+		if i := strings.IndexByte(l.input[l.pos:], l.leftDelim[0]); i == -1 {
 			l.pos = Pos(len(l.input))
 			break
 		} else {
 			l.pos += Pos(i)
-			if strings.HasPrefix(l.input[l.pos:], leftDelim) {
+			if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
 				if l.pos > l.start {
 					l.emit(itemText)
 				}
@@ -286,7 +302,7 @@ func lexText(l *lexer) stateFn {
 }
 
 func lexLeftDelim(l *lexer) stateFn {
-	l.pos += Pos(len(leftDelim))
+	l.pos += Pos(len(l.leftDelim))
 	l.emit(itemLeftDelim)
 	l.parenDepth = 0
 	return lexInsideAction
@@ -306,7 +322,7 @@ func lexComment(l *lexer) stateFn {
 
 // lexRightDelim scans the right delimiter, which is known to be present.
 func lexRightDelim(l *lexer) stateFn {
-	l.pos += Pos(len(rightDelim))
+	l.pos += Pos(len(l.rightDelim))
 	l.emit(itemRightDelim)
 	return lexText
 }
@@ -316,7 +332,7 @@ func lexInsideAction(l *lexer) stateFn {
 	// Either number, quoted string, or identifier.
 	// Spaces separate arguments; runs of spaces turn into itemSpace.
 	// Pipe symbols separate and are emitted.
-	if strings.HasPrefix(l.input[l.pos:], rightDelim) {
+	if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
 		if l.parenDepth == 0 {
 			return lexRightDelim
 		}
@@ -546,7 +562,7 @@ func (l *lexer) atTerminator() bool {
 	// Does r start the delimiter? This can be ambiguous (with delim=="//", $x/2 will
 	// succeed but should fail) but only in extremely rare cases caused by willfully
 	// bad choice of delimiter.
-	if rd, _ := utf8.DecodeRuneInString(rightDelim); rd == r {
+	if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r {
 		return true
 	}
 	return false

lex_test.go

@@ -16,8 +16,8 @@ package jet
 
 import "testing"
 
-func lexerTestCase(t *testing.T, input string, items ...itemType) {
-	lexer := lex("test.flowRender", input)
+func lexerTestCaseCustomLexer(t *testing.T, lexer *lexer, input string, items ...itemType) {
+	t.Helper()
 	for i := 0; i < len(items); i++ {
 		item := lexer.nextItem()
 
@@ -36,6 +36,18 @@ func lexerTestCase(t *testing.T, input string, items ...itemType) {
 	}
 }
 
+func lexerTestCase(t *testing.T, input string, items ...itemType) {
+	lexer := lex("test.flowRender", input, true)
+	lexerTestCaseCustomLexer(t, lexer, input, items...)
+}
+
+func lexerTestCaseCustomDelimiters(t *testing.T, leftDelim, rightDelim, input string, items ...itemType) {
+	lexer := lex("test.customDelimiters", input, false)
+	lexer.setDelimiters(leftDelim, rightDelim)
+	lexer.run()
+	lexerTestCaseCustomLexer(t, lexer, input, items...)
+}
+
 func TestLexer(t *testing.T) {
 	lexerTestCase(t, `{{}}`, itemLeftDelim, itemRightDelim)
 	lexerTestCase(t, `{{ line }}`, itemLeftDelim, itemIdentifier, itemRightDelim)
@@ -53,7 +65,25 @@ func TestLexer(t *testing.T) {
 	lexerTestCase(t, `{{.Ex!1}}`, itemLeftDelim, itemField, itemNot, itemNumber, itemRightDelim)
 	lexerTestCase(t, `{{.Ex==1}}`, itemLeftDelim, itemField, itemEquals, itemNumber, itemRightDelim)
 	lexerTestCase(t, `{{.Ex&&1}}`, itemLeftDelim, itemField, itemAnd, itemNumber, itemRightDelim)
+}
 
+func TestCustomDelimiters(t *testing.T) {
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[]]`, itemLeftDelim, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ line ]]`, itemLeftDelim, itemIdentifier, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ . ]]`, itemLeftDelim, itemIdentifier, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ .Field ]]`, itemLeftDelim, itemField, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ "value" ]]`, itemLeftDelim, itemString, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[ call: value ]]`, itemLeftDelim, itemIdentifier, itemColon, itemIdentifier, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex+1]]`, itemLeftDelim, itemField, itemAdd, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex-1]]`, itemLeftDelim, itemField, itemMinus, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex*1]]`, itemLeftDelim, itemField, itemMul, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex/1]]`, itemLeftDelim, itemField, itemDiv, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex%1]]`, itemLeftDelim, itemField, itemMod, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex=1]]`, itemLeftDelim, itemField, itemAssign, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[Ex:=1]]`, itemLeftDelim, itemIdentifier, itemAssign, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex!1]]`, itemLeftDelim, itemField, itemNot, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex==1]]`, itemLeftDelim, itemField, itemEquals, itemNumber, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[.Ex&&1]]`, itemLeftDelim, itemField, itemAnd, itemNumber, itemRightDelim)
 }
 
 func TestLexNegatives(t *testing.T) {
@@ -68,4 +98,5 @@ func TestLexNegatives(t *testing.T) {
 
 func TestLexer_Bug35(t *testing.T) {
 	lexerTestCase(t, `{{if x>y}}blahblah...{{end}}`, itemLeftDelim, itemIf, itemIdentifier, itemGreat, itemIdentifier, itemRightDelim, itemText, itemLeftDelim, itemEnd, itemRightDelim)
+	lexerTestCaseCustomDelimiters(t, "[[", "]]", `[[if x>y]]blahblah...[[end]]`, itemLeftDelim, itemIf, itemIdentifier, itemGreat, itemIdentifier, itemRightDelim, itemText, itemLeftDelim, itemEnd, itemRightDelim)
 }

parse.go

@@ -167,7 +167,10 @@ func (s *Set) parse(name, text string) (t *Template, err error) {
 	defer t.recover(&err)
 
 	t.ParseName = t.Name
-	t.startParse(lex(t.Name, text))
+	lexer := lex(t.Name, text, false)
+	lexer.setDelimiters(s.leftDelim, s.rightDelim)
+	lexer.run()
+	t.startParse(lexer)
 	t.parseTemplate()
 	t.stopParse()
 

parse_test.go

@@ -26,10 +26,15 @@ var parseSet = NewSet(nil, "./testData")
 
 type ParserTestCase struct {
 	*testing.T
+	set *Set
 }
 
 func (t ParserTestCase) ExpectPrintName(name, input, output string) {
-	template, err := parseSet.parse(name, input)
+	set := parseSet
+	if t.set != nil {
+		set = t.set
+	}
+	template, err := set.parse(name, input)
 	if err != nil {
 		t.Errorf("%q %s", input, err.Error())
 		return
@@ -60,36 +65,43 @@ func (t ParserTestCase) ExpectPrintSame(input string) {
 }
 
 func TestParseTemplateAndImport(t *testing.T) {
-	p := ParserTestCase{t}
+	p := ParserTestCase{T: t}
 	p.TestPrintFile("extends.jet")
 	p.TestPrintFile("imports.jet")
 }
 
 func TestParseTemplateControl(t *testing.T) {
-	p := ParserTestCase{t}
+	p := ParserTestCase{T: t}
 	p.TestPrintFile("if.jet")
 	p.TestPrintFile("range.jet")
 }
 
 func TestParseTemplateExpressions(t *testing.T) {
-	p := ParserTestCase{t}
+	p := ParserTestCase{T: t}
 	p.TestPrintFile("simple_expression.jet")
 	p.TestPrintFile("additive_expression.jet")
 	p.TestPrintFile("multiplicative_expression.jet")
 }
 
 func TestParseTemplateBlockYield(t *testing.T) {
-	p := ParserTestCase{t}
+	p := ParserTestCase{T: t}
 	p.TestPrintFile("block_yield.jet")
 	p.TestPrintFile("new_block_yield.jet")
 }
 
 func TestParseTemplateIndexSliceExpression(t *testing.T) {
-	p := ParserTestCase{t}
+	p := ParserTestCase{T: t}
 	p.TestPrintFile("index_slice_expression.jet")
 }
 
 func TestParseTemplateAssignment(t *testing.T) {
-	p := ParserTestCase{t}
+	p := ParserTestCase{T: t}
 	p.TestPrintFile("assignment.jet")
 }
+
+func TestParseTemplateWithCustomDelimiters(t *testing.T) {
+	set := NewSet(nil, "./testData")
+	set.Delims("[[", "]]")
+	p := ParserTestCase{T: t, set: set}
+	p.TestPrintFile("custom_delimiters.jet")
+}

template.go

@@ -41,6 +41,8 @@ type Set struct {
 	gmx               *sync.RWMutex        // global variables map mutex
 	defaultExtensions []string
 	developmentMode   bool
+	leftDelim         string
+	rightDelim        string
 }
 
 // SetDevelopmentMode set's development mode on/off, in development mode template will be recompiled on every run
@@ -111,6 +113,13 @@ func (s *Set) AddGopathPath(path string) {
 	}
 }
 
+// Delims sets the delimiters to the specified strings. Parsed templates will
+// inherit the settings. Not setting them leaves them at the default: {{ or }}.
+func (s *Set) Delims(left, right string) {
+	s.leftDelim = left
+	s.rightDelim = right
+}
+
 // resolveName try to resolve a template name, the steps as follow
 //	1. try provided path
 //	2. try provided path+defaultExtensions

testData/custom_delimiters.jet

@@ -0,0 +1,21 @@
+[[ . ]]
+[[ singleValue ]]
+[[ nil ]]
+[[ "" ]]
+[[ val.Field ]]
+[[ url: "" ]]
+[[ url: "","" |pipe ]]
+[[ url("") |pipe |pipe ]]
+[[ url("","").Field |pipe ]]
+[[ url("","").Method("") |pipe ]]
+===
+{{.}}
+{{singleValue}}
+{{nil}}
+{{""}}
+{{val.Field}}
+{{url:""}}
+{{url:"", "" | pipe}}
+{{url("") | pipe | pipe}}
+{{url("", "").Field | pipe}}
+{{url("", "").Method("") | pipe}}