[thirdparty/gcc.git] / libgo / go / text / template / parse / lex_test.go

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parse

import (
	"fmt"
	"testing"
)

// Make the types prettyprint.
var itemName = map[itemType]string{
	itemError:        "error",
	itemBool:         "bool",
	itemChar:         "char",
	itemCharConstant: "charconst",
	itemComplex:      "complex",
	itemColonEquals:  ":=",
	itemEOF:          "EOF",
	itemField:        "field",
	itemIdentifier:   "identifier",
	itemLeftDelim:    "left delim",
	itemLeftParen:    "(",
	itemNumber:       "number",
	itemPipe:         "pipe",
	itemRawString:    "raw string",
	itemRightDelim:   "right delim",
	itemRightParen:   ")",
	itemSpace:        "space",
	itemString:       "string",
	itemVariable:     "variable",

	// keywords
	itemDot:      ".",
	itemDefine:   "define",
	itemElse:     "else",
	itemIf:       "if",
	itemEnd:      "end",
	itemNil:      "nil",
	itemRange:    "range",
	itemTemplate: "template",
	itemWith:     "with",
}

func (i itemType) String() string {
	s := itemName[i]
	if s == "" {
		return fmt.Sprintf("item%d", int(i))
	}
	return s
}

type lexTest struct {
	name  string
	input string
	items []item
}

var (
	tEOF      = item{itemEOF, 0, ""}
	tFor      = item{itemIdentifier, 0, "for"}
	tLeft     = item{itemLeftDelim, 0, "{{"}
	tLpar     = item{itemLeftParen, 0, "("}
	tPipe     = item{itemPipe, 0, "|"}
	tQuote    = item{itemString, 0, `"abc \n\t\" "`}
	tRange    = item{itemRange, 0, "range"}
	tRight    = item{itemRightDelim, 0, "}}"}
	tRpar     = item{itemRightParen, 0, ")"}
	tSpace    = item{itemSpace, 0, " "}
	raw       = "`" + `abc\n\t\" ` + "`"
	tRawQuote = item{itemRawString, 0, raw}
)

var lexTests = []lexTest{
	{"empty", "", []item{tEOF}},
	{"spaces", " \t\n", []item{{itemText, 0, " \t\n"}, tEOF}},
	{"text", `now is the time`, []item{{itemText, 0, "now is the time"}, tEOF}},
	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
		{itemText, 0, "hello-"},
		{itemText, 0, "-world"},
		tEOF,
	}},
	{"punctuation", "{{,@% }}", []item{
		tLeft,
		{itemChar, 0, ","},
		{itemChar, 0, "@"},
		{itemChar, 0, "%"},
		tSpace,
		tRight,
		tEOF,
	}},
	{"parens", "{{((3))}}", []item{
		tLeft,
		tLpar,
		tLpar,
		{itemNumber, 0, "3"},
		tRpar,
		tRpar,
		tRight,
		tEOF,
	}},
	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
	{"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{
		tLeft,
		{itemNumber, 0, "1"},
		tSpace,
		{itemNumber, 0, "02"},
		tSpace,
		{itemNumber, 0, "0x14"},
		tSpace,
		{itemNumber, 0, "-7.2i"},
		tSpace,
		{itemNumber, 0, "1e3"},
		tSpace,
		{itemNumber, 0, "+1.2e-4"},
		tSpace,
		{itemNumber, 0, "4.2i"},
		tSpace,
		{itemComplex, 0, "1+2i"},
		tRight,
		tEOF,
	}},
	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
		tLeft,
		{itemCharConstant, 0, `'a'`},
		tSpace,
		{itemCharConstant, 0, `'\n'`},
		tSpace,
		{itemCharConstant, 0, `'\''`},
		tSpace,
		{itemCharConstant, 0, `'\\'`},
		tSpace,
		{itemCharConstant, 0, `'\u00FF'`},
		tSpace,
		{itemCharConstant, 0, `'\xFF'`},
		tSpace,
		{itemCharConstant, 0, `'本'`},
		tRight,
		tEOF,
	}},
	{"bools", "{{true false}}", []item{
		tLeft,
		{itemBool, 0, "true"},
		tSpace,
		{itemBool, 0, "false"},
		tRight,
		tEOF,
	}},
	{"dot", "{{.}}", []item{
		tLeft,
		{itemDot, 0, "."},
		tRight,
		tEOF,
	}},
	{"nil", "{{nil}}", []item{
		tLeft,
		{itemNil, 0, "nil"},
		tRight,
		tEOF,
	}},
	{"dots", "{{.x . .2 .x.y.z}}", []item{
		tLeft,
		{itemField, 0, ".x"},
		tSpace,
		{itemDot, 0, "."},
		tSpace,
		{itemNumber, 0, ".2"},
		tSpace,
		{itemField, 0, ".x"},
		{itemField, 0, ".y"},
		{itemField, 0, ".z"},
		tRight,
		tEOF,
	}},
	{"keywords", "{{range if else end with}}", []item{
		tLeft,
		{itemRange, 0, "range"},
		tSpace,
		{itemIf, 0, "if"},
		tSpace,
		{itemElse, 0, "else"},
		tSpace,
		{itemEnd, 0, "end"},
		tSpace,
		{itemWith, 0, "with"},
		tRight,
		tEOF,
	}},
	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
		tLeft,
		{itemVariable, 0, "$c"},
		tSpace,
		{itemColonEquals, 0, ":="},
		tSpace,
		{itemIdentifier, 0, "printf"},
		tSpace,
		{itemVariable, 0, "$"},
		tSpace,
		{itemVariable, 0, "$hello"},
		tSpace,
		{itemVariable, 0, "$23"},
		tSpace,
		{itemVariable, 0, "$"},
		tSpace,
		{itemVariable, 0, "$var"},
		{itemField, 0, ".Field"},
		tSpace,
		{itemField, 0, ".Method"},
		tRight,
		tEOF,
	}},
	{"variable invocation", "{{$x 23}}", []item{
		tLeft,
		{itemVariable, 0, "$x"},
		tSpace,
		{itemNumber, 0, "23"},
		tRight,
		tEOF,
	}},
	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
		{itemText, 0, "intro "},
		tLeft,
		{itemIdentifier, 0, "echo"},
		tSpace,
		{itemIdentifier, 0, "hi"},
		tSpace,
		{itemNumber, 0, "1.2"},
		tSpace,
		tPipe,
		{itemIdentifier, 0, "noargs"},
		tPipe,
		{itemIdentifier, 0, "args"},
		tSpace,
		{itemNumber, 0, "1"},
		tSpace,
		{itemString, 0, `"hi"`},
		tRight,
		{itemText, 0, " outro"},
		tEOF,
	}},
	{"declaration", "{{$v := 3}}", []item{
		tLeft,
		{itemVariable, 0, "$v"},
		tSpace,
		{itemColonEquals, 0, ":="},
		tSpace,
		{itemNumber, 0, "3"},
		tRight,
		tEOF,
	}},
	{"2 declarations", "{{$v , $w := 3}}", []item{
		tLeft,
		{itemVariable, 0, "$v"},
		tSpace,
		{itemChar, 0, ","},
		tSpace,
		{itemVariable, 0, "$w"},
		tSpace,
		{itemColonEquals, 0, ":="},
		tSpace,
		{itemNumber, 0, "3"},
		tRight,
		tEOF,
	}},
	{"field of parenthesized expression", "{{(.X).Y}}", []item{
		tLeft,
		tLpar,
		{itemField, 0, ".X"},
		tRpar,
		{itemField, 0, ".Y"},
		tRight,
		tEOF,
	}},
	// errors
	{"badchar", "#{{\x01}}", []item{
		{itemText, 0, "#"},
		tLeft,
		{itemError, 0, "unrecognized character in action: U+0001"},
	}},
	{"unclosed action", "{{\n}}", []item{
		tLeft,
		{itemError, 0, "unclosed action"},
	}},
	{"EOF in action", "{{range", []item{
		tLeft,
		tRange,
		{itemError, 0, "unclosed action"},
	}},
	{"unclosed quote", "{{\"\n\"}}", []item{
		tLeft,
		{itemError, 0, "unterminated quoted string"},
	}},
	{"unclosed raw quote", "{{`xx\n`}}", []item{
		tLeft,
		{itemError, 0, "unterminated raw quoted string"},
	}},
	{"unclosed char constant", "{{'\n}}", []item{
		tLeft,
		{itemError, 0, "unterminated character constant"},
	}},
	{"bad number", "{{3k}}", []item{
		tLeft,
		{itemError, 0, `bad number syntax: "3k"`},
	}},
	{"unclosed paren", "{{(3}}", []item{
		tLeft,
		tLpar,
		{itemNumber, 0, "3"},
		{itemError, 0, `unclosed left paren`},
	}},
	{"extra right paren", "{{3)}}", []item{
		tLeft,
		{itemNumber, 0, "3"},
		tRpar,
		{itemError, 0, `unexpected right paren U+0029 ')'`},
	}},

	// Fixed bugs
	// Many elements in an action blew the lookahead until
	// we made lexInsideAction not loop.
	{"long pipeline deadlock", "{{|||||}}", []item{
		tLeft,
		tPipe,
		tPipe,
		tPipe,
		tPipe,
		tPipe,
		tRight,
		tEOF,
	}},
	{"text with bad comment", "hello-{{/*/}}-world", []item{
		{itemText, 0, "hello-"},
		{itemError, 0, `unclosed comment`},
	}},
}

// collect gathers the emitted items into a slice.
func collect(t *lexTest, left, right string) (items []item) {
	l := lex(t.name, t.input, left, right)
	for {
		item := l.nextItem()
		items = append(items, item)
		if item.typ == itemEOF || item.typ == itemError {
			break
		}
	}
	return
}

func equal(i1, i2 []item, checkPos bool) bool {
	if len(i1) != len(i2) {
		return false
	}
	for k := range i1 {
		if i1[k].typ != i2[k].typ {
			return false
		}
		if i1[k].val != i2[k].val {
			return false
		}
		if checkPos && i1[k].pos != i2[k].pos {
			return false
		}
	}
	return true
}

func TestLex(t *testing.T) {
	for _, test := range lexTests {
		items := collect(&test, "", "")
		if !equal(items, test.items, false) {
			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
		}
	}
}

// Some easy cases from above, but with delimiters $$ and @@
var lexDelimTests = []lexTest{
	{"punctuation", "$$,@%{{}}@@", []item{
		tLeftDelim,
		{itemChar, 0, ","},
		{itemChar, 0, "@"},
		{itemChar, 0, "%"},
		{itemChar, 0, "{"},
		{itemChar, 0, "{"},
		{itemChar, 0, "}"},
		{itemChar, 0, "}"},
		tRightDelim,
		tEOF,
	}},
	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
}

var (
	tLeftDelim  = item{itemLeftDelim, 0, "$$"}
	tRightDelim = item{itemRightDelim, 0, "@@"}
)

func TestDelims(t *testing.T) {
	for _, test := range lexDelimTests {
		items := collect(&test, "$$", "@@")
		if !equal(items, test.items, false) {
			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
		}
	}
}

var lexPosTests = []lexTest{
	{"empty", "", []item{tEOF}},
	{"punctuation", "{{,@%#}}", []item{
		{itemLeftDelim, 0, "{{"},
		{itemChar, 2, ","},
		{itemChar, 3, "@"},
		{itemChar, 4, "%"},
		{itemChar, 5, "#"},
		{itemRightDelim, 6, "}}"},
		{itemEOF, 8, ""},
	}},
	{"sample", "0123{{hello}}xyz", []item{
		{itemText, 0, "0123"},
		{itemLeftDelim, 4, "{{"},
		{itemIdentifier, 6, "hello"},
		{itemRightDelim, 11, "}}"},
		{itemText, 13, "xyz"},
		{itemEOF, 16, ""},
	}},
}

// The other tests don't check position, to make the test cases easier to construct.
// This one does.
func TestPos(t *testing.T) {
	for _, test := range lexPosTests {
		items := collect(&test, "", "")
		if !equal(items, test.items, true) {
			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
			if len(items) == len(test.items) {
				// Detailed print; avoid item.String() to expose the position value.
				for i := range items {
					if !equal(items[i:i+1], test.items[i:i+1], true) {
						i1 := items[i]
						i2 := test.items[i]
						t.Errorf("\t#%d: got {%v %d %q} expected  {%v %d %q}", i, i1.typ, i1.pos, i1.val, i2.typ, i2.pos, i2.val)
					}
				}
			}
		}
	}
}
Commit	Line	Data
adb0401d ILT	1	// Copyright 2011 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package parse
	6
	7	import (
4ccad563	8	"fmt"
adb0401d ILT	9	"testing"
	10	)
	11
4ccad563 ILT	12	// Make the types prettyprint.
	13	var itemName = map[itemType]string{
	14	itemError: "error",
	15	itemBool: "bool",
	16	itemChar: "char",
	17	itemCharConstant: "charconst",
	18	itemComplex: "complex",
	19	itemColonEquals: ":=",
	20	itemEOF: "EOF",
	21	itemField: "field",
	22	itemIdentifier: "identifier",
	23	itemLeftDelim: "left delim",
	24	itemLeftParen: "(",
	25	itemNumber: "number",
	26	itemPipe: "pipe",
	27	itemRawString: "raw string",
	28	itemRightDelim: "right delim",
	29	itemRightParen: ")",
	30	itemSpace: "space",
	31	itemString: "string",
	32	itemVariable: "variable",
	33
	34	// keywords
	35	itemDot: ".",
	36	itemDefine: "define",
	37	itemElse: "else",
	38	itemIf: "if",
	39	itemEnd: "end",
	40	itemNil: "nil",
	41	itemRange: "range",
	42	itemTemplate: "template",
	43	itemWith: "with",
	44	}
	45
	46	func (i itemType) String() string {
	47	s := itemName[i]
	48	if s == "" {
	49	return fmt.Sprintf("item%d", int(i))
	50	}
	51	return s
	52	}
	53
adb0401d ILT	54	type lexTest struct {
	55	name string
	56	input string
	57	items []item
	58	}
	59
	60	var (
4ccad563 ILT	61	tEOF = item{itemEOF, 0, ""}
	62	tFor = item{itemIdentifier, 0, "for"}
	63	tLeft = item{itemLeftDelim, 0, "{{"}
	64	tLpar = item{itemLeftParen, 0, "("}
	65	tPipe = item{itemPipe, 0, "\|"}
	66	tQuote = item{itemString, 0, `"abc \n\t\" "`}
	67	tRange = item{itemRange, 0, "range"}
	68	tRight = item{itemRightDelim, 0, "}}"}
	69	tRpar = item{itemRightParen, 0, ")"}
	70	tSpace = item{itemSpace, 0, " "}
adb0401d	71	raw = "`" + `abc\n\t\" ` + "`"
4ccad563	72	tRawQuote = item{itemRawString, 0, raw}
adb0401d ILT	73	)
	74
	75	var lexTests = []lexTest{
	76	{"empty", "", []item{tEOF}},
4ccad563 ILT	77	{"spaces", " \t\n", []item{{itemText, 0, " \t\n"}, tEOF}},
4ccad563 ILT	78	{"text", `now is the time`, []item{{itemText, 0, "now is the time"}, tEOF}},
adb0401d	79	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
4ccad563 ILT	80	{itemText, 0, "hello-"},
4ccad563 ILT	81	{itemText, 0, "-world"},
adb0401d ILT	82	tEOF,
adb0401d ILT	83	}},
4ccad563	84	{"punctuation", "{{,@% }}", []item{
adb0401d	85	tLeft,
4ccad563 ILT	86	{itemChar, 0, ","},
	87	{itemChar, 0, "@"},
	88	{itemChar, 0, "%"},
	89	tSpace,
	90	tRight,
	91	tEOF,
	92	}},
	93	{"parens", "{{((3))}}", []item{
	94	tLeft,
	95	tLpar,
	96	tLpar,
	97	{itemNumber, 0, "3"},
	98	tRpar,
	99	tRpar,
adb0401d ILT	100	tRight,
	101	tEOF,
	102	}},
	103	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
4ccad563	104	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
adb0401d ILT	105	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
	106	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
	107	{"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{
	108	tLeft,
4ccad563 ILT	109	{itemNumber, 0, "1"},
	110	tSpace,
	111	{itemNumber, 0, "02"},
	112	tSpace,
	113	{itemNumber, 0, "0x14"},
	114	tSpace,
	115	{itemNumber, 0, "-7.2i"},
	116	tSpace,
	117	{itemNumber, 0, "1e3"},
	118	tSpace,
	119	{itemNumber, 0, "+1.2e-4"},
	120	tSpace,
	121	{itemNumber, 0, "4.2i"},
	122	tSpace,
	123	{itemComplex, 0, "1+2i"},
adb0401d ILT	124	tRight,
	125	tEOF,
	126	}},
	127	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
	128	tLeft,
4ccad563 ILT	129	{itemCharConstant, 0, `'a'`},
	130	tSpace,
	131	{itemCharConstant, 0, `'\n'`},
	132	tSpace,
	133	{itemCharConstant, 0, `'\''`},
	134	tSpace,
	135	{itemCharConstant, 0, `'\\'`},
	136	tSpace,
	137	{itemCharConstant, 0, `'\u00FF'`},
	138	tSpace,
	139	{itemCharConstant, 0, `'\xFF'`},
	140	tSpace,
	141	{itemCharConstant, 0, `'本'`},
adb0401d ILT	142	tRight,
	143	tEOF,
	144	}},
	145	{"bools", "{{true false}}", []item{
	146	tLeft,
4ccad563 ILT	147	{itemBool, 0, "true"},
	148	tSpace,
	149	{itemBool, 0, "false"},
adb0401d ILT	150	tRight,
	151	tEOF,
	152	}},
	153	{"dot", "{{.}}", []item{
	154	tLeft,
4ccad563	155	{itemDot, 0, "."},
adb0401d ILT	156	tRight,
	157	tEOF,
	158	}},
4ccad563	159	{"nil", "{{nil}}", []item{
adb0401d	160	tLeft,
4ccad563 ILT	161	{itemNil, 0, "nil"},
	162	tRight,
	163	tEOF,
	164	}},
	165	{"dots", "{{.x . .2 .x.y.z}}", []item{
	166	tLeft,
	167	{itemField, 0, ".x"},
	168	tSpace,
	169	{itemDot, 0, "."},
	170	tSpace,
	171	{itemNumber, 0, ".2"},
	172	tSpace,
	173	{itemField, 0, ".x"},
	174	{itemField, 0, ".y"},
	175	{itemField, 0, ".z"},
adb0401d ILT	176	tRight,
	177	tEOF,
	178	}},
	179	{"keywords", "{{range if else end with}}", []item{
	180	tLeft,
4ccad563 ILT	181	{itemRange, 0, "range"},
	182	tSpace,
	183	{itemIf, 0, "if"},
	184	tSpace,
	185	{itemElse, 0, "else"},
	186	tSpace,
	187	{itemEnd, 0, "end"},
	188	tSpace,
	189	{itemWith, 0, "with"},
adb0401d ILT	190	tRight,
	191	tEOF,
	192	}},
	193	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
	194	tLeft,
4ccad563 ILT	195	{itemVariable, 0, "$c"},
	196	tSpace,
	197	{itemColonEquals, 0, ":="},
	198	tSpace,
	199	{itemIdentifier, 0, "printf"},
	200	tSpace,
	201	{itemVariable, 0, "$"},
	202	tSpace,
	203	{itemVariable, 0, "$hello"},
	204	tSpace,
	205	{itemVariable, 0, "$23"},
	206	tSpace,
	207	{itemVariable, 0, "$"},
	208	tSpace,
	209	{itemVariable, 0, "$var"},
	210	{itemField, 0, ".Field"},
	211	tSpace,
	212	{itemField, 0, ".Method"},
	213	tRight,
	214	tEOF,
	215	}},
	216	{"variable invocation", "{{$x 23}}", []item{
	217	tLeft,
	218	{itemVariable, 0, "$x"},
	219	tSpace,
	220	{itemNumber, 0, "23"},
adb0401d ILT	221	tRight,
	222	tEOF,
	223	}},
	224	{"pipeline", `intro {{echo hi 1.2 \|noargs\|args 1 "hi"}} outro`, []item{
4ccad563	225	{itemText, 0, "intro "},
adb0401d	226	tLeft,
4ccad563 ILT	227	{itemIdentifier, 0, "echo"},
	228	tSpace,
	229	{itemIdentifier, 0, "hi"},
	230	tSpace,
	231	{itemNumber, 0, "1.2"},
	232	tSpace,
adb0401d	233	tPipe,
4ccad563	234	{itemIdentifier, 0, "noargs"},
adb0401d	235	tPipe,
4ccad563 ILT	236	{itemIdentifier, 0, "args"},
	237	tSpace,
	238	{itemNumber, 0, "1"},
	239	tSpace,
	240	{itemString, 0, `"hi"`},
adb0401d	241	tRight,
4ccad563	242	{itemText, 0, " outro"},
adb0401d ILT	243	tEOF,
	244	}},
	245	{"declaration", "{{$v := 3}}", []item{
	246	tLeft,
4ccad563 ILT	247	{itemVariable, 0, "$v"},
	248	tSpace,
	249	{itemColonEquals, 0, ":="},
	250	tSpace,
	251	{itemNumber, 0, "3"},
adb0401d ILT	252	tRight,
	253	tEOF,
	254	}},
	255	{"2 declarations", "{{$v , $w := 3}}", []item{
	256	tLeft,
4ccad563 ILT	257	{itemVariable, 0, "$v"},
	258	tSpace,
	259	{itemChar, 0, ","},
	260	tSpace,
	261	{itemVariable, 0, "$w"},
	262	tSpace,
	263	{itemColonEquals, 0, ":="},
	264	tSpace,
	265	{itemNumber, 0, "3"},
	266	tRight,
	267	tEOF,
	268	}},
	269	{"field of parenthesized expression", "{{(.X).Y}}", []item{
	270	tLeft,
	271	tLpar,
	272	{itemField, 0, ".X"},
	273	tRpar,
	274	{itemField, 0, ".Y"},
adb0401d ILT	275	tRight,
	276	tEOF,
	277	}},
	278	// errors
	279	{"badchar", "#{{\x01}}", []item{
4ccad563	280	{itemText, 0, "#"},
adb0401d	281	tLeft,
4ccad563	282	{itemError, 0, "unrecognized character in action: U+0001"},
adb0401d ILT	283	}},
	284	{"unclosed action", "{{\n}}", []item{
	285	tLeft,
4ccad563	286	{itemError, 0, "unclosed action"},
adb0401d ILT	287	}},
	288	{"EOF in action", "{{range", []item{
	289	tLeft,
	290	tRange,
4ccad563	291	{itemError, 0, "unclosed action"},
adb0401d ILT	292	}},
	293	{"unclosed quote", "{{\"\n\"}}", []item{
	294	tLeft,
4ccad563	295	{itemError, 0, "unterminated quoted string"},
adb0401d ILT	296	}},
	297	{"unclosed raw quote", "{{`xx\n`}}", []item{
	298	tLeft,
4ccad563	299	{itemError, 0, "unterminated raw quoted string"},
adb0401d ILT	300	}},
	301	{"unclosed char constant", "{{'\n}}", []item{
	302	tLeft,
4ccad563	303	{itemError, 0, "unterminated character constant"},
adb0401d ILT	304	}},
	305	{"bad number", "{{3k}}", []item{
	306	tLeft,
4ccad563 ILT	307	{itemError, 0, `bad number syntax: "3k"`},
	308	}},
	309	{"unclosed paren", "{{(3}}", []item{
	310	tLeft,
	311	tLpar,
	312	{itemNumber, 0, "3"},
	313	{itemError, 0, `unclosed left paren`},
	314	}},
	315	{"extra right paren", "{{3)}}", []item{
	316	tLeft,
	317	{itemNumber, 0, "3"},
	318	tRpar,
	319	{itemError, 0, `unexpected right paren U+0029 ')'`},
adb0401d ILT	320	}},
	321
	322	// Fixed bugs
	323	// Many elements in an action blew the lookahead until
	324	// we made lexInsideAction not loop.
	325	{"long pipeline deadlock", "{{\|\|\|\|\|}}", []item{
	326	tLeft,
	327	tPipe,
	328	tPipe,
	329	tPipe,
	330	tPipe,
	331	tPipe,
	332	tRight,
	333	tEOF,
	334	}},
bd2e46c8	335	{"text with bad comment", "hello-{{/*/}}-world", []item{
4ccad563 ILT	336	{itemText, 0, "hello-"},
4ccad563 ILT	337	{itemError, 0, `unclosed comment`},
bd2e46c8	338	}},
adb0401d ILT	339	}
	340
	341	// collect gathers the emitted items into a slice.
d8f41257 ILT	342	func collect(t *lexTest, left, right string) (items []item) {
d8f41257 ILT	343	l := lex(t.name, t.input, left, right)
adb0401d ILT	344	for {
	345	item := l.nextItem()
	346	items = append(items, item)
	347	if item.typ == itemEOF \|\| item.typ == itemError {
	348	break
	349	}
	350	}
	351	return
	352	}
	353
4ccad563 ILT	354	func equal(i1, i2 []item, checkPos bool) bool {
	355	if len(i1) != len(i2) {
	356	return false
	357	}
	358	for k := range i1 {
	359	if i1[k].typ != i2[k].typ {
	360	return false
	361	}
	362	if i1[k].val != i2[k].val {
	363	return false
	364	}
	365	if checkPos && i1[k].pos != i2[k].pos {
	366	return false
	367	}
	368	}
	369	return true
	370	}
	371
adb0401d ILT	372	func TestLex(t *testing.T) {
adb0401d ILT	373	for _, test := range lexTests {
d8f41257	374	items := collect(&test, "", "")
4ccad563 ILT	375	if !equal(items, test.items, false) {
4ccad563 ILT	376	t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
d8f41257 ILT	377	}
	378	}
	379	}
	380
	381	// Some easy cases from above, but with delimiters $$ and @@
	382	var lexDelimTests = []lexTest{
	383	{"punctuation", "$$,@%{{}}@@", []item{
	384	tLeftDelim,
4ccad563 ILT	385	{itemChar, 0, ","},
	386	{itemChar, 0, "@"},
	387	{itemChar, 0, "%"},
	388	{itemChar, 0, "{"},
	389	{itemChar, 0, "{"},
	390	{itemChar, 0, "}"},
	391	{itemChar, 0, "}"},
d8f41257 ILT	392	tRightDelim,
	393	tEOF,
	394	}},
	395	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
4ccad563	396	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
d8f41257 ILT	397	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
	398	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
	399	}
	400
	401	var (
4ccad563 ILT	402	tLeftDelim = item{itemLeftDelim, 0, "$$"}
4ccad563 ILT	403	tRightDelim = item{itemRightDelim, 0, "@@"}
d8f41257 ILT	404	)
	405
	406	func TestDelims(t *testing.T) {
	407	for _, test := range lexDelimTests {
	408	items := collect(&test, "$$", "@@")
4ccad563 ILT	409	if !equal(items, test.items, false) {
	410	t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
	411	}
	412	}
	413	}
	414
	415	var lexPosTests = []lexTest{
	416	{"empty", "", []item{tEOF}},
	417	{"punctuation", "{{,@%#}}", []item{
	418	{itemLeftDelim, 0, "{{"},
	419	{itemChar, 2, ","},
	420	{itemChar, 3, "@"},
	421	{itemChar, 4, "%"},
	422	{itemChar, 5, "#"},
	423	{itemRightDelim, 6, "}}"},
	424	{itemEOF, 8, ""},
	425	}},
	426	{"sample", "0123{{hello}}xyz", []item{
	427	{itemText, 0, "0123"},
	428	{itemLeftDelim, 4, "{{"},
	429	{itemIdentifier, 6, "hello"},
	430	{itemRightDelim, 11, "}}"},
	431	{itemText, 13, "xyz"},
	432	{itemEOF, 16, ""},
	433	}},
	434	}
	435
	436	// The other tests don't check position, to make the test cases easier to construct.
	437	// This one does.
	438	func TestPos(t *testing.T) {
	439	for _, test := range lexPosTests {
	440	items := collect(&test, "", "")
	441	if !equal(items, test.items, true) {
adb0401d	442	t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
4ccad563 ILT	443	if len(items) == len(test.items) {
	444	// Detailed print; avoid item.String() to expose the position value.
	445	for i := range items {
	446	if !equal(items[i:i+1], test.items[i:i+1], true) {
	447	i1 := items[i]
	448	i2 := test.items[i]
	449	t.Errorf("\t#%d: got {%v %d %q} expected {%v %d %q}", i, i1.typ, i1.pos, i1.val, i2.typ, i2.pos, i2.val)
	450	}
	451	}
	452	}
adb0401d ILT	453	}
	454	}
	455	}