golang template token patch (go version devel +a2f5d644d3 Sat Aug 3 08:47:32 2019 +0000 linux/amd64)

example.go

package main

import (
	"fmt"
)

contract stringer@<type T> {
	T String() string
}

type Pair@<type K,V> struct {
	k K
	v V
}

func (p *Pair<K,V>) First() K {
	return p.k
}

func Print@<T>(t T) {
	fmt.Sprintf("%v", t)
}

func main() {
	fmt.Println("Hello World")
	
	var p1 Pair@<string, string>
	
	// closing nested templates with > > or >> is fine
	var p2 Pair@<string, Pair@<int,int>>
	var p3 Pair@<string, Pair@<int,int> >

	var p4 Pair@<string, chan<- string>
	var p5 Pair@<string, <-chan string>

	var p6 Pair@<chan<- string, string>
	var p7 Pair@<<-chan string, string>
}

parse.go

package main

import (
	"fmt"
	"go/scanner"
	"go/token"
	"io/ioutil"
	"os"
)

func main() {

	// src that we want to tokenize
	src, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}

	var s scanner.Scanner
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, src, func(pos token.Position, msg string) {
		fmt.Printf("Error: %v: %v\n", pos, msg)
	}, scanner.ScanComments)

	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		fmt.Printf("%s\t%s\t%d\t%q\n", fset.Position(pos), tok, tok, lit)
	}
}

patch.diff

diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go
index fbb3e1a40e..b236ba7cf9 100644
--- a/src/cmd/compile/internal/syntax/scanner.go
+++ b/src/cmd/compile/internal/syntax/scanner.go
@@ -39,6 +39,7 @@ type scanner struct {
 	kind      LitKind  // valid if tok is _Literal
 	op        Operator // valid if tok is _Operator, _AssignOp, or _IncOp
 	prec      int      // valid if tok is _Operator, _AssignOp, or _IncOp
+	template  int      // set to template depth; > 0 if in template type scope
 }
 
 func (s *scanner) init(src io.Reader, errh func(line, col uint, msg string), mode uint) {
@@ -252,6 +253,17 @@ redo:
 		c = s.getr()
 		goto assignop
 
+	case '@':
+		c = s.getr()
+		if c == '<' {
+			s.tok = _Ltempl
+			s.template++
+			break
+		}
+		s.tok = 0
+		s.errorf("invalid character %#U", c)
+		goto redo
+
 	case '<':
 		c = s.getr()
 		if c == '=' {
@@ -273,6 +285,11 @@ redo:
 		s.tok = _Operator
 
 	case '>':
+		if s.template > 0 {
+			s.tok = _Rtempl
+			s.template--
+			break
+		}
 		c = s.getr()
 		if c == '=' {
 			s.op, s.prec = Geq, precCmp
diff --git a/src/cmd/compile/internal/syntax/token_string.go b/src/cmd/compile/internal/syntax/token_string.go
index 3cf5473feb..ec3e487c25 100644
--- a/src/cmd/compile/internal/syntax/token_string.go
+++ b/src/cmd/compile/internal/syntax/token_string.go
@@ -4,9 +4,9 @@ package syntax
 
 import "strconv"
 
-const _token_name = "EOFnameliteralopop=opop=:=<-*([{)]},;:....breakcasechanconstcontinuedefaultdeferelsefallthroughforfuncgogotoifimportinterfacemappackagerangereturnselectstructswitchtypevar"
+const _token_name = "EOFnameliteralopop=opop=:=<-*([{@<)]}>,;:....breakcasechanconstcontinuedefaultdeferelsefallthroughforfuncgogotoifimportinterfacemappackagerangereturnselectstructswitchtypevar"
 
-var _token_index = [...]uint8{0, 3, 7, 14, 16, 19, 23, 24, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 42, 47, 51, 55, 60, 68, 75, 80, 84, 95, 98, 102, 104, 108, 110, 116, 125, 128, 135, 140, 146, 152, 158, 164, 168, 171, 171}
+var _token_index = [...]uint8{0, 3, 7, 14, 16, 19, 23, 24, 26, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 45, 50, 54, 58, 63, 71, 78, 83, 87, 98, 101, 105, 107, 111, 113, 119, 128, 131, 138, 143, 149, 155, 161, 167, 171, 174, 174}
 
 func (i token) String() string {
 	i -= 1
diff --git a/src/cmd/compile/internal/syntax/tokens.go b/src/cmd/compile/internal/syntax/tokens.go
index 9b26c9f12f..98011db2da 100644
--- a/src/cmd/compile/internal/syntax/tokens.go
+++ b/src/cmd/compile/internal/syntax/tokens.go
@@ -30,9 +30,11 @@ const (
 	_Lparen    // (
 	_Lbrack    // [
 	_Lbrace    // {
+	_Ltempl    // @<
 	_Rparen    // )
 	_Rbrack    // ]
 	_Rbrace    // }
+	_Rtempl    // >
 	_Comma     // ,
 	_Semi      // ;
 	_Colon     // :
diff --git a/src/go/scanner/scanner.go b/src/go/scanner/scanner.go
index 00fe2dc0b1..9aa65359b3 100644
--- a/src/go/scanner/scanner.go
+++ b/src/go/scanner/scanner.go
@@ -43,6 +43,7 @@ type Scanner struct {
 	rdOffset   int  // reading offset (position after current character)
 	lineOffset int  // current line offset
 	insertSemi bool // insert a semicolon before next newline
+	template   int  // set to template depth; > 0 if in template type scope
 
 	// public state - ok to modify
 	ErrorCount int // number of errors encountered
@@ -900,6 +901,14 @@ scanAgain:
 			tok = s.switch2(token.REM, token.REM_ASSIGN)
 		case '^':
 			tok = s.switch2(token.XOR, token.XOR_ASSIGN)
+		case '@':
+			if s.ch == '<' {
+				s.next()
+				s.template++
+				tok = token.LTEMPL
+			} else {
+				tok = token.ILLEGAL
+			}
 		case '<':
 			if s.ch == '-' {
 				s.next()
@@ -908,7 +917,12 @@ scanAgain:
 				tok = s.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
 			}
 		case '>':
-			tok = s.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+			if s.template > 0 {
+				tok = token.RTEMPL
+				s.template--
+			} else {
+				tok = s.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+			}
 		case '=':
 			tok = s.switch2(token.ASSIGN, token.EQL)
 		case '!':
diff --git a/src/go/scanner/scanner_test.go b/src/go/scanner/scanner_test.go
index 9d3bbbbb24..8658020e1b 100644
--- a/src/go/scanner/scanner_test.go
+++ b/src/go/scanner/scanner_test.go
@@ -669,14 +669,14 @@ func TestInit(t *testing.T) {
 }
 
 func TestStdErrorHander(t *testing.T) {
-	const src = "@\n" + // illegal character, cause an error
-		"@ @\n" + // two errors on the same line
+	const src = "#\n" + // illegal character, cause an error
+		"# #\n" + // two errors on the same line
 		"//line File2:20\n" +
-		"@\n" + // different file, but same line
+		"#\n" + // different file, but same line
 		"//line File2:1\n" +
-		"@ @\n" + // same file, decreasing line number
+		"# #\n" + // same file, decreasing line number
 		"//line File1:1\n" +
-		"@ @ @" // original file, line 1 again
+		"# # #" // original file, line 1 again
 
 	var list ErrorList
 	eh := func(pos token.Position, msg string) { list.Add(pos, msg) }
diff --git a/src/go/token/token.go b/src/go/token/token.go
index 96a1079ec3..439f013912 100644
--- a/src/go/token/token.go
+++ b/src/go/token/token.go
@@ -83,12 +83,14 @@ const (
 	LPAREN // (
 	LBRACK // [
 	LBRACE // {
+	// LTEMPL // @<
 	COMMA  // ,
 	PERIOD // .
 
-	RPAREN    // )
-	RBRACK    // ]
-	RBRACE    // }
+	RPAREN // )
+	RBRACK // ]
+	RBRACE // }
+	// RTEMPL    // > when in LTEMPL scope
 	SEMICOLON // ;
 	COLON     // :
 	operator_end
@@ -125,6 +127,13 @@ const (
 	TYPE
 	VAR
 	keyword_end
+
+	// api/go1.1.txt fixed the constant values for all tokens above
+	// need to append the new tokens
+	operator_ext_beg
+	LTEMPL // @<
+	RTEMPL // > in LTEMPL scope
+	operator_ext_end
 )
 
 var tokens = [...]string{
@@ -187,12 +196,14 @@ var tokens = [...]string{
 	LPAREN: "(",
 	LBRACK: "[",
 	LBRACE: "{",
+	// LTEMPL: "@<",
 	COMMA:  ",",
 	PERIOD: ".",
 
-	RPAREN:    ")",
-	RBRACK:    "]",
-	RBRACE:    "}",
+	RPAREN: ")",
+	RBRACK: "]",
+	RBRACE: "}",
+	// RTEMPL:    ">",
 	SEMICOLON: ";",
 	COLON:     ":",
 
@@ -225,6 +236,9 @@ var tokens = [...]string{
 	SWITCH: "switch",
 	TYPE:   "type",
 	VAR:    "var",
+
+	LTEMPL: "@<",
+	RTEMPL: ">",
 }
 
 // String returns the string corresponding to the token tok.
@@ -304,7 +318,9 @@ func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_en
 // IsOperator returns true for tokens corresponding to operators and
 // delimiters; it returns false otherwise.
 //
-func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
+func (tok Token) IsOperator() bool {
+	return operator_beg < tok && tok < operator_end || operator_ext_beg < tok && tok < operator_ext_end
+}
 
 // IsKeyword returns true for tokens corresponding to keywords;
 // it returns false otherwise.

tokens

1:1	package	78	"package"
1:9	IDENT	4	"main"
1:13	;	57	"\n"
3:1	import	75	"import"
3:8	(	49	""
4:2	STRING	9	"\"fmt\""
4:7	;	57	"\n"
5:1	)	54	""
5:2	;	57	"\n"
7:1	IDENT	4	"contract"
7:10	IDENT	4	"stringer"
7:18	@<	88	""
7:20	type	84	"type"
7:25	IDENT	4	"T"
7:26	>	89	""
7:28	{	51	""
8:2	IDENT	4	"T"
8:4	IDENT	4	"String"
8:10	(	49	""
8:11	)	54	""
8:13	IDENT	4	"string"
8:19	;	57	"\n"
9:1	}	56	""
9:2	;	57	"\n"
11:1	type	84	"type"
11:6	IDENT	4	"Pair"
11:10	@<	88	""
11:12	type	84	"type"
11:17	IDENT	4	"K"
11:18	,	52	""
11:19	IDENT	4	"V"
11:20	>	89	""
11:22	struct	82	"struct"
11:29	{	51	""
12:2	IDENT	4	"k"
12:4	IDENT	4	"K"
12:5	;	57	"\n"
13:2	IDENT	4	"v"
13:4	IDENT	4	"V"
13:5	;	57	"\n"
14:1	}	56	""
14:2	;	57	"\n"
16:1	func	71	"func"
16:6	(	49	""
16:7	IDENT	4	"p"
16:9	*	14	""
16:10	IDENT	4	"Pair"
16:14	<	40	""
16:15	IDENT	4	"K"
16:16	,	52	""
16:17	IDENT	4	"V"
16:18	>	41	""
16:19	)	54	""
16:21	IDENT	4	"First"
16:26	(	49	""
16:27	)	54	""
16:29	IDENT	4	"K"
16:31	{	51	""
17:2	return	80	"return"
17:9	IDENT	4	"p"
17:10	.	53	""
17:11	IDENT	4	"k"
17:12	;	57	"\n"
18:1	}	56	""
18:2	;	57	"\n"
20:1	func	71	"func"
20:6	IDENT	4	"Print"
20:11	@<	88	""
20:13	IDENT	4	"T"
20:14	>	89	""
20:15	(	49	""
20:16	IDENT	4	"t"
20:18	IDENT	4	"T"
20:19	)	54	""
20:21	{	51	""
21:2	IDENT	4	"fmt"
21:5	.	53	""
21:6	IDENT	4	"Sprintf"
21:13	(	49	""
21:14	STRING	9	"\"%v\""
21:18	,	52	""
21:20	IDENT	4	"t"
21:21	)	54	""
21:22	;	57	"\n"
22:1	}	56	""
22:2	;	57	"\n"
24:1	func	71	"func"
24:6	IDENT	4	"main"
24:10	(	49	""
24:11	)	54	""
24:13	{	51	""
25:2	IDENT	4	"fmt"
25:5	.	53	""
25:6	IDENT	4	"Println"
25:13	(	49	""
25:14	STRING	9	"\"Hello World\""
25:27	)	54	""
25:28	;	57	"\n"
27:2	var	85	"var"
27:6	IDENT	4	"p1"
27:9	IDENT	4	"Pair"
27:13	@<	88	""
27:15	IDENT	4	"string"
27:21	,	52	""
27:23	IDENT	4	"string"
27:29	>	89	""
29:2	COMMENT	2	"// closing nested templates with > > or >> is fine"
30:2	var	85	"var"
30:6	IDENT	4	"p2"
30:9	IDENT	4	"Pair"
30:13	@<	88	""
30:15	IDENT	4	"string"
30:21	,	52	""
30:23	IDENT	4	"Pair"
30:27	@<	88	""
30:29	IDENT	4	"int"
30:32	,	52	""
30:33	IDENT	4	"int"
30:36	>	89	""
30:37	>	89	""
31:2	var	85	"var"
31:6	IDENT	4	"p3"
31:9	IDENT	4	"Pair"
31:13	@<	88	""
31:15	IDENT	4	"string"
31:21	,	52	""
31:23	IDENT	4	"Pair"
31:27	@<	88	""
31:29	IDENT	4	"int"
31:32	,	52	""
31:33	IDENT	4	"int"
31:36	>	89	""
31:38	>	89	""
33:2	var	85	"var"
33:6	IDENT	4	"p4"
33:9	IDENT	4	"Pair"
33:13	@<	88	""
33:15	IDENT	4	"string"
33:21	,	52	""
33:23	chan	63	"chan"
33:27	<-	36	""
33:30	IDENT	4	"string"
33:36	>	89	""
34:2	var	85	"var"
34:6	IDENT	4	"p5"
34:9	IDENT	4	"Pair"
34:13	@<	88	""
34:15	IDENT	4	"string"
34:21	,	52	""
34:23	<-	36	""
34:25	chan	63	"chan"
34:30	IDENT	4	"string"
34:36	>	89	""
36:2	var	85	"var"
36:6	IDENT	4	"p6"
36:9	IDENT	4	"Pair"
36:13	@<	88	""
36:15	chan	63	"chan"
36:19	<-	36	""
36:22	IDENT	4	"string"
36:28	,	52	""
36:30	IDENT	4	"string"
36:36	>	89	""
37:2	var	85	"var"
37:6	IDENT	4	"p7"
37:9	IDENT	4	"Pair"
37:13	@<	88	""
37:15	<-	36	""
37:17	chan	63	"chan"
37:22	IDENT	4	"string"
37:28	,	52	""
37:30	IDENT	4	"string"
37:36	>	89	""
38:1	}	56	""
38:2	;	57	"\n"