Merge pull request #201 from juanjux/fix/public_tokens

Restore Tokens as a public func since its also used by drivers' tests

uast/uast.go

@@ -515,3 +515,21 @@ func (p Path) Parent() Path {
 	copy(dst, p)
 	return dst
 }
+
+
+func Tokens(n *Node) []string {
+	var tokens []string
+	iter := NewOrderPathIter(NewPath(n))
+	for {
+		p := iter.Next()
+		if p.IsEmpty() {
+			break
+		}
+
+		n := p.Node()
+		if n.Token != "" {
+			tokens = append(tokens, n.Token)
+		}
+	}
+	return tokens
+}

uast/uast_test.go

@@ -6,23 +6,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func tokens(n *Node) []string {
-	var tokens []string
-	iter := NewOrderPathIter(NewPath(n))
-	for {
-		p := iter.Next()
-		if p.IsEmpty() {
-			break
-		}
-
-		n := p.Node()
-		if n.Token != "" {
-			tokens = append(tokens, n.Token)
-		}
-	}
-	return tokens
-}
-
 func TestPrefixTokens(t *testing.T) {
 	require := require.New(t)
 
@@ -34,7 +17,7 @@ func TestPrefixTokens(t *testing.T) {
 				{InternalType: "left", Token: "tok_pre_left"},
 				{InternalType: "right", Token: "tok_pre_right"},
 			}}}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id3", "Prefix+", "tok_pre_left", "tok_pre_right"}
 	require.Equal(expected, result)
 }
@@ -60,7 +43,7 @@ func TestPrefixTokensSubtree(t *testing.T) {
 				{InternalType: "right", Token: "tok_pre_right"},
 			},
 			}}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id3", "Prefix+", "tok_pre_left", "subleft_1a", "subleft_1a_2a",
 		"subleft_1a_2b", "subleft_1b", "subleft_b_2a", "subleft_b_2b",
 		"tok_pre_right"}
@@ -78,7 +61,7 @@ func TestPrefixTokensPlain(t *testing.T) {
 			{InternalType: "left", Token: "tok_pre_left"},
 			{InternalType: "right", Token: "tok_pre_right"},
 		}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id3", "Prefix+", "tok_pre_left", "tok_pre_right"}
 	require.Equal(expected, result)
 }
@@ -92,7 +75,7 @@ func TestInfixTokens(t *testing.T) {
 				{InternalType: "left", Token: "tok_in_left"},
 				{InternalType: "right", Token: "tok_in_right"},
 			}}}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id1", "tok_in_left", "Infix+", "tok_in_right"}
 	require.Equal(expected, result)
 }
@@ -118,7 +101,7 @@ func TestInfixTokensSubtree(t *testing.T) {
 				{InternalType: "right", Token: "right"},
 			},
 			}}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id3", "subleft_1a_2a", "subleft_1a", "subleft_1a_2b", "left",
 		"subleft_1b_2a", "subleft_1b", "subleft_1b_2b", "op_infix", "right"}
 
@@ -134,7 +117,7 @@ func TestInfixTokensPlain(t *testing.T) {
 			{InternalType: "op_infix", Roles: []Role{Infix}, Token: "Infix+"},
 			{InternalType: "right", Token: "tok_in_right"},
 		}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id1", "tok_in_left", "Infix+", "tok_in_right"}
 	require.Equal(expected, result)
 }
@@ -148,7 +131,7 @@ func TestPostfixTokens(t *testing.T) {
 				{InternalType: "left", Token: "tok_post_left"},
 				{InternalType: "right", Token: "tok_post_right"},
 			}}}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id2", "tok_post_left", "tok_post_right", "Postfix+"}
 	require.Equal(expected, result)
 }
@@ -174,7 +157,7 @@ func TestPostfixTokensSubtree(t *testing.T) {
 				{InternalType: "right", Token: "right"},
 			},
 			}}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id2", "subleft_1a_2a", "subleft_1a_2b", "subleft_1a", "subleft_1b_2a",
 		"subleft_1b_2b", "subleft_1b", "left", "right", "op_postfix"}
 	require.Equal(expected, result)
@@ -189,7 +172,7 @@ func TestPostfixTokensPlain(t *testing.T) {
 			{InternalType: "right", Token: "tok_post_right"},
 			{InternalType: "op_postfix", Roles: []Role{Postfix}, Token: "Postfix+"},
 		}}
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id2", "tok_post_left", "tok_post_right", "Postfix+"}
 	require.Equal(expected, result)
 }
@@ -228,7 +211,7 @@ func TestOrderTokens(t *testing.T) {
 				}},
 			}}}}
 
-	result := tokens(n)
+	result := Tokens(n)
 	expected := []string{"id1", "tok_in_left", "Infix+", "subright1", "subright2", "tok_in_right",
 		"id2", "tok_post_left", "tok_post_right", "subright_pre1", "subright_pre2", "Postfix+",
 		"id3", "Prefix+", "tok_pre_left", "subright_in1", "tok_pre_right", "subright_in2"}