Refactor Dictionary parser

backend/go.mod

@@ -17,7 +17,6 @@ require (
 	github.com/testcontainers/testcontainers-go v0.31.0
 	github.com/vektah/gqlparser/v2 v2.5.16
 	golang.org/x/net v0.26.0
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f
 	gorm.io/driver/postgres v1.5.9
 	gorm.io/gorm v1.25.10
 )

backend/go.sum

@@ -254,8 +254,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g=
 google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw=
 google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=

backend/pkg/textdic/lexer.go

@@ -14,7 +14,7 @@ type lexer struct {
 }
 
 func newLexer(input string) *lexer {
-	return &lexer{input: strings.NewReader(input), lineNo: 0}
+	return &lexer{input: strings.NewReader(input), lineNo: 1}
 }
 
 func (l *lexer) Peek() rune {
@@ -42,23 +42,23 @@ func (l *lexer) isJapanese(r rune) bool {
 }
 
 func (l *lexer) Lex(lval *yySymType) int {
-	for {
-		r, err := l.skipWhiteSpace()
-		if err != nil {
-			return EOF
-		}
-
-		if l.isNewLine(r) {
-			l.lineNo++
-			return NEWLINE
-		}
+	r, err := l.skipWhiteSpace()
+	if err != nil {
+		// Done with parsing
+		return 0
+	}
 
-		if l.isEnglishAndWhitespace(r) {
-			return l.lexWord(lval)
-		} else if l.isJapanese(r) {
-			return l.lexDefinition(lval)
-		}
+	if l.isNewLine(r) {
+		l.lineNo++
+		return NEWLINE
+	}
+	if l.isEnglishAndWhitespace(r) {
+		return l.lexWord(lval)
+	}
+	if l.isJapanese(r) {
+		return l.lexDefinition(lval)
 	}
+	return EOF
 }
 
 func (l *lexer) lexWord(lval *yySymType) int {

backend/pkg/textdic/parser.y

@@ -27,7 +27,7 @@ type Nodes []Node
 
 %%
 start
-	: entries EOF { $$ = $1; yyrcvr.setNodes($1); }
+	: entries { $$ = $1; yyrcvr.setNodes($1); }
 	;
 
 entries
@@ -37,7 +37,7 @@ entries
 
 entry
 	: WORD DEFINITION NEWLINE { $$ = Node{Word: $1, Definition: $2} }
-	| NEWLINE { $$ = Node{} } // 空行を無視する
+	| NEWLINE { $$ = Node{} } // Ignore empty line
 	;
 
 %%

backend/pkg/textdic/parser_test.go

@@ -1,6 +1,7 @@
 package textdic
 
 import (
+	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
@@ -17,6 +18,8 @@ jarring 気に障る
 
 rube 田舎者
 
+  
+
 out of touch 情報に疎い、
 
 opaque 不透明な
@@ -83,21 +86,25 @@ There is no leeway to provide services free of charge for the sake of others. 
 		var input = `
 trot out 自慢げに話題に持ち出す
 #エラーになる
+!もう一つエラー
 `
-
 		// Create a new lexer with the input
 		l := newLexer(input)
-		yyDebug = 1
+		yyDebug = 5
 		yyErrorVerbose = true
 
 		// Parse the input using the parser instance
-		NewParser(l)
+		parser := NewParser(l)
 		errors := l.GetErrors()
 
 		if len(errors) == 0 {
 			t.Errorf("expected errors, but got none")
 		}
 
+		// It should get one correct data in the node
+		nodes := parser.GetNodes()
+		assert.Equal(t, 1, len(nodes))
+
 		for _, err := range errors {
 			if err.Error() == "" {
 				t.Errorf("expected error message, but got empty string")

backend/pkg/textdic/text_dictionary_service.go

@@ -1,10 +1,8 @@
 package textdic
 
 import (
-	"backend/pkg/logger"
 	"encoding/base64"
 	"fmt"
-	"golang.org/x/net/context"
 )
 
 // textDictionaryService struct definition
@@ -16,7 +14,7 @@ func NewTextDictionaryService() *textDictionaryService {
 }
 
 // Process processes a given dictionary string and returns the parsed Nodes or an error
-func (tds *textDictionaryService) Process(ctx context.Context, dic string) ([]Node, error) {
+func (tds *textDictionaryService) Process(dic string) ([]Node, []error) {
 
 	// Use the new parser to parse the input
 	l := newLexer(dic)
@@ -26,17 +24,19 @@ func (tds *textDictionaryService) Process(ctx context.Context, dic string) ([]No
 	parser := NewParser(l)
 	parsedNodes := parser.GetNodes()
 
+	if len(l.GetErrors()) != 0 {
+		return nil, l.GetErrors()
+	}
 	if len(parsedNodes) == 0 {
 		err := fmt.Errorf("no nodes were parsed")
-		logger.Logger.ErrorContext(ctx, err.Error())
-		return nil, err
+		return nil, []error{err}
 	}
 
 	return parsedNodes, nil
 }
 
-// decodeBase64 decodes a Base64 encoded string
-func (tds *textDictionaryService) decodeBase64(s string) (string, error) {
+// DecodeBase64 decodes a Base64 encoded string
+func (tds *textDictionaryService) DecodeBase64(s string) (string, error) {
 
 	decoded, err := base64.StdEncoding.DecodeString(s)
 	if err != nil {

backend/pkg/textdic/text_dictionary_service_test.go

@@ -1,12 +1,11 @@
 package textdic
 
 import (
-	"golang.org/x/net/context"
 	"sync"
 	"testing"
 )
 
-func TestParserServiceAll(t *testing.T) {
+func TestParserService(t *testing.T) {
 	// Test input
 	var input = `
 trot out 自慢げに話題に持ち出す
@@ -63,10 +62,11 @@ There is no leeway to provide services free of charge for the sake of others. 
 
 				// Create a new parser service
 				service := NewTextDictionaryService()
-				ctx := context.Background()
 
+				yyDebug = 5
+				yyErrorVerbose = true
 				// Process the dictionary input
-				parsedNodes, err := service.Process(ctx, tc.input)
+				parsedNodes, err := service.Process(tc.input)
 				if err != nil {
 					t.Fatalf("unexpected error: %v", err)
 				}
@@ -112,13 +112,12 @@ There is no leeway to provide services free of charge for the sake of others. 
 			tc := tc // capture range variable to avoid issues in parallel tests
 			t.Run(tc.name, func(t *testing.T) {
 				t.Parallel() // Mark the test to run in parallel
-				ctx := context.Background()
 
 				// Create a new parser service
 				service := NewTextDictionaryService()
 
 				// Process the dictionary input
-				parsedNodes, err := service.Process(ctx, tc.input)
+				parsedNodes, err := service.Process(tc.input)
 
 				if tc.wantErr {
 					if err == nil {
@@ -150,10 +149,9 @@ There is no leeway to provide services free of charge for the sake of others. 
 
 					// Create a new parser service
 					service := NewTextDictionaryService()
-					ctx := context.Background()
 
 					// Process the dictionary input
-					parsedNodes, err := service.Process(ctx, tc.input)
+					parsedNodes, err := service.Process(tc.input)
 					if err != nil {
 						t.Errorf("unexpected error: %v", err)
 						return
@@ -198,7 +196,7 @@ There is no leeway to provide services free of charge for the sake of others. 
 			t.Run(tc.name, func(t *testing.T) {
 				t.Parallel() // Mark the test to run in parallel
 
-				result, err := service.decodeBase64(tc.input)
+				result, err := service.DecodeBase64(tc.input)
 				if tc.wantErr {
 					if err == nil {
 						t.Errorf("expected error but got none")
@@ -214,5 +212,4 @@ There is no leeway to provide services free of charge for the sake of others. 
 			})
 		}
 	})
-
 }