Fix parser could not handle dictionaly with EOF right after

yasuflatland-lf · Aug 14, 2024 · f8724f3 · f8724f3
1 parent 4d055df
commit f8724f3
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 15 deletions.
diff --git a/backend/pkg/textdic/lexer.go b/backend/pkg/textdic/lexer.go
@@ -2,6 +2,7 @@ package textdic
 
 import (
 	"fmt"
+	"io"
 	"regexp"
 	"strings"
 	"unicode"
@@ -43,7 +44,7 @@ func (l *lexer) isJapanese(r rune) bool {
 
 func (l *lexer) Lex(lval *yySymType) int {
 	r, err := l.skipWhiteSpace()
-	if err != nil {
+	if err == io.EOF {
 		// Done with parsing
 		return 0
 	}
@@ -58,7 +59,7 @@ func (l *lexer) Lex(lval *yySymType) int {
 	if l.isJapanese(r) {
 		return l.lexDefinition(lval)
 	}
-	return EOF
+	return 0
 }
 
 func (l *lexer) lexWord(lval *yySymType) int {

diff --git a/backend/pkg/textdic/parser.go b/backend/pkg/textdic/parser.go
diff --git a/backend/pkg/textdic/parser.y b/backend/pkg/textdic/parser.y
@@ -17,7 +17,7 @@ type Nodes []Node
 	nodes Nodes
 }
 
-%token<str> WORD DEFINITION NEWLINE EOF
+%token<str> WORD DEFINITION NEWLINE
 %type<node> entry
 %type<nodes> entries
 %type<nodes> start
@@ -36,7 +36,7 @@ entries
 	;
 
 entry
-	: WORD DEFINITION NEWLINE { $$ = Node{Word: $1, Definition: $2} }
+	: WORD DEFINITION { $$ = Node{Word: $1, Definition: $2} }
 	| NEWLINE { $$ = Node{} } // Ignore empty line
 	;
 

diff --git a/backend/pkg/textdic/text_dictionary_service_test.go b/backend/pkg/textdic/text_dictionary_service_test.go
@@ -28,6 +28,8 @@ leeway 〔自分の好きなように行動・思考できる〕自由（裁量
 There is no leeway to provide services free of charge for the sake of others. 他人のために無償でサービスをする余裕はない。
 
 `
+	var inputEOF = `leeway 〔自分の好きなように行動・思考できる〕自由（裁量）度◆不可〔時間・金などの〕余裕、ゆとり
+There is no leeway to provide services free of charge for the sake of others. 他人のために無償でサービスをする余裕はない。`
 
 	// Define the test cases
 	var testCases = []struct {
@@ -51,6 +53,14 @@ There is no leeway to provide services free of charge for the sake of others.
 				{Word: "There is no leeway to provide services free of charge for the sake of others.", Definition: "他人のために無償でサービスをする余裕はない。"},
 			},
 		},
+		{
+			name:  "Valid input EOF",
+			input: inputEOF,
+			expected: []Node{
+				{Word: "leeway", Definition: "〔自分の好きなように行動・思考できる〕自由（裁量）度◆不可〔時間・金などの〕余裕、ゆとり"},
+				{Word: "There is no leeway to provide services free of charge for the sake of others.", Definition: "他人のために無償でサービスをする余裕はない。"},
+			},
+		},
 	}
 
 	// Run TestParserService