From f8724f384dca9de7c8527cc171f630a55cbebd49 Mon Sep 17 00:00:00 2001 From: Yasuyuki Takeo Date: Thu, 15 Aug 2024 04:57:05 +0900 Subject: [PATCH] Fix parser could not handle dictionaly with EOF right after --- backend/pkg/textdic/lexer.go | 5 +++-- backend/pkg/textdic/parser.go | 20 +++++++++---------- backend/pkg/textdic/parser.y | 4 ++-- .../textdic/text_dictionary_service_test.go | 10 ++++++++++ 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/backend/pkg/textdic/lexer.go b/backend/pkg/textdic/lexer.go index 31ff474..5101fcf 100644 --- a/backend/pkg/textdic/lexer.go +++ b/backend/pkg/textdic/lexer.go @@ -2,6 +2,7 @@ package textdic import ( "fmt" + "io" "regexp" "strings" "unicode" @@ -43,7 +44,7 @@ func (l *lexer) isJapanese(r rune) bool { func (l *lexer) Lex(lval *yySymType) int { r, err := l.skipWhiteSpace() - if err != nil { + if err == io.EOF { // Done with parsing return 0 } @@ -58,7 +59,7 @@ func (l *lexer) Lex(lval *yySymType) int { if l.isJapanese(r) { return l.lexDefinition(lval) } - return EOF + return 0 } func (l *lexer) lexWord(lval *yySymType) int { diff --git a/backend/pkg/textdic/parser.go b/backend/pkg/textdic/parser.go index 9863cc1..a8e4516 100644 --- a/backend/pkg/textdic/parser.go +++ b/backend/pkg/textdic/parser.go @@ -26,7 +26,6 @@ type yySymType struct { const WORD = 57346 const DEFINITION = 57347 const NEWLINE = 57348 -const EOF = 57349 var yyToknames = [...]string{ "$end", @@ -35,7 +34,6 @@ var yyToknames = [...]string{ "WORD", "DEFINITION", "NEWLINE", - "EOF", } var yyStatenames = [...]string{} @@ -74,18 +72,18 @@ var yyExca = [...]int8{ const yyPrivate = 57344 -const yyLast = 8 +const yyLast = 7 var yyAct = [...]int8{ - 4, 8, 5, 7, 3, 1, 2, 6, + 4, 3, 5, 7, 6, 1, 2, } var yyPact = [...]int16{ - -4, -1000, -4, -1000, -2, -1000, -1000, -5, -1000, + -4, -1000, -4, -1000, -2, -1000, -1000, -1000, } var yyPgo = [...]int8{ - 0, 4, 6, 5, + 0, 1, 6, 5, } var yyR1 = [...]int8{ @@ -93,15 +91,15 @@ var yyR1 = [...]int8{ } var yyR2 = [...]int8{ - 0, 1, 2, 1, 3, 1, + 0, 1, 2, 1, 2, 1, } var yyChk = [...]int16{ - -1000, -3, -2, -1, 4, 6, -1, 5, 6, + -1000, -3, -2, -1, 4, 6, -1, 5, } var yyDef = [...]int8{ - 0, -2, 1, 3, 0, 5, 2, 0, 4, + 0, -2, 1, 3, 0, 5, 2, 4, } var yyTok1 = [...]int8{ @@ -109,7 +107,7 @@ var yyTok1 = [...]int8{ } var yyTok2 = [...]int8{ - 2, 3, 4, 5, 6, 7, + 2, 3, 4, 5, 6, } var yyTok3 = [...]int8{ @@ -481,7 +479,7 @@ yydefault: } } case 4: - yyDollar = yyS[yypt-3 : yypt+1] + yyDollar = yyS[yypt-2 : yypt+1] //line ./pkg/textdic/parser.y:39 { yyVAL.node = Node{Word: yyDollar[1].str, Definition: yyDollar[2].str} diff --git a/backend/pkg/textdic/parser.y b/backend/pkg/textdic/parser.y index 43e7a1d..60162b3 100644 --- a/backend/pkg/textdic/parser.y +++ b/backend/pkg/textdic/parser.y @@ -17,7 +17,7 @@ type Nodes []Node nodes Nodes } -%token WORD DEFINITION NEWLINE EOF +%token WORD DEFINITION NEWLINE %type entry %type entries %type start @@ -36,7 +36,7 @@ entries ; entry - : WORD DEFINITION NEWLINE { $$ = Node{Word: $1, Definition: $2} } + : WORD DEFINITION { $$ = Node{Word: $1, Definition: $2} } | NEWLINE { $$ = Node{} } // Ignore empty line ; diff --git a/backend/pkg/textdic/text_dictionary_service_test.go b/backend/pkg/textdic/text_dictionary_service_test.go index f44b39d..7b4c3f7 100644 --- a/backend/pkg/textdic/text_dictionary_service_test.go +++ b/backend/pkg/textdic/text_dictionary_service_test.go @@ -28,6 +28,8 @@ leeway 〔自分の好きなように行動・思考できる〕自由（裁量 There is no leeway to provide services free of charge for the sake of others. 他人のために無償でサービスをする余裕はない。 ` + var inputEOF = `leeway 〔自分の好きなように行動・思考できる〕自由（裁量）度◆不可〔時間・金などの〕余裕、ゆとり +There is no leeway to provide services free of charge for the sake of others. 他人のために無償でサービスをする余裕はない。` // Define the test cases var testCases = []struct { @@ -51,6 +53,14 @@ There is no leeway to provide services free of charge for the sake of others. {Word: "There is no leeway to provide services free of charge for the sake of others.", Definition: "他人のために無償でサービスをする余裕はない。"}, }, }, + { + name: "Valid input EOF", + input: inputEOF, + expected: []Node{ + {Word: "leeway", Definition: "〔自分の好きなように行動・思考できる〕自由（裁量）度◆不可〔時間・金などの〕余裕、ゆとり"}, + {Word: "There is no leeway to provide services free of charge for the sake of others.", Definition: "他人のために無償でサービスをする余裕はない。"}, + }, + }, } // Run TestParserService