From 2d3c48ce14bcfd06e4f5a4b86966b1ca5c249386 Mon Sep 17 00:00:00 2001 From: Yasuyuki Takeo Date: Sun, 11 Aug 2024 18:47:11 +0900 Subject: [PATCH] Refactor Dictionary parser --- backend/go.mod | 1 - backend/go.sum | 2 -- backend/pkg/textdic/lexer.go | 32 +++++++++---------- backend/pkg/textdic/parser.go | 17 +++++----- backend/pkg/textdic/parser.y | 4 +-- backend/pkg/textdic/parser_test.go | 13 ++++++-- .../pkg/textdic/text_dictionary_service.go | 14 ++++---- .../textdic/text_dictionary_service_test.go | 17 ++++------ 8 files changed, 50 insertions(+), 50 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 738dde2..bbd87d6 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -17,7 +17,6 @@ require ( github.com/testcontainers/testcontainers-go v0.31.0 github.com/vektah/gqlparser/v2 v2.5.16 golang.org/x/net v0.26.0 - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.10 ) diff --git a/backend/go.sum b/backend/go.sum index a40d5ff..de99081 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -254,8 +254,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= diff --git a/backend/pkg/textdic/lexer.go b/backend/pkg/textdic/lexer.go index cd0d019..31ff474 100644 --- a/backend/pkg/textdic/lexer.go +++ b/backend/pkg/textdic/lexer.go @@ -14,7 +14,7 @@ type lexer struct { } func newLexer(input string) *lexer { - return &lexer{input: strings.NewReader(input), lineNo: 0} + return &lexer{input: strings.NewReader(input), lineNo: 1} } func (l *lexer) Peek() rune { @@ -42,23 +42,23 @@ func (l *lexer) isJapanese(r rune) bool { } func (l *lexer) Lex(lval *yySymType) int { - for { - r, err := l.skipWhiteSpace() - if err != nil { - return EOF - } - - if l.isNewLine(r) { - l.lineNo++ - return NEWLINE - } + r, err := l.skipWhiteSpace() + if err != nil { + // Done with parsing + return 0 + } - if l.isEnglishAndWhitespace(r) { - return l.lexWord(lval) - } else if l.isJapanese(r) { - return l.lexDefinition(lval) - } + if l.isNewLine(r) { + l.lineNo++ + return NEWLINE + } + if l.isEnglishAndWhitespace(r) { + return l.lexWord(lval) + } + if l.isJapanese(r) { + return l.lexDefinition(lval) } + return EOF } func (l *lexer) lexWord(lval *yySymType) int { diff --git a/backend/pkg/textdic/parser.go b/backend/pkg/textdic/parser.go index 3cf1857..9863cc1 100644 --- a/backend/pkg/textdic/parser.go +++ b/backend/pkg/textdic/parser.go @@ -74,19 +74,18 @@ var yyExca = [...]int8{ const yyPrivate = 57344 -const yyLast = 11 +const yyLast = 8 var yyAct = [...]int8{ - 4, 9, 5, 6, 4, 3, 5, 8, 7, 1, - 2, + 4, 8, 5, 7, 3, 1, 2, 6, } var yyPact = [...]int16{ - 0, -1000, -4, -1000, 2, -1000, -1000, -1000, -5, -1000, + -4, -1000, -4, -1000, -2, -1000, -1000, -5, -1000, } var yyPgo = [...]int8{ - 0, 5, 10, 9, + 0, 4, 6, 5, } var yyR1 = [...]int8{ @@ -94,15 +93,15 @@ var yyR1 = [...]int8{ } var yyR2 = [...]int8{ - 0, 2, 2, 1, 3, 1, + 0, 1, 2, 1, 3, 1, } var yyChk = [...]int16{ - -1000, -3, -2, -1, 4, 6, 7, -1, 5, 6, + -1000, -3, -2, -1, 4, 6, -1, 5, 6, } var yyDef = [...]int8{ - 0, -2, 0, 3, 0, 5, 1, 2, 0, 4, + 0, -2, 1, 3, 0, 5, 2, 0, 4, } var yyTok1 = [...]int8{ @@ -455,7 +454,7 @@ yydefault: switch yynt { case 1: - yyDollar = yyS[yypt-2 : yypt+1] + yyDollar = yyS[yypt-1 : yypt+1] //line ./pkg/textdic/parser.y:30 { yyVAL.nodes = yyDollar[1].nodes diff --git a/backend/pkg/textdic/parser.y b/backend/pkg/textdic/parser.y index 630ac00..43e7a1d 100644 --- a/backend/pkg/textdic/parser.y +++ b/backend/pkg/textdic/parser.y @@ -27,7 +27,7 @@ type Nodes []Node %% start - : entries EOF { $$ = $1; yyrcvr.setNodes($1); } + : entries { $$ = $1; yyrcvr.setNodes($1); } ; entries @@ -37,7 +37,7 @@ entries entry : WORD DEFINITION NEWLINE { $$ = Node{Word: $1, Definition: $2} } - | NEWLINE { $$ = Node{} } // 空行を無視する + | NEWLINE { $$ = Node{} } // Ignore empty line ; %% diff --git a/backend/pkg/textdic/parser_test.go b/backend/pkg/textdic/parser_test.go index b76f3f2..d0daeea 100644 --- a/backend/pkg/textdic/parser_test.go +++ b/backend/pkg/textdic/parser_test.go @@ -1,6 +1,7 @@ package textdic import ( + "github.com/stretchr/testify/assert" "testing" ) @@ -17,6 +18,8 @@ jarring 気に障る rube 田舎者 + + out of touch 情報に疎い、 opaque 不透明な @@ -83,21 +86,25 @@ There is no leeway to provide services free of charge for the sake of others. var input = ` trot out 自慢げに話題に持ち出す #エラーになる +!もう一つエラー ` - // Create a new lexer with the input l := newLexer(input) - yyDebug = 1 + yyDebug = 5 yyErrorVerbose = true // Parse the input using the parser instance - NewParser(l) + parser := NewParser(l) errors := l.GetErrors() if len(errors) == 0 { t.Errorf("expected errors, but got none") } + // It should get one correct data in the node + nodes := parser.GetNodes() + assert.Equal(t, 1, len(nodes)) + for _, err := range errors { if err.Error() == "" { t.Errorf("expected error message, but got empty string") diff --git a/backend/pkg/textdic/text_dictionary_service.go b/backend/pkg/textdic/text_dictionary_service.go index f675aed..dcde542 100644 --- a/backend/pkg/textdic/text_dictionary_service.go +++ b/backend/pkg/textdic/text_dictionary_service.go @@ -1,10 +1,8 @@ package textdic import ( - "backend/pkg/logger" "encoding/base64" "fmt" - "golang.org/x/net/context" ) // textDictionaryService struct definition @@ -16,7 +14,7 @@ func NewTextDictionaryService() *textDictionaryService { } // Process processes a given dictionary string and returns the parsed Nodes or an error -func (tds *textDictionaryService) Process(ctx context.Context, dic string) ([]Node, error) { +func (tds *textDictionaryService) Process(dic string) ([]Node, []error) { // Use the new parser to parse the input l := newLexer(dic) @@ -26,17 +24,19 @@ func (tds *textDictionaryService) Process(ctx context.Context, dic string) ([]No parser := NewParser(l) parsedNodes := parser.GetNodes() + if len(l.GetErrors()) != 0 { + return nil, l.GetErrors() + } if len(parsedNodes) == 0 { err := fmt.Errorf("no nodes were parsed") - logger.Logger.ErrorContext(ctx, err.Error()) - return nil, err + return nil, []error{err} } return parsedNodes, nil } -// decodeBase64 decodes a Base64 encoded string -func (tds *textDictionaryService) decodeBase64(s string) (string, error) { +// DecodeBase64 decodes a Base64 encoded string +func (tds *textDictionaryService) DecodeBase64(s string) (string, error) { decoded, err := base64.StdEncoding.DecodeString(s) if err != nil { diff --git a/backend/pkg/textdic/text_dictionary_service_test.go b/backend/pkg/textdic/text_dictionary_service_test.go index 2ed97d6..f44b39d 100644 --- a/backend/pkg/textdic/text_dictionary_service_test.go +++ b/backend/pkg/textdic/text_dictionary_service_test.go @@ -1,12 +1,11 @@ package textdic import ( - "golang.org/x/net/context" "sync" "testing" ) -func TestParserServiceAll(t *testing.T) { +func TestParserService(t *testing.T) { // Test input var input = ` trot out 自慢げに話題に持ち出す @@ -63,10 +62,11 @@ There is no leeway to provide services free of charge for the sake of others. // Create a new parser service service := NewTextDictionaryService() - ctx := context.Background() + yyDebug = 5 + yyErrorVerbose = true // Process the dictionary input - parsedNodes, err := service.Process(ctx, tc.input) + parsedNodes, err := service.Process(tc.input) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -112,13 +112,12 @@ There is no leeway to provide services free of charge for the sake of others. tc := tc // capture range variable to avoid issues in parallel tests t.Run(tc.name, func(t *testing.T) { t.Parallel() // Mark the test to run in parallel - ctx := context.Background() // Create a new parser service service := NewTextDictionaryService() // Process the dictionary input - parsedNodes, err := service.Process(ctx, tc.input) + parsedNodes, err := service.Process(tc.input) if tc.wantErr { if err == nil { @@ -150,10 +149,9 @@ There is no leeway to provide services free of charge for the sake of others. // Create a new parser service service := NewTextDictionaryService() - ctx := context.Background() // Process the dictionary input - parsedNodes, err := service.Process(ctx, tc.input) + parsedNodes, err := service.Process(tc.input) if err != nil { t.Errorf("unexpected error: %v", err) return @@ -198,7 +196,7 @@ There is no leeway to provide services free of charge for the sake of others. t.Run(tc.name, func(t *testing.T) { t.Parallel() // Mark the test to run in parallel - result, err := service.decodeBase64(tc.input) + result, err := service.DecodeBase64(tc.input) if tc.wantErr { if err == nil { t.Errorf("expected error but got none") @@ -214,5 +212,4 @@ There is no leeway to provide services free of charge for the sake of others. }) } }) - }