From 8047de333bcba836cee5ee17a762cf717e05f879 Mon Sep 17 00:00:00 2001
From: Manav Patel <manavp1996.mp@gmail.com>
Date: Mon, 15 Apr 2024 17:37:56 -0400
Subject: [PATCH] =?UTF-8?q?feat:=20Adding=20ability=20to=20parse=20files?=
 =?UTF-8?q?=20containing=20diacritical=20characters=E2=80=A6=20(#22)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat: Adding ability to parse files containing diacritical characters and filtering out symbols. Adding support for credit and debit reversal record parsing.

* fix: language

---------

Co-authored-by: Manav <manav@synctera.com>
---
 file_streamer.go |  8 +++++++-
 normalize.go     |  2 +-
 reader.go        | 19 +++++++++++++++++--
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/file_streamer.go b/file_streamer.go
index ba0d46f..fdcfc44 100644
--- a/file_streamer.go
+++ b/file_streamer.go
@@ -113,7 +113,13 @@ func (fs *FileStreamer) ScanTxn() (Transaction, error) {
 			return nil, io.EOF
 		}
 		fs.currentLine++
-		fs.lineContents = strings.TrimSpace(fs.scanner.Text())
+
+		line, err := normalize(strings.TrimSpace(fs.scanner.Text()))
+		if err != nil {
+			return nil, fmt.Errorf("failed to read transaction line: %w", err)
+		}
+
+		fs.lineContents = line
 
 		if len(fs.lineContents) == 0 || isFooterRecord(string(fs.lineContents[0])) {
 			return nil, io.EOF
diff --git a/normalize.go b/normalize.go
index 77321e6..ad06631 100644
--- a/normalize.go
+++ b/normalize.go
@@ -11,7 +11,7 @@ import (
 )
 
 // normalizer removes diacritical characters and replaces them with their ASCII representation
-var normalizer = transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+var normalizer = transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), runes.Remove(runes.In(unicode.So)), norm.NFC)
 
 func normalize(in string) (string, error) {
 	s, _, err := transform.String(normalizer, in)
diff --git a/reader.go b/reader.go
index 855e7e7..09d7d7c 100644
--- a/reader.go
+++ b/reader.go
@@ -22,7 +22,10 @@ func NewReader(in io.Reader) *Reader {
 // Use the FileStreamer object to be able ignore errors and proceed parsing the file.
 func (r *Reader) ReadFile() (File, error) {
 	for r.scanner.Scan() {
-		line := r.scanner.Text()
+		line, err := normalize(r.scanner.Text())
+		if err != nil {
+			return File{}, fmt.Errorf("failed to read line: %w", err)
+		}
 		recordType := line[:1]
 		if recordType == string(HeaderRecord) {
 			if err := r.parseARecord(line); err != nil {
@@ -96,7 +99,19 @@ func (r *Reader) parseTxnRecord(data string) error {
 				return fmt.Errorf("failed to parse credit return transaction: %w", err)
 			}
 			r.File.Txns = append(r.File.Txns, &creditReturns)
-		case HeaderRecord, FooterRecord, CreditReverseRecord, DebitReverseRecord, NoticeOfChangeRecord, NoticeOfChangeHeader, NoticeOfChangeFooter:
+		case CreditReverseRecord:
+			creditReverse := CreditReverse{}
+			if err := creditReverse.Parse(rawTxnSegment[startIdx:endIdx]); err != nil {
+				return fmt.Errorf("failed to parse credit reverse transaction: %w", err)
+			}
+			r.File.Txns = append(r.File.Txns, &creditReverse)
+		case DebitReverseRecord:
+			debitReverseRecord := DebitReverse{}
+			if err := debitReverseRecord.Parse(rawTxnSegment[startIdx:endIdx]); err != nil {
+				return fmt.Errorf("failed to parse debit reverse transaction: %w", err)
+			}
+			r.File.Txns = append(r.File.Txns, &debitReverseRecord)
+		case HeaderRecord, FooterRecord, NoticeOfChangeRecord, NoticeOfChangeHeader, NoticeOfChangeFooter:
 			return fmt.Errorf("unexpected %s record", recType)
 		}
 		startIdx = endIdx