Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: escape tabs in tag strings #111

Merged
merged 4 commits into from
Oct 18, 2024
+25 −11
Merged

feat: escape tabs in tag strings #111

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f2d0d59
feat: escape tab chars in tags
karel-rehor Oct 17, 2024
d0fc985
chore: refactor escaping tag values
karel-rehor Oct 17, 2024
0d31a4b
chore: replace custom replacer with strings.Replacer
karel-rehor Oct 17, 2024
33c78a2
docs: update CHANGELOG.md
karel-rehor Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions influxdb3/client_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,8 @@ func TestEscapedStringValues(t *testing.T) {
map[string]string{
"tag1": "new\nline and space",
"tag2": "escaped\\nline and space",
"tag3": "escaped\nline and\ttab",
"tag4": "preescaped\\nline and\\ttab",
},
map[string]interface{}{
"fVal": 41.3,
Expand All @@ -359,5 +361,7 @@ func TestEscapedStringValues(t *testing.T) {
assert.EqualValues(t, "greetings\\nearthlings", qit.Value()["sVal"])
assert.EqualValues(t, "new\\nline and space", qit.Value()["tag1"])
assert.EqualValues(t, "escaped\\nline and space", qit.Value()["tag2"])
assert.EqualValues(t, "escaped\\nline and\\ttab", qit.Value()["tag3"])
assert.EqualValues(t, "preescaped\\nline and\\ttab", qit.Value()["tag4"])
}
}
24 changes: 19 additions & 5 deletions influxdb3/point.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,22 @@ func (p *Point) MarshalBinaryWithDefaultTags(precision lineprotocol.Precision, d
enc.SetPrecision(precision)
enc.StartLine(p.Values.MeasurementName)

// N.B. Some customers have requested support for newline and tab chars in tag values (EAR 5476)
// Though this is outside the lineprotocol specification, it was supported in
// previous GO client versions.
tagEscapes := map[string]string{
"\n": "\\n",
"\t": "\\t",
}

replacer := func(reps map[string]string, s string) string {
result := s
for key, val := range reps {
result = strings.ReplaceAll(result, key, val)
}
return result
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you thinks about using package level Replacer?

Something like:

var replacer = strings.NewReplacer(
	"\n", "\\n",
	"\t", "\\t",
)

and then use:

		if value, ok := p.Values.Tags[tagKey]; ok {
			enc.AddTag(tagKey, replacer.Replace(value))
		} else {
			enc.AddTag(tagKey, replacer.Replace(defaultTags[tagKey]))
		}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aha... I was unaware that the Replacer type already exists in the strings package. Updated.


// sort Tags
tagKeys := make([]string, 0, len(p.Values.Tags)+len(defaultTags))
for k := range p.Values.Tags {
Expand All @@ -269,13 +285,11 @@ func (p *Point) MarshalBinaryWithDefaultTags(precision lineprotocol.Precision, d
}
lastKey = tagKey

// N.B. Some customers have requested support for newline chars in tag values (EAR 5476)
// Though this is outside the lineprotocol specification, it was supported in
// previous GO client versions.
// N.B. Some customers have requested support for newline and tab chars in tag values (EAR 5476)
if value, ok := p.Values.Tags[tagKey]; ok {
enc.AddTag(tagKey, strings.ReplaceAll(value, "\n", "\\n"))
enc.AddTag(tagKey, replacer(tagEscapes, value))
} else {
enc.AddTag(tagKey, strings.ReplaceAll(defaultTags[tagKey], "\n", "\\n"))
enc.AddTag(tagKey, replacer(tagEscapes, defaultTags[tagKey]))
}
}

Expand Down
14 changes: 8 additions & 6 deletions influxdb3/point_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,12 +178,14 @@ func TestPointDefaultTags(t *testing.T) {
assert.EqualValues(t, `test,tag1=a,tag2=b,tag3=f float64=80.1234567 60000000070`+"\n", string(line))
}

func TestPointWithNewlineTags(t *testing.T) {
func TestPointWithEscapedTags(t *testing.T) {
p := NewPoint("test",
map[string]string{
"tag1": "new\nline and space",
"tag2": "escaped\\nline and space",
"ambiTag": "ambiguous\ntag",
"tabTag1": "drink\tTab",
"tabTag2": "Tab\\tulator",
},
map[string]interface{}{
"fVal": 41.3,
Expand All @@ -198,16 +200,16 @@ func TestPointWithNewlineTags(t *testing.T) {
line, err := p.MarshalBinary(lineprotocol.Nanosecond)
require.NoError(t, err)
assert.EqualValues(t,
"test,ambiTag=ambiguous\\ntag,tag1=new\\nline\\ and\\ space,tag2=escaped\\nline\\ and\\ space "+
"fVal=41.3 60000000070\n",
"test,ambiTag=ambiguous\\ntag,tabTag1=drink\\tTab,tabTag2=Tab\\tulator,"+
"tag1=new\\nline\\ and\\ space,tag2=escaped\\nline\\ and\\ space fVal=41.3 60000000070\n",
string(line))

line, err = p.MarshalBinaryWithDefaultTags(lineprotocol.Nanosecond, defaultTags)
require.NoError(t, err)
assert.EqualValues(t,
"test,ambiTag=ambiguous\\ntag,defTag1=default\\nline\\ and\\ space,defTag2=escaped"+
"\\ndefault\\ line\\ and\\ space,tag1=new\\nline\\ and\\ space,tag2=escaped\\nline\\ and\\ space "+
"fVal=41.3 60000000070\n",
"test,ambiTag=ambiguous\\ntag,defTag1=default\\nline\\ and\\ space,"+
"defTag2=escaped\\ndefault\\ line\\ and\\ space,tabTag1=drink\\tTab,tabTag2=Tab\\tulator,"+
"tag1=new\\nline\\ and\\ space,tag2=escaped\\nline\\ and\\ space fVal=41.3 60000000070\n",
string(line))

pInvalid := NewPoint("test", map[string]string{
Expand Down