Skip to content

Commit 5968aef

Browse files
tomtaylortomtaylor
committed
Support pluggable JSON libraries
It's handy to be able to drop in a faster JSON library, such as Jsonrs, for use cases with very large requests and responses, as JSON encoding and decoding time is greatly reduced.
1 parent 16c0623 commit 5968aef

12 files changed

+193
-161
lines changed

‎lib/snap.ex

+3
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ defmodule Snap do
8787
`[:my_app, :snap]`)
8888
* `index_namespace` - see `Snap.Cluster.Namespace` for details (defaults to
8989
`nil`)
90+
* `json_library` - the library used for encoding/decoding JSON (defaults to
91+
`Jason`. You may wish to switch this to [`Jsonrs`](https://hex.pm/packages/jsonrs)
92+
for better performance encoding and decoding large requests and responses)
9093
9194
## Telemetry
9295

‎lib/snap/bulk/action.ex

+84-64
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,143 @@
1+
defmodule Snap.Bulk.Action do
2+
@moduledoc false
3+
@callback to_action_json(struct()) :: map()
4+
@callback to_document_json(struct()) :: map() | nil
5+
end
6+
17
defmodule Snap.Bulk.Action.Create do
28
@moduledoc """
39
Represents a create step in a `Snap.Bulk` operation
410
"""
11+
@behaviour Snap.Bulk.Action
12+
513
@enforce_keys [:doc]
6-
defstruct [:_index, :_id, :require_alias, :doc]
14+
defstruct [:index, :id, :require_alias, :doc]
715

816
@type t :: %__MODULE__{
9-
_index: String.t() | nil,
10-
_id: String.t() | nil,
17+
index: String.t() | nil,
18+
id: String.t() | nil,
1119
require_alias: boolean() | nil,
1220
doc: map()
1321
}
22+
23+
@doc false
24+
def to_action_json(%__MODULE__{index: index, id: id, require_alias: require_alias}) do
25+
values = %{_index: index, _id: id, require_alias: require_alias}
26+
27+
values
28+
|> Enum.reject(&is_nil(elem(&1, 1)))
29+
|> Enum.into(%{})
30+
|> then(fn values -> %{"create" => values} end)
31+
end
32+
33+
@doc false
34+
def to_document_json(%__MODULE__{doc: doc}) do
35+
doc
36+
end
1437
end
1538

1639
defmodule Snap.Bulk.Action.Delete do
1740
@moduledoc """
1841
Represents a delete step in a `Snap.Bulk` operation
1942
"""
20-
@enforce_keys [:_id]
21-
defstruct [:_index, :_id, :require_alias]
43+
@behaviour Snap.Bulk.Action
44+
45+
@enforce_keys [:id]
46+
defstruct [:index, :id, :require_alias]
2247

2348
@type t :: %__MODULE__{
24-
_index: String.t() | nil,
25-
_id: String.t(),
49+
index: String.t() | nil,
50+
id: String.t(),
2651
require_alias: boolean() | nil
2752
}
53+
54+
@doc false
55+
def to_action_json(%__MODULE__{index: index, id: id, require_alias: require_alias}) do
56+
values = %{_index: index, _id: id, require_alias: require_alias}
57+
58+
values
59+
|> Enum.reject(&is_nil(elem(&1, 1)))
60+
|> Enum.into(%{})
61+
|> then(fn values -> %{"delete" => values} end)
62+
end
63+
64+
@doc false
65+
def to_document_json(_), do: nil
2866
end
2967

3068
defmodule Snap.Bulk.Action.Index do
3169
@moduledoc """
3270
Represents an index step in a `Snap.Bulk` operation
3371
"""
72+
@behaviour Snap.Bulk.Action
73+
3474
@enforce_keys [:doc]
35-
defstruct [:_index, :_id, :require_alias, :doc]
75+
defstruct [:index, :id, :require_alias, :doc]
3676

3777
@type t :: %__MODULE__{
38-
_index: String.t() | nil,
39-
_id: String.t() | nil,
78+
index: String.t() | nil,
79+
id: String.t() | nil,
4080
require_alias: boolean() | nil,
4181
doc: map()
4282
}
83+
84+
@doc false
85+
def to_action_json(%__MODULE__{index: index, id: id, require_alias: require_alias}) do
86+
values = %{_index: index, _id: id, require_alias: require_alias}
87+
88+
values
89+
|> Enum.reject(&is_nil(elem(&1, 1)))
90+
|> Enum.into(%{})
91+
|> then(fn values -> %{"index" => values} end)
92+
end
93+
94+
@doc false
95+
def to_document_json(%__MODULE__{doc: doc}) do
96+
doc
97+
end
4398
end
4499

45100
defmodule Snap.Bulk.Action.Update do
46101
@moduledoc """
47102
Represents an update step in a `Snap.Bulk` operation
48103
"""
104+
@behaviour Snap.Bulk.Action
105+
49106
@enforce_keys [:doc]
50107
defstruct [
51-
:_id,
52-
:_index,
53-
:_source,
108+
:id,
109+
:index,
110+
:require_alias,
54111
:doc,
55112
:doc_as_upsert,
56-
:require_alias,
57-
:script,
58-
:upsert
113+
:script
59114
]
60115

61116
@type t :: %__MODULE__{
62-
_id: String.t() | nil,
63-
_index: String.t() | nil,
64-
_source: boolean() | nil,
117+
id: String.t() | nil,
118+
index: String.t() | nil,
119+
require_alias: boolean() | nil,
65120
doc: map(),
66121
doc_as_upsert: boolean() | nil,
67-
require_alias: boolean() | nil,
68-
script: map() | nil,
69-
upsert: map() | nil
122+
script: map() | nil
70123
}
71-
end
72-
73-
defimpl Jason.Encoder, for: Snap.Bulk.Action.Create do
74-
require Jason.Helpers
75-
76-
def encode(%Snap.Bulk.Action.Create{_index: index, _id: id, require_alias: require_alias}, opts) do
77-
values = [_index: index, _id: id, require_alias: require_alias]
78-
79-
values
80-
|> Enum.reject(&is_nil(elem(&1, 1)))
81-
|> then(fn values -> %{"create" => Jason.OrderedObject.new(values)} end)
82-
|> Jason.Encode.map(opts)
83-
end
84-
end
85-
86-
defimpl Jason.Encoder, for: Snap.Bulk.Action.Delete do
87-
require Jason.Helpers
88-
89-
def encode(%Snap.Bulk.Action.Delete{_index: index, _id: id, require_alias: require_alias}, opts) do
90-
values = [_index: index, _id: id, require_alias: require_alias]
91-
92-
values
93-
|> Enum.reject(&is_nil(elem(&1, 1)))
94-
|> then(fn values -> %{"delete" => Jason.OrderedObject.new(values)} end)
95-
|> Jason.Encode.map(opts)
96-
end
97-
end
98124

99-
defimpl Jason.Encoder, for: Snap.Bulk.Action.Update do
100-
require Jason.Helpers
101-
102-
def encode(%Snap.Bulk.Action.Update{_index: index, _id: id, require_alias: require_alias}, opts) do
103-
values = [_index: index, _id: id, require_alias: require_alias]
125+
@doc false
126+
def to_action_json(%__MODULE__{index: index, id: id, require_alias: require_alias}) do
127+
values = %{_index: index, _id: id, require_alias: require_alias}
104128

105129
values
106130
|> Enum.reject(&is_nil(elem(&1, 1)))
107-
|> then(fn values -> %{"update" => Jason.OrderedObject.new(values)} end)
108-
|> Jason.Encode.map(opts)
131+
|> Enum.into(%{})
132+
|> then(fn values -> %{"update" => values} end)
109133
end
110-
end
111-
112-
defimpl Jason.Encoder, for: Snap.Bulk.Action.Index do
113-
require Jason.Helpers
114134

115-
def encode(%Snap.Bulk.Action.Index{_index: index, _id: id, require_alias: require_alias}, opts) do
116-
values = [_index: index, _id: id, require_alias: require_alias]
135+
@doc false
136+
def to_document_json(%__MODULE__{doc: doc, doc_as_upsert: doc_as_upsert, script: script}) do
137+
values = %{doc: doc, doc_as_upsert: doc_as_upsert, script: script}
117138

118139
values
119140
|> Enum.reject(&is_nil(elem(&1, 1)))
120-
|> then(fn values -> %{"index" => Jason.OrderedObject.new(values)} end)
121-
|> Jason.Encode.map(opts)
141+
|> Enum.into(%{})
122142
end
123143
end

‎lib/snap/bulk/actions.ex

+16-37
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,35 @@
11
defmodule Snap.Bulk.Actions do
22
@moduledoc false
33

4-
alias Snap.Bulk.Action.{Create, Index, Update, Delete}
5-
64
@doc """
75
Encodes a list of bulk action structs into line separated JSON for feeding to the
86
/_bulk endpoint.
97
"""
10-
def encode(actions) do
11-
encode_actions([], actions)
8+
def encode(actions, json_library \\ Jason) do
9+
encode_actions([], actions, json_library)
1210
end
1311

14-
defp encode_actions(iolist, []) do
12+
defp encode_actions(iolist, [], _json_library) do
1513
iolist
1614
end
1715

18-
defp encode_actions(iolist, [head | tail]) do
19-
updated_iolist = [iolist, encode_action(head)]
20-
encode_actions(updated_iolist, tail)
21-
end
22-
23-
defp encode_action(%type{} = action) when type in [Create, Index] do
24-
doc = action.doc
25-
26-
doc_json =
27-
doc
28-
|> Jason.encode!()
29-
30-
action_json = encode_action_command(action)
31-
32-
[action_json, "\n", doc_json, "\n"]
16+
defp encode_actions(iolist, [head | tail], json_library) do
17+
updated_iolist = [iolist, encode_action(head, json_library)]
18+
encode_actions(updated_iolist, tail, json_library)
3319
end
3420

35-
defp encode_action(%Delete{} = action) do
36-
action_json = encode_action_command(action)
37-
38-
[action_json, "\n"]
39-
end
40-
41-
defp encode_action(%Update{} = action) do
42-
doc = action.doc
43-
44-
doc_json =
45-
%{doc: doc}
46-
|> Jason.encode!()
47-
48-
action_json = encode_action_command(action)
21+
defp encode_action(%type{} = action, json_library) do
22+
action_json = type.to_action_json(action)
23+
doc_json = type.to_document_json(action)
4924

50-
[action_json, "\n", doc_json, "\n"]
25+
if doc_json do
26+
[encode_json(action_json, json_library), "\n", encode_json(doc_json, json_library), "\n"]
27+
else
28+
[encode_json(action_json, json_library), "\n"]
29+
end
5130
end
5231

53-
defp encode_action_command(action) do
54-
Jason.encode!(action)
32+
defp encode_json(json, json_library) do
33+
json_library.encode!(json)
5534
end
5635
end

‎lib/snap/bulk/bulk.ex

+5-4
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ defmodule Snap.Bulk do
2121
2222
```
2323
actions = [
24-
%Snap.Bulk.Action.Create{_id: 1, doc: %{foo: "bar"}},
25-
%Snap.Bulk.Action.Create{_id: 2, doc: %{foo: "bar"}},
26-
%Snap.Bulk.Action.Create{_id: 3, doc: %{foo: "bar"}}
24+
%Snap.Bulk.Action.Create{id: 1, doc: %{foo: "bar"}},
25+
%Snap.Bulk.Action.Create{id: 2, doc: %{foo: "bar"}},
26+
%Snap.Bulk.Action.Create{id: 3, doc: %{foo: "bar"}}
2727
]
2828
2929
actions
@@ -111,7 +111,8 @@ defmodule Snap.Bulk do
111111
end
112112

113113
defp process_chunk(actions, cluster, index, params, request_opts, error_count, _max_errors) do
114-
body = Actions.encode(actions)
114+
json_library = cluster.json_library()
115+
body = Actions.encode(actions, json_library)
115116

116117
headers = [{"content-type", "application/x-ndjson"}]
117118

‎lib/snap/cluster.ex

+9-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ defmodule Snap.Cluster do
4444
Supervisor.config(__MODULE__)
4545
end
4646

47+
@doc """
48+
Returns the JSON library configured for the Cluster.
49+
"""
50+
def json_library() do
51+
Keyword.get(config(), :json_library, Jason)
52+
end
53+
4754
@doc """
4855
Returns the otp_app that the Cluster was defined with.
4956
"""
@@ -124,7 +131,8 @@ defmodule Snap.Cluster do
124131
index_namespace: String.t() | nil,
125132
telemetry_prefix: list(atom()),
126133
http_client_adapter:
127-
Snap.HTTPClient.t() | {Snap.HTTPClient.t(), adapter_config :: Keyword.t()}
134+
Snap.HTTPClient.t() | {Snap.HTTPClient.t(), adapter_config :: Keyword.t()},
135+
json_library: module()
128136
]
129137

130138
@doc """

‎lib/snap/multi/multi.ex

+10-9
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ defmodule Snap.Multi do
6666
{:ok, Snap.Multi.Response.t()} | {:error, Snap.Cluster.error()}
6767
def run(%__MODULE__{} = multi, cluster, index_or_alias, params \\ [], headers \\ [], opts \\ []) do
6868
ids = build_ids(multi.searches)
69-
body = encode(multi)
69+
json_library = cluster.json_library()
70+
body = encode(multi, json_library)
7071
headers = headers ++ [{"content-type", "application/x-ndjson"}]
7172
namespaced_index = Namespace.add_namespace_to_index(index_or_alias, cluster)
7273

@@ -76,23 +77,23 @@ defmodule Snap.Multi do
7677
end
7778
end
7879

79-
defp encode(%__MODULE__{} = multi) do
80+
defp encode(%__MODULE__{} = multi, json_library) do
8081
multi.searches
81-
|> Enum.flat_map(&encode_search/1)
82+
|> Enum.flat_map(&encode_search(&1, json_library))
8283
end
8384

84-
defp encode_search(%Search{headers: headers, body: body}) do
85-
[encode_headers(headers), "\n", encode_body(body), "\n"]
85+
defp encode_search(%Search{headers: headers, body: body}, json_library) do
86+
[encode_headers(headers, json_library), "\n", encode_body(body, json_library), "\n"]
8687
end
8788

88-
defp encode_headers(headers) do
89+
defp encode_headers(headers, json_library) do
8990
headers
9091
|> Enum.into(%{})
91-
|> Jason.encode!(pretty: false)
92+
|> json_library.encode!(pretty: false)
9293
end
9394

94-
defp encode_body(body) do
95-
Jason.encode!(body, pretty: false)
95+
defp encode_body(body, json_library) do
96+
json_library.encode!(body, pretty: false)
9697
end
9798

9899
defp build_ids(searches) do

0 commit comments

Comments
 (0)