Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

case insensitive search #925

Merged
merged 13 commits into from
Feb 20, 2025
38 changes: 32 additions & 6 deletions src/dso_api/dynamic_api/filters/lookups.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,31 @@ def as_sql(self, compiler, connection):
lhs, lhs_params = self.process_lhs(compiler, connection, lhs=lhs) # (field, [])
rhs, rhs_params = self.process_rhs(compiler, connection) # ("%s", [value])

field_type = self.lhs.output_field.get_internal_type()
if lhs_nullable and rhs is not None:
# Allow field__not=value to return NULL fields too.
return (
f"({lhs} IS NULL OR {lhs} != {rhs})",
list(lhs_params + lhs_params) + rhs_params,
)
if field_type in ["CharField", "TextField"]:
return (
f"({lhs}) IS NULL OR LOWER({lhs}) != LOWER({rhs}))",
list(lhs_params + lhs_params)
+ [rhs.lower() if isinstance(rhs, str) else rhs for rhs in rhs_params],
)
else:
return (
f"({lhs} IS NULL OR {lhs} != {rhs})",
list(lhs_params + lhs_params) + rhs_params,
)

elif rhs_params and rhs_params[0] is None:
# Allow field__not=None to work.
return f"{lhs} IS NOT NULL", lhs_params
else:
return f"{lhs} != {rhs}", list(lhs_params) + rhs_params
if field_type in ["CharField", "TextField"]:
return f"LOWER({lhs}) != LOWER({rhs})", list(lhs_params) + [
rhs.lower() if isinstance(rhs, str) else rhs for rhs in rhs_params
]
else:
return f"{lhs} != {rhs}", list(lhs_params) + rhs_params


@models.CharField.register_lookup
Expand All @@ -71,9 +85,10 @@ def as_sql(self, compiler, connection):
# rhs = %s
# lhs_params = []
# lhs_params = ["prep-value"]

lhs, lhs_params = self.process_lhs(compiler, connection)
rhs, rhs_params = self.process_rhs(compiler, connection)
return f"{lhs} LIKE {rhs}", lhs_params + rhs_params
return f"LOWER({lhs}) LIKE {rhs}", lhs_params + [rhs.lower() for rhs in rhs_params]

def get_db_prep_lookup(self, value, connection):
"""Apply the wildcard logic to the right-hand-side value"""
Expand All @@ -92,3 +107,14 @@ def _sql_wildcards(value: str) -> str:
.replace("*", "%")
.replace("?", "_")
)


@models.ForeignKey.register_lookup
class CaseInsensitiveExact(lookups.Lookup):
lookup_name = "iexact"

def as_sql(self, compiler, connection):
"""Generate the required SQL."""
lhs, lhs_params = self.process_lhs(compiler, connection)
rhs, rhs_params = self.process_rhs(compiler, connection)
return f"LOWER({lhs}) = LOWER({rhs})", lhs_params + rhs_params
15 changes: 12 additions & 3 deletions src/dso_api/dynamic_api/filters/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,6 @@ def _compile_filter(
value = self._translate_raw_value(filter_input, parts[-1])
lookup = self._translate_lookup(filter_input, parts[-1], value)
q_path = f"{orm_path}__{lookup}"

if filter_input.lookup == "not":
# for [not] lookup: field != 1 AND field != 2
q_object = reduce(operator.and_, (Q(**{q_path: v}) for v in value))
Expand Down Expand Up @@ -305,7 +304,6 @@ def _translate_lookup(
lookup = filter_input.lookup

# Find whether the lookup is allowed at all.
# This prevents doing a "like" lookup on an integer/date-time field for example.
allowed_lookups = self.get_allowed_lookups(filter_part.field)
if lookup not in allowed_lookups:
if not allowed_lookups:
Expand All @@ -323,11 +321,22 @@ def _translate_lookup(
}
) from None

# Handle case-insensitive exact matches for string fields only,
# but not for relations or formatted fields
if filter_part.field.format == "date-time" and not isinstance(value, datetime):
# When something different then a full datetime is given, only compare dates.
# Otherwise, the "lte" comparison happens against 00:00:00.000 of that date,
# instead of anything that includes that day itself.
lookup = f"date__{lookup or 'exact'}"
return f"date__{lookup or 'exact'}"

# Only apply iexact for direct string field lookups (not through relations)
if (
not lookup
and filter_part.field.type == "string"
and filter_part.field.format not in ["date-time", "time", "date"]
and not filter_part.field.is_relation
):
return "iexact"

return lookup or "exact"

Expand Down
7 changes: 6 additions & 1 deletion src/tests/test_dynamic_api/test_filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def test_like_filter_sql(self, django_assert_num_queries):
list(Dataset.objects.filter(name__like="foo*bar?"))

sql = context.captured_queries[0]["sql"]
assert r"""."name" LIKE 'foo%bar_'""" in sql
assert r"""."name") LIKE 'foo%bar_'""" in sql


def test_sql_wildcards():
Expand Down Expand Up @@ -96,6 +96,11 @@ def movie2(self, movies_model, movies_category):
("url[in]=foobar,http://example.com/someurl", {"movie2"}),
("url[like]=http:*", {"movie2"}),
("url[isnull]=true", {"movie1"}),
# Case insensitive match
("name=movie1", {"movie1"}),
("name=Movie1", {"movie1"}),
("name[like]=movie1", {"movie1"}),
("name[like]=Movie1", {"movie1"}),
],
)
def test_filter_logic(self, movies_model, movie1, movie2, query, expect):
Expand Down
Loading