Field filter chain qualifiers

CHANGELOG.md

@@ -1,5 +1,29 @@
 ## HEAD
 
+### Chained qualifiers
+Chained qualifiers have been added [T35707](https://phabricator.codeyellow.nl/T35707). For more information on how they work and how to use it see [documentation](/docs/api.md)
+
+#### Breaking change
+
+The `_filter_field` method signature has been changed from
+```
+def _filter_field(self, field_name, qualifier, value, invert, request, include_annotations, partial=''):
+```
+to
+```
+def _filter_field(self, field_name, qualifiers, value, invert, request, include_annotations, partial=''):
+```
+
+So it now contains an array of qualifiers instead of THE qualifier. 
+So for the call `api/caretaker?.last_seen:date:range` it will contain both date and range qualifiers.
+
+To get previously used qualifier variable and upgrade previously written code that was overriding `_filter_field` use: 
+```
+qualifier = qualifiers[0]
+```
+See full changes [here](https://github.com/CodeYellowBV/django-binder/pull/206/files#diff-0d5633deb444395cd44b49e7a39f87da98bb86de20666038277730991c62b1a5).
+
+
 ### Breaking changes
 
 The `_filter_field` method now returns a `Q()` object, not a queryset.

binder/models.py

@@ -153,6 +153,8 @@ class FieldFilter(object):
 	fields = []
 	# The list of allowed qualifiers
 	allowed_qualifiers = []
+	# The mapping of allowed chain qualifiers to the relevant Field
+	allowed_chain_qualifiers = {}
 
 	def __init__(self, field):
 		self.field = field
@@ -192,12 +194,65 @@ def check_qualifier(self, qualifier):
 					.format(qualifier, self.__class__.__name__, self.field_description()))
 
 
+	# This returns a (cached) filterclass for a field class.
+	def get_field_filter(self, field_class, reset=False):
+		f = not reset and getattr(self, '_field_filters', None)
+
+		if not f:
+			f = {}
+			for field_filter_cls in FieldFilter.__subclasses__():
+				for field_cls in field_filter_cls.fields:
+					if f.get(field_cls):
+						raise ValueError('Field-Filter mapping conflict: {} vs {}'.format(field_filter_cls.name, field_cls.name))
+					else:
+						f[field_cls] = field_filter_cls
+
+			self._field_filters = f
+
+		return f.get(field_class)
+
+
+
+	def get_q(self, qualifiers, value, invert, partial=''):
+		i = 0
+		field_filter = self
+
+		# First we try to handle chain qualifiers
+		while (
+			# If its not the last qualifier it has to be a chain qualifier
+			i < len(qualifiers) - 1 or
+			# For the last one we check if it is in chain qualifiers
+			(i < len(qualifiers) and qualifiers[i] in field_filter.allowed_chain_qualifiers)
+		):
+			chain_qualifier = qualifiers[i]
+			i += 1
+
+			field_cls = field_filter.allowed_chain_qualifiers[chain_qualifier]
+			if field_cls is None:
+				raise BinderRequestError(
+					'Qualifier {} not supported for type {} ({}).'
+					.format(chain_qualifier, field_filter.__class__.__name__, field_filter.field_description())
+				)
 
-	def get_q(self, qualifier, value, invert, partial=''):
-		self.check_qualifier(qualifier)
-		qualifier, cleaned_value = self.clean_qualifier(qualifier, value)
+			field = field_cls()
+			field.model = self.field.model
+			field.name = self.field.name + ':' + chain_qualifier
 
-		suffix = '__' + qualifier if qualifier else ''
+			field_filter_cls = self.get_field_filter(field_cls)
+			field_filter = field_filter_cls(field)
+
+		try:
+			qualifier = qualifiers[i]
+		except IndexError:
+			qualifier = None
+
+		field_filter.check_qualifier(qualifier)
+		qualifier, cleaned_value = field_filter.clean_qualifier(qualifier, value)
+
+		if 0 <= i < len(qualifiers):
+			qualifiers[i] = qualifier
+
+		suffix = ''.join('__' + qualifier for qualifier in qualifiers)
 		if invert:
 			return ~Q(**{partial + self.field.name + suffix: cleaned_value})
 		else:
@@ -254,6 +309,7 @@ class DateTimeFieldFilter(FieldFilter):
 	fields = [models.DateTimeField]
 	# Maybe allow __startswith? And __year etc?
 	allowed_qualifiers = [None, 'in', 'gt', 'gte', 'lt', 'lte', 'range', 'isnull']
+	allowed_chain_qualifiers = {'date': models.DateField}
 
 	def clean_value(self, qualifier, v):
 		if re.match('^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}([.][0-9]+)?([A-Za-z]+|[+-][0-9]{1,4})$', v):
@@ -275,6 +331,7 @@ def clean_qualifier(self, qualifier, value):
 		else:
 			value_type = type(cleaned_value)
 
+		# [TODO] Support for chained qualifiers is added, still needed for backwards compat
 		if issubclass(value_type, date) and not issubclass(value_type, datetime):
 			if qualifier is None:
 				qualifier = 'date'
@@ -337,6 +394,7 @@ def clean_value(self, qualifier, v):
 class TextFieldFilter(FieldFilter):
 	fields = [models.CharField, models.TextField]
 	allowed_qualifiers = [None, 'in', 'iexact', 'contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'isnull']
+	allowed_chain_qualifiers = {'unaccent': models.TextField}
 
 	# Always valid(?)
 	def clean_value(self, qualifier, v):
@@ -357,22 +415,6 @@ class ArrayFieldFilter(FieldFilter):
 	fields = [ArrayField]
 	allowed_qualifiers = [None, 'contains', 'contained_by', 'overlap', 'isnull']
 
-	# Some copy/pasta involved....
-	def get_field_filter(self, field_class, reset=False):
-		f = not reset and getattr(self, '_field_filter', None)
-
-		if not f:
-			f = None
-			for field_filter_cls in FieldFilter.__subclasses__():
-				for field_cls in field_filter_cls.fields:
-					if field_cls == field_class:
-						f = field_filter_cls
-						break
-			self._field_filter = f
-
-		return f
-
-
 	def clean_value(self, qualifier, v):
 		Filter = self.get_field_filter(self.field.base_field.__class__)
 		filter = Filter(self.field.base_field)

binder/views.py

@@ -1052,18 +1052,12 @@ def _parse_filter(self, field, value, request, include_annotations, partial=''):
 
 		if not tail:
 			invert = False
-			try:
-				head, qualifier = head.split(':', 1)
-				if qualifier == 'not':
-					qualifier = None
-					invert = True
-				elif qualifier.startswith('not:'):
-					qualifier = qualifier[4:]
-					invert = True
-			except ValueError:
-				qualifier = None
+			head, *qualifiers = head.split(':')
+			if qualifiers and qualifiers[0] == 'not':
+				qualifiers = qualifiers[1:]
+				invert = True
 
-			q = self._filter_field(head, qualifier, value, invert, request, include_annotations, partial)
+			q = self._filter_field(head, qualifiers, value, invert, request, include_annotations, partial)
 		else:
 			q = Q()
 
@@ -1095,7 +1089,7 @@ def _parse_filter(self, field, value, request, include_annotations, partial=''):
 
 
 
-	def _filter_field(self, field_name, qualifier, value, invert, request, include_annotations, partial=''):
+	def _filter_field(self, field_name, qualifiers, value, invert, request, include_annotations, partial=''):
 		try:
 			if field_name in self.hidden_fields:
 				raise FieldDoesNotExist()
@@ -1108,7 +1102,7 @@ def _filter_field(self, field_name, qualifier, value, invert, request, include_a
 			if partial:
 				# NOTE: This creates a subquery; try to avoid this!
 				qs = annotate(self.model.objects.all(), request, annotations)
-				qs = qs.filter(self._filter_field(field_name, qualifier, value, invert, request, {
+				qs = qs.filter(self._filter_field(field_name, qualifiers, value, invert, request, {
 					rel_[len(rel) + 1:]: annotations
 					for rel_, annotations in include_annotations.items()
 					if rel_ == rel or rel_.startswith(rel + '.')
@@ -1121,7 +1115,7 @@ def _filter_field(self, field_name, qualifier, value, invert, request, include_a
 			if filter_class:
 				filter = filter_class(field)
 				try:
-					return filter.get_q(qualifier, value, invert, partial)
+					return filter.get_q(qualifiers, value, invert, partial)
 				except ValidationError as e:
 					# TODO: Maybe convert to a BinderValidationError later?
 					raise BinderRequestError(e.message)

docs/api.md

@@ -49,6 +49,13 @@ To use a partial case-insensitive match, you can use `api/animal?.name:icontains
 
 Note that currently, it is not possible to search on many-to-many fields.
 
+#### Chaining multiple qualifiers
+Sometimes there is a need to chain multiple qualifiers in filtering. For example if you want to convert datetime to date and then do a range filtering.
+This can be achieved by using `api/caretaker?.last_seen:date:range`
+
+Other example would be if you want to search for city names, but without using accent in the names. For example search for `Weißenhorn` using term `weiss` or search for `Lünen` using `Lunen`. The syntax for it would be `.city_name:unaccent:icontains`. Take into account that for unaccent to work you need to install postgres `unaccent` extension.
+
+
 #### More advanced searching
 
 Sometimes you want to search on multiple fields at once.

tests/__init__.py

@@ -25,7 +25,7 @@
 	}
 else:
 	db_settings = {
-		'ENGINE': 'django.db.backends.postgresql_psycopg2',
+		'ENGINE': 'django.db.backends.postgresql',
 		'NAME': 'binder-test',
 		'HOST': 'localhost',
 		'USER': 'postgres',
@@ -51,6 +51,11 @@
 		'django.contrib.auth',
 		'django.contrib.contenttypes',
 		'django.contrib.sessions',
+		*(
+			['django.contrib.postgres']
+			if db_settings['ENGINE'] == 'django.db.backends.postgresql' else
+			[]
+		),
 		'binder',
 		'binder.plugins.token_auth',
 		'tests',
@@ -116,7 +121,7 @@
 # Do the dance to ensure the models are synched to the DB.
 # This saves us from having to include migrations
 from django.core.management.commands.migrate import Command as MigrationCommand # noqa
-from django.db import connections # noqa
+from django.db import connection, connections # noqa
 from django.db.migrations.executor import MigrationExecutor # noqa
 
 # This is oh so hacky....
@@ -132,3 +137,8 @@
 Permission.objects.get_or_create(content_type=content_type, codename='view_country')
 call_command('define_groups')
 
+
+# Create postgres extensions
+if db_settings['ENGINE'] == 'django.db.backends.postgresql':
+	with connection.cursor() as cursor:
+		cursor.execute('CREATE EXTENSION IF NOT EXISTS unaccent;')

tests/filters/test_datetime_filters.py

@@ -140,6 +140,29 @@ def test_datetime_filter_syntax_variations(self):
 		response = self.client.get('/caretaker/', data={'.last_seen:range': '2017-03-23T00:00:00Z,2017-03-24', 'order_by': 'last_seen'})
 		self.assertEqual(response.status_code, 418)
 
+	def test_datetime_filter_syntax_variations_with_chained_qualifiers(self):
+		# Implicitly we add T23:59:59Z here to make this correct.
+		response = self.client.get(
+			'/caretaker/', data={'.last_seen:date:gt': '2017-03-23', 'order_by': 'last_seen'})
+		self.assertEqual(response.status_code, 200)
+
+		result = jsonloads(response.content)
+		self.assertEqual(1, len(result['data']))
+
+		# Same as above, but to the range start we add T00:00:00Z
+		response = self.client.get(
+			'/caretaker/', data={'.last_seen:date:range': '2017-03-23,2017-03-23', 'order_by': 'last_seen'})
+		self.assertEqual(response.status_code, 200)
+
+		result = jsonloads(response.content)
+		self.assertEqual(1, len(result['data']))
+
+		# Just a sanity check
+		response = self.client.get('/caretaker/', data={'.last_seen:date:range': '2017-03-23,2017-03-24', 'order_by': 'last_seen'})
+		self.assertEqual(response.status_code, 200)
+
+		result = jsonloads(response.content)
+		self.assertEqual(2, len(result['data']))
 
 	def test_datetime_filter_syntax_errors_cause_error_response(self):
 		response = self.client.get('/caretaker/', data={'.last_seen': '1838-05'})