Python elasticsearch-dsl django pagination

Question

How can I use django pagination on elasticsearch dsl. My code is:

query = MultiMatch(query=q, fields=['title', 'body'], fuzziness='AUTO') s = Search(using=elastic_client, index='post').query(query).sort('-created_at') response = s.execute() // this always returns page count 1 paginator = Paginator(response, 100) page = request.GET.get('page') try: posts = paginator.page(page) except PageNotAnInteger: posts = paginator.page(1) except EmptyPage: posts = paginator.page(paginator.num_pages) 

Any solution for this?

Answer 1

I found this paginator at this link :

 from django.core.paginator import Paginator, Page class DSEPaginator(Paginator): """ Override Django built-in Paginator class to take in a count/total number of items; Elasticsearch provides the total as a part of the query results, so we can minimize hits. """ def __init__(self, *args, **kwargs): super(DSEPaginator, self).__init__(*args, **kwargs) self._count = self.object_list.hits.total def page(self, number): # this is overridden to prevent any slicing of the object_list - Elasticsearch has # returned the sliced data already. number = self.validate_number(number) return Page(self.object_list, number, self) 

and then I see:

  q = request.GET.get('q', None) page = int(request.GET.get('page', '1')) start = (page-1) * 10 end = start + 10 query = MultiMatch(query=q, fields=['title', 'body'], fuzziness='AUTO') s = Search(using=elastic_client, index='post').query(query)[start:end] response = s.execute() paginator = DSEPaginator(response, settings.POSTS_PER_PAGE) try: posts = paginator.page(page) except PageNotAnInteger: posts = paginator.page(1) except EmptyPage: posts = paginator.page(paginator.num_pages) 

so it works great.

Answer 2

Another way forward is to create a proxy between the Paginator request and Elasticsearch. Paginator requires two things: __len__ (or count ) and __getitem__ (which takes a slice). The rough version of the proxy works as follows:

 class ResultsProxy(object): """ A proxy object for returning Elasticsearch results that is able to be passed to a Paginator. """ def __init__(self, es, index=None, body=None): self.es = es self.index = index self.body = body def __len__(self): result = self.es.count(index=self.index, body=self.body) return result['count'] def __getitem__(self, item): assert isinstance(item, slice) results = self.es.search( index=self.index, body=self.body, from_=item.start, size=item.stop - item.start, ) return results['hits']['hits'] 

A proxy instance can be passed to Paginator and will request the ES as needed.

Answer 3

Following Danielle Madeley's advice, I also created a proxy server for finding results that works well with the latest version of django-elasticsearch-dsl==0.4.4 .

 from django.utils.functional import LazyObject class SearchResults(LazyObject): def __init__(self, search_object): self._wrapped = search_object def __len__(self): return self._wrapped.count() def __getitem__(self, index): search_results = self._wrapped[index] if isinstance(index, slice): search_results = list(search_results) return search_results 

Then you can use it in your search type as follows:

 paginate_by = 20 search = MyModelDocument.search() # ... do some filtering ... search_results = SearchResults(search) paginator = Paginator(search_results, paginate_by) page_number = request.GET.get("page") try: page = paginator.page(page_number) except PageNotAnInteger: # If page parameter is not an integer, show first page. page = paginator.page(1) except EmptyPage: # If page parameter is out of range, show last existing page. page = paginator.page(paginator.num_pages) 

Django LazyObject proxies all attributes and methods from the object assigned to the _wrapped attribute. I override several methods that are required in paginator Django, but do not work out of the box with Search () instances.