From d22fd1fe223be6406a6311afcc02720bc3ed9c1d Mon Sep 17 00:00:00 2001 From: Reto Schneider Date: Mon, 13 Dec 2021 02:00:30 +0100 Subject: [PATCH] service: Support for partial listings (server-side pagination) Reading the spec, using the __next field, not $top/$skip/__count seems to be the pristine way of fetching an entity collection in full: > In response payloads, representing Collections of Entries, if the > server does not include an object for every Entry in the Collection of > Entries identified by the request URI then the response represents a > partial listings of the Collection. In this case, "__next" name/value > pair is included to indicate the response represents a partial > listing. The value of the name/value pair is a URI which identifies > the next partial set of entities from the originally identified > complete set. --- pyodata/v2/service.py | 45 +++++++++++++++++++++++++++---- tests/test_service_v2.py | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/pyodata/v2/service.py b/pyodata/v2/service.py index 1fa3627d..f75f0d92 100644 --- a/pyodata/v2/service.py +++ b/pyodata/v2/service.py @@ -238,6 +238,7 @@ def __init__(self, url, connection, handler, headers=None): self._headers = headers or dict() self._logger = logging.getLogger(LOGGER_NAME) self._customs = {} # string -> string hash + self._next_url = None @property def handler(self): @@ -299,7 +300,10 @@ def execute(self): Fetches HTTP response and returns processed result""" - url = urljoin(self._url, self.get_path()) + if self._next_url: + url = self._next_url + else: + url = urljoin(self._url, self.get_path()) # pylint: disable=assignment-from-none body = self.get_body() @@ -616,6 +620,17 @@ def count(self, inline=False): self._count = True return self + def next_url(self, next_url): + """ + Sets URL which identifies the next partial set of entities from the originally identified complete set. Once + set, this URL takes precedence over all query parameters. + + For details, see section "6. Representing Collections of Entries" on + https://www.odata.org/documentation/odata-version-2-0/json-format/ + """ + self._next_url = next_url + return self + def expand(self, expand): """Sets the expand expressions.""" self._expand = expand @@ -667,6 +682,9 @@ def get_default_headers(self): } def get_query_params(self): + if self._next_url: + return {} + qparams = super(QueryRequest, self).get_query_params() if self._top is not None: @@ -1250,11 +1268,24 @@ def filter(self, *args, **kwargs): class ListWithTotalCount(list): - """A list with the additional property total_count""" + """ + A list with the additional property total_count and next_url. + + If set, use next_url to fetch the next batch of entities. + """ - def __init__(self, total_count): + def __init__(self, total_count, next_url): super(ListWithTotalCount, self).__init__() self._total_count = total_count + self._next_url = next_url + + @property + def next_url(self): + """ + URL which identifies the next partial set of entities from the originally identified complete set. None if no + entities remaining. + """ + return self._next_url @property def total_count(self): @@ -1390,7 +1421,8 @@ def get_entity_handler(response): return EntityGetRequest(get_entity_handler, entity_key, self) def get_entities(self): - """Get all entities""" + """Get some, potentially all entities""" + def get_entities_handler(response): """Gets entity set from HTTP Response""" @@ -1405,15 +1437,18 @@ def get_entities_handler(response): entities = content['d'] total_count = None + next_url = None if isinstance(entities, dict): if '__count' in entities: total_count = int(entities['__count']) + if '__next' in entities: + next_url = entities['__next'] entities = entities['results'] self._logger.info('Fetched %d entities', len(entities)) - result = ListWithTotalCount(total_count) + result = ListWithTotalCount(total_count, next_url) for props in entities: entity = EntityProxy(self._service, self._entity_set, self._entity_set.entity_type, props) result.append(entity) diff --git a/tests/test_service_v2.py b/tests/test_service_v2.py index 6e032c44..c35123f6 100644 --- a/tests/test_service_v2.py +++ b/tests/test_service_v2.py @@ -2005,6 +2005,64 @@ def test_count_with_chainable_filter(service): assert request.execute() == 3 +@responses.activate +def test_partial_listing(service): + """Using __next URI to fetch all entities in a collection""" + + # pylint: disable=redefined-outer-name + + responses.add( + responses.GET, + f"{service.url}/Employees?$inlinecount=allpages", + json={'d': { + '__count': 3, + '__next': f"{service.url}/Employees?$inlinecount=allpages&$skiptoken='opaque'", + 'results': [ + { + 'ID': 21, + 'NameFirst': 'George', + 'NameLast': 'Doe' + },{ + 'ID': 22, + 'NameFirst': 'John', + 'NameLast': 'Doe' + } + ] + }}, + status=200) + + responses.add( + responses.GET, + f"{service.url}/Employees?$inlinecount=allpages&$skiptoken='opaque'", + json={'d': { + '__count': 3, + 'results': [ + { + 'ID': 23, + 'NameFirst': 'Rob', + 'NameLast': 'Ickes' + } + ] + }}, + status=200) + + # Fetching (potentially) all entities, actually getting 2 + request = service.entity_sets.Employees.get_entities().count(inline=True) + assert isinstance(request, pyodata.v2.service.GetEntitySetRequest) + result = request.execute() + assert len(result) == 2 + assert result.total_count == 3 + assert result.next_url is not None + + # Fetching next batch, receive the one remaining entity + request = service.entity_sets.Employees.get_entities().next_url(result.next_url) + assert isinstance(request, pyodata.v2.service.GetEntitySetRequest) + result = request.execute() + assert len(result) == 1 + assert result.total_count == 3, "(inline) count flag inherited from first request" + assert result.next_url is None + + @responses.activate def test_count_with_chainable_filter_lt_operator(service): """Check getting $count with $filter with new filter syntax using multiple filters"""