From 9d0c525b98eb84fbb2ef96dd4755af6b5f23050d Mon Sep 17 00:00:00 2001 From: Javen O'Neal Date: Wed, 9 Aug 2017 01:57:04 -0700 Subject: [PATCH 1/3] Encode set, frozenset, bytearray, and iterators as json arrays Based on a discussion on Stack Overflow https://stackoverflow.com/questions/45457946/python-efficient-reverse-list-json-serialisation/45458128#45458128 I wrote out a solution that is general enough to support most of core Python. If there are performance issues with the `iterable_types` nonlocal lookup, these could be pulled in as local variables to the inner `_iterencode_*` functions. As written, a user would have to monkey patch `json.encoder._get_iterable_types()` (or one of the functions it calls) if they wanted to be able to encode their own sequence-like containers or iterators as a json array. If there are plans for the json library to sequence-like classes and iterators that are part of the core language, part of the standard library, and in third party modules, then some modifications will be needed so that monkey patching isn't necessary. These changes are contributed under the Python Software License 2.0 or similar. I can sign a contributor license agreement if needed. --- Lib/json/encoder.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 41a497c5da01603..ba6edb4fbe7a92c 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -256,8 +256,32 @@ def floatstr(o, allow_nan=self.allow_nan, self.skipkeys, _one_shot) return _iterencode(o, 0) +def _get_sequence_types(): + """Non-iterator sequence types that should be encoded as a json array""" + # FIXME: there are a bunch of other sequences in the Python standard library, + # such as an array.array, queue.Queue, collections.deque. + return (list, tuple, set, frozenset, bytearray) + +def _get_iterator_types(): + """Iterator types that should be encoded as a json array""" + # FIXME: Are these types defined in builtins or the standard library somewhere? + str_iterator = type(iter( str() )) + list_iterator = type(iter( list() )) + tuple_iterator = type(iter( tuple() )) + range_iterator = type(iter( range(0) )) + list_reverseiterator = type(reversed( list() )) + reverseiterator = type(reversed( tuple() )) #same as + + return (str_iterator, list_iterator, tuple_iterator, range_iterator, + list_reverseiterator, reverseiterator) + +def _get_iterable_types(): + """Returns a tuple of all types that should be encoded as a json array""" + return _get_sequence_types() + _get_iterator_types() + def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, + iterable_types=_get_iterable_types(), ## HACK: hand-optimized bytecode; turn globals into locals ValueError=ValueError, dict=dict, @@ -316,7 +340,7 @@ def _iterencode_list(lst, _current_indent_level): yield buf + _floatstr(value) else: yield buf - if isinstance(value, (list, tuple)): + if isinstance(value, iterable_types): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) @@ -395,7 +419,7 @@ def _iterencode_dict(dct, _current_indent_level): # see comment for int/float in _make_iterencode yield _floatstr(value) else: - if isinstance(value, (list, tuple)): + if isinstance(value, iterable_types): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) @@ -424,7 +448,7 @@ def _iterencode(o, _current_indent_level): elif isinstance(o, float): # see comment for int/float in _make_iterencode yield _floatstr(o) - elif isinstance(o, (list, tuple)): + elif isinstance(o, iterable_types): yield from _iterencode_list(o, _current_indent_level) elif isinstance(o, dict): yield from _iterencode_dict(o, _current_indent_level) From a13820c75c85305261e4387f3d6cb5cb65352e6d Mon Sep 17 00:00:00 2001 From: Javen O'Neal Date: Wed, 9 Aug 2017 02:37:23 -0700 Subject: [PATCH 2/3] whitespace --- Lib/json/encoder.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index ba6edb4fbe7a92c..7489dd5c83062ea 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -261,19 +261,19 @@ def _get_sequence_types(): # FIXME: there are a bunch of other sequences in the Python standard library, # such as an array.array, queue.Queue, collections.deque. return (list, tuple, set, frozenset, bytearray) - + def _get_iterator_types(): """Iterator types that should be encoded as a json array""" # FIXME: Are these types defined in builtins or the standard library somewhere? - str_iterator = type(iter( str() )) - list_iterator = type(iter( list() )) - tuple_iterator = type(iter( tuple() )) - range_iterator = type(iter( range(0) )) - list_reverseiterator = type(reversed( list() )) - reverseiterator = type(reversed( tuple() )) #same as - + str_iterator = type(iter(str())) + list_iterator = type(iter(list())) + tuple_iterator = type(iter(tuple())) + range_iterator = type(iter(range(0))) + list_reverseiterator = type(reversed(list())) + # same as + reverseiterator = type(reversed(tuple()) return (str_iterator, list_iterator, tuple_iterator, range_iterator, - list_reverseiterator, reverseiterator) + list_reverseiterator, reverseiterator) def _get_iterable_types(): """Returns a tuple of all types that should be encoded as a json array""" From 4b21e7f87c95004e8cb02c9550f712c08dc7c16a Mon Sep 17 00:00:00 2001 From: Javen O'Neal Date: Wed, 9 Aug 2017 23:31:10 -0700 Subject: [PATCH 3/3] Support for dict iterators If the user passes in an iterator to a dict rather than the dict itself, handle the special dict_keys, dict_values, or dict_items iterators for serialization as a json array (perhaps the dict_itemiterator should write out a json dictionary instead). --- Lib/json/encoder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 7489dd5c83062ea..4260e59ed1f3de3 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -270,10 +270,14 @@ def _get_iterator_types(): tuple_iterator = type(iter(tuple())) range_iterator = type(iter(range(0))) list_reverseiterator = type(reversed(list())) - # same as - reverseiterator = type(reversed(tuple()) + # reverseiterator is same as + reverseiterator = type(reversed(tuple())) + dict_keyiterator = type(iter(dict().keys())) + dict_valueiterator = type(iter(dict().values())) + dict_itemiterator = type(iter(dict().items())) return (str_iterator, list_iterator, tuple_iterator, range_iterator, - list_reverseiterator, reverseiterator) + list_reverseiterator, reverseiterator, + dict_keyiterator, dict_valueiterator, dict_itemiterator) def _get_iterable_types(): """Returns a tuple of all types that should be encoded as a json array"""