From 442c7a486f91a434e0ba7535f17aecdb83d442f4 Mon Sep 17 00:00:00 2001 From: "Node.js GitHub Bot" Date: Sun, 6 Sep 2026 00:59:46 +0000 Subject: [PATCH] deps: update simdjson to 4.6.11 --- deps/simdjson/simdjson.cpp | 339 +++++++++++++++++++++++++------------ deps/simdjson/simdjson.h | 258 +++++++++++++++++++++++----- 2 files changed, 453 insertions(+), 144 deletions(-) diff --git a/deps/simdjson/simdjson.cpp b/deps/simdjson/simdjson.cpp index 92d971fd448b..71f443b3d2bb 100644 --- a/deps/simdjson/simdjson.cpp +++ b/deps/simdjson/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2026-08-24 17:10:01 -0400. version 4.6.9 Do not edit! */ +/* auto-generated on 2026-09-04 16:04:31 -0400. version 4.6.11 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -240,7 +240,7 @@ using std::size_t; #endif #elif defined(__PPC64__) || defined(_M_PPC64) #define SIMDJSON_IS_PPC64 1 -#if defined(__ALTIVEC__) +#if defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #define SIMDJSON_IS_PPC64_VMX 1 #endif // defined(__ALTIVEC__) #else @@ -10052,9 +10052,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -10065,6 +10070,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -13541,7 +13547,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -13575,7 +13588,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -13597,7 +13610,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -13976,11 +13989,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -14005,6 +14013,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -14021,7 +14035,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -16588,9 +16602,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -16601,6 +16620,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -19936,7 +19956,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -19970,7 +19997,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -19992,7 +20019,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -20371,11 +20398,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -20400,6 +20422,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -20416,7 +20444,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -22979,9 +23007,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -22992,6 +23025,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -26326,7 +26360,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -26360,7 +26401,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -26382,7 +26423,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -26761,11 +26802,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -26790,6 +26826,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -26806,7 +26848,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -29527,9 +29569,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -29540,6 +29587,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -32987,7 +33035,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -33021,7 +33076,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -33043,7 +33098,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -33422,11 +33477,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -33451,6 +33501,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -33467,7 +33523,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -36435,9 +36491,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -36448,6 +36509,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -40210,7 +40272,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -40244,7 +40313,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -40266,7 +40335,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -40645,11 +40714,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -40674,6 +40738,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -40690,7 +40760,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -43189,9 +43259,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -43202,6 +43277,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -46464,7 +46540,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -46498,7 +46581,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -46520,7 +46603,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -46899,11 +46982,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -46928,6 +47006,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -46944,7 +47028,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -49380,9 +49464,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -49393,6 +49482,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -52622,7 +52712,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -52656,7 +52753,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -52678,7 +52775,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -53057,11 +53154,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -53086,6 +53178,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -53102,7 +53200,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -55559,9 +55657,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -55572,6 +55675,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -59199,7 +59303,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -59233,7 +59344,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -59255,7 +59366,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -59634,11 +59745,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -59663,6 +59769,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -59679,7 +59791,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -61715,9 +61827,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -61728,6 +61845,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -63685,7 +63803,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -63719,7 +63844,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -63741,7 +63866,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -64949,8 +65074,8 @@ simdjson_inline void validate_utf8_character() { // 2-byte if ((buf[idx] & 0x20) == 0) { // missing continuation - if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { - if (idx+1 > len && is_streaming(partial)) { idx = len; return; } + if (simdjson_unlikely(idx+1 >= len || !is_continuation(buf[idx+1]))) { + if (idx+1 >= len && is_streaming(partial)) { idx = len; return; } error = UTF8_ERROR; idx++; return; @@ -64964,8 +65089,8 @@ simdjson_inline void validate_utf8_character() { // 3-byte if ((buf[idx] & 0x10) == 0) { // missing continuation - if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { - if (idx+2 > len && is_streaming(partial)) { idx = len; return; } + if (simdjson_unlikely(idx+2 >= len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { + if (idx+2 >= len && is_streaming(partial)) { idx = len; return; } error = UTF8_ERROR; idx++; return; @@ -64980,8 +65105,8 @@ simdjson_inline void validate_utf8_character() { // 4-byte // missing continuation - if (simdjson_unlikely(idx+3 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { - if (idx+2 > len && is_streaming(partial)) { idx = len; return; } + if (simdjson_unlikely(idx+3 >= len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { + if (idx+3 >= len && is_streaming(partial)) { idx = len; return; } error = UTF8_ERROR; idx++; return; @@ -65103,9 +65228,15 @@ simdjson_warn_unused simdjson_inline error_code scan() { add_structural(); // Primitive or invalid character (invalid characters will be checked in stage 2) } else { - // Anything else, add the structural and go until we find the next one + // Anything else, add the structural and go until we find the next one. + // We also stop on '"' so that an unclosed string still reaches + // validate_string(); a quote swallowed by the run would hide it. A + // quote cannot occur inside a valid primitive. We deliberately do not + // stop on every ESC_ASCII character: that also covers a backslash and the + // control characters, and ending the run there makes the fallback + // disagree with the SIMD kernels. add_structural(); - while (idx+1 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is diff --git a/deps/simdjson/simdjson.h b/deps/simdjson/simdjson.h index 55346b59ef59..43fe09631c01 100644 --- a/deps/simdjson/simdjson.h +++ b/deps/simdjson/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2026-08-24 17:10:01 -0400. version 4.6.9 Do not edit! */ +/* auto-generated on 2026-09-04 16:04:31 -0400. version 4.6.11 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -260,7 +260,7 @@ using std::size_t; #endif #elif defined(__PPC64__) || defined(_M_PPC64) #define SIMDJSON_IS_PPC64 1 -#if defined(__ALTIVEC__) +#if defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #define SIMDJSON_IS_PPC64_VMX 1 #endif // defined(__ALTIVEC__) #else @@ -2538,7 +2538,7 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.6.9" +#define SIMDJSON_VERSION "4.6.11" namespace simdjson { enum { @@ -2553,7 +2553,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 9 + SIMDJSON_VERSION_REVISION = 11 }; } // namespace simdjson @@ -4734,6 +4734,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept { inline padded_string::padded_string() noexcept = default; inline padded_string::padded_string(size_t length) noexcept : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if (data_ptr == nullptr) { viable_size = 0; } } inline padded_string::padded_string(const char *data, size_t length) noexcept : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { @@ -6351,6 +6352,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; /** @@ -9945,6 +9958,9 @@ simdjson_inline size_t document_stream::iterator::current_index() const noexcept simdjson_inline std::string_view document_stream::iterator::source() const noexcept { const char* start = reinterpret_cast(stream->buf) + current_index(); + if (stream->error) { + return std::string_view(start, stream->len - current_index()); + } bool object_or_array = ((*start == '[') || (*start == '{')); if(object_or_array) { size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index - 1]; @@ -14993,9 +15009,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -15006,6 +15027,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -17206,9 +17228,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -17219,6 +17246,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -19906,9 +19934,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -19919,6 +19952,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -22606,9 +22640,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -22619,6 +22658,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -25421,9 +25461,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -25434,6 +25479,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -28553,9 +28599,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -28566,6 +28617,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -31185,9 +31237,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -31198,6 +31255,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -33795,9 +33853,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -33808,6 +33871,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -36422,9 +36486,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -36435,6 +36504,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -39845,7 +39915,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -41940,7 +42010,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -44522,7 +44592,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -47104,7 +47174,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -49801,7 +49871,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -52815,7 +52885,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -55303,7 +55373,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -57814,7 +57884,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -60329,7 +60399,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -66951,6 +67021,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -70987,7 +71069,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -80330,6 +80412,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -84366,7 +84460,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -94196,6 +94290,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -98232,7 +98338,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -108062,6 +108168,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -112098,7 +112216,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -122043,6 +122161,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -126079,7 +126209,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -136341,6 +136471,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -140377,7 +140519,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -150113,6 +150255,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -154149,7 +154303,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -163908,6 +164062,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -167944,7 +168110,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -177707,6 +177873,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -181743,7 +181921,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index();