Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions ext/json/ext/parser/parser.c
Original file line number Diff line number Diff line change
Expand Up @@ -1457,7 +1457,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat
return (int)(state->cursor - start);
}

static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable)
{
bool integer = true;
const char first_digit = *state->cursor;
Expand Down Expand Up @@ -1514,6 +1514,16 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
}
}

// A number touching the end of the buffer may still grow in a later chunk,
// so the caller will rewind and wait. Decoding it now would build a value
// -- for a long run of digits, an expensive bignum -- only to discard it,
// and repeating that on every resumed chunk is quadratic in the number's
// length. The digit scan above already advanced the cursor, which is all
// the caller needs to detect the incomplete number.
if (RB_UNLIKELY(resumable && eos(state))) {
return Qundef;
}

if (integer) {
return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor);
}
Expand Down Expand Up @@ -1638,7 +1648,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
case '-': {
state->cursor++;

value = json_parse_number(state, config, true, value_start);
value = json_parse_number(state, config, true, value_start, resumable);

if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
state->cursor = value_start;
Expand All @@ -1661,7 +1671,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
}

case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
value = json_parse_number(state, config, false, value_start);
value = json_parse_number(state, config, false, value_start, resumable);

// Top level numbers are ambiguous when parsing streams, we can't
// know if we parsed all the digits if we hit EOS.
Expand Down
17 changes: 17 additions & 0 deletions test/json/resumable_parser_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,23 @@ def test_parse_byte_by_byte_numbers
assert_resumed_parsing('123 ', trailing_bytes: 1)
end

def test_large_numbers_split_across_feeds_are_decoded_correctly
{
'12345678901234567890123456789012345678901234567890 ' => 12345678901234567890123456789012345678901234567890,
'-98765432109876543210987654321 ' => -98765432109876543210987654321,
'3.14159265358979323846264338327950288 ' => 3.14159265358979323846264338327950288,
'-1.5e-300 ' => -1.5e-300,
}.each do |doc, expected|
parser = new_parser
value = nil
doc.each_char do |char|
parser << char
value = parser.value if parser.parse
end
assert_equal expected, value, doc.inspect
end
end

def test_nul_byte_is_a_syntax_error
# A NUL byte in a structural position must raise, not stall forever waiting for more input
# (peek() returns 0 both at EOS and for a literal NUL byte).
Expand Down
Loading