lexical analysis More...
#include <json.h>
lexical analysis
This class organizes the lexical analysis during JSON deserialization.
Public Types | |
using | token_type = typename lexer_base< BasicJsonType >::token_type |
Public Types inherited from nlohmann::detail::lexer_base< BasicJsonType > | |
enum class | token_type { uninitialized , literal_true , literal_false , literal_null , value_string , value_unsigned , value_integer , value_float , begin_array , begin_object , end_array , end_object , name_separator , value_separator , parse_error , end_of_input , literal_or_value } |
token types for the parser More... | |
Public Member Functions | |
lexer (InputAdapterType &&adapter, bool ignore_comments_=false) | |
lexer (const lexer &)=delete | |
lexer (lexer &&)=default | |
lexer & | operator= (lexer &)=delete |
lexer & | operator= (lexer &&)=default |
~lexer ()=default | |
constexpr number_integer_t | get_number_integer () const noexcept |
return integer value More... | |
constexpr number_unsigned_t | get_number_unsigned () const noexcept |
return unsigned integer value More... | |
constexpr number_float_t | get_number_float () const noexcept |
return floating-point value More... | |
string_t & | get_string () |
return current string value (implicitly resets the token; useful only once) More... | |
constexpr position_t | get_position () const noexcept |
return position of last read token More... | |
std::string | get_token_string () const |
constexpr JSON_HEDLEY_RETURNS_NON_NULL const char * | get_error_message () const noexcept |
return syntax error message More... | |
bool | skip_bom () |
skip the UTF-8 byte order mark More... | |
void | skip_whitespace () |
token_type | scan () |
Additional Inherited Members | |
Static Public Member Functions inherited from nlohmann::detail::lexer_base< BasicJsonType > | |
JSON_HEDLEY_RETURNS_NON_NULL static JSON_HEDLEY_CONST const char * | token_type_name (const token_type t) noexcept |
return name of values of type token_type (only used for errors) More... | |
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
using nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_type = typename lexer_base<BasicJsonType>::token_type |
|
inlineexplicit |
|
delete |
|
default |
|
default |
|
inlineprivate |
add a character to token_buffer
Definition at line 7272 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_buffer.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_byte_in_range(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string().
|
inlineprivate |
Definition at line 7175 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message, and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get().
|
inlineprivate |
Definition at line 7208 of file json.h.
References nlohmann::detail::position_t::chars_read_current_line, nlohmann::detail::position_t::chars_read_total, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::ia, nlohmann::detail::position_t::lines_read, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_unget, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::position, and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_string.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::for(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_codepoint(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_byte_in_range(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_comment(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::skip_bom(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::skip_whitespace().
|
inlineprivate |
get codepoint from 4 hex characters following \u
For input "\u c1 c2 c3 c4" the codepoint is: (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4 = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f' must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The conversion is done by subtracting the offset (0x30, 0x37, and 0x57) between the ASCII value of the character and the desired integer value.
Definition at line 6040 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), and nlohmann::detail::JSON_ASSERT().
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string().
|
inlinestaticprivatenoexcept |
return the locale-dependent decimal point
Definition at line 6014 of file json.h.
References nlohmann::detail::JSON_ASSERT().
|
inlineconstexprnoexcept |
return syntax error message
Definition at line 7344 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message.
|
inlineconstexprnoexcept |
return floating-point value
Definition at line 7295 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::value_float.
|
inlineconstexprnoexcept |
return integer value
Definition at line 7283 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::value_integer.
|
inlineconstexprnoexcept |
return unsigned integer value
Definition at line 7289 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::value_unsigned.
|
inlineconstexprnoexcept |
return position of last read token
Definition at line 7311 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::position.
Referenced by nlohmann::detail::parser< BasicJsonType, InputAdapterType >::parse().
|
inline |
return current string value (implicitly resets the token; useful only once)
Definition at line 7301 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_buffer.
|
inline |
return the last read token (for errors only). Will never contain EOF (an arbitrary value that is not a valid char value, often -1), because 255 may legitimately occur. May contain NUL, which should be escaped.
Definition at line 7319 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_string.
Referenced by nlohmann::detail::parser< BasicJsonType, InputAdapterType >::parse().
|
private |
|
private |
|
private |
|
private |
[in] | literal_text | the literal text to expect |
[in] | length | the length of the passed literal text |
[in] | return_type | the token type to return on success |
|
inlineprivate |
check if the next byte(s) are inside a given range
Adds the current byte and, for each passed range, reads a new byte and checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true.
[in] | ranges | list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively |
Definition at line 6088 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::add(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), and nlohmann::detail::JSON_ASSERT().
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string().
|
default |
|
delete |
|
inlineprivatenoexcept |
reset token_buffer; current character is beginning of token
Definition at line 7191 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_buffer, and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_string.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string().
|
inline |
Definition at line 7380 of file json.h.
References nlohmann::detail::position_t::chars_read_total, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::ignore_comments, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::position, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_comment(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::skip_bom(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::skip_whitespace().
|
inlineprivate |
scan a comment
Definition at line 6715 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget().
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan().
|
inlineprivate |
scan a number literal
This function scans a string according to Sect. 6 of RFC 7159.
The function is realized with a deterministic finite state machine derived from the grammar described in RFC 7159. Starting in state "init", the input is read and used to determined the next state. Only state "done" accepts the number. State "error" is a trap state to model errors. In the table below, "anything" means any character but the ones listed before.
state | 0 | 1-9 | e E | + | - | . | anything |
---|---|---|---|---|---|---|---|
init | zero | any1 | [error] | [error] | minus | [error] | [error] |
minus | zero | any1 | [error] | [error] | [error] | [error] | [error] |
zero | done | done | exponent | done | done | decimal1 | done |
any1 | any1 | any1 | exponent | done | done | decimal1 | done |
decimal1 | decimal2 | decimal2 | [error] | [error] | [error] | [error] | [error] |
decimal2 | decimal2 | decimal2 | exponent | done | done | done | done |
exponent | any2 | any2 | [error] | sign | sign | [error] | [error] |
sign | any2 | any2 | [error] | [error] | [error] | [error] | [error] |
any2 | any2 | any2 | done | done | done | done | done |
The state machine is realized with one label per state (prefixed with "scan_number_") and goto
statements between them. The state machine contains cycles, but any cycle can be left when EOF is read. Therefore, the function is guaranteed to terminate.
During scanning, the read bytes are stored in token_buffer. This string is then converted to a signed integer, an unsigned integer, or a floating-point number.
.
to work with the locale-dependent converters. Definition at line 6840 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::add(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::decimal_point_char, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), nlohmann::detail::JSON_ASSERT(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::reset(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_buffer, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::value_float, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::value_integer, and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::value_unsigned.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan().
|
inlineprivate |
scan a string literal
This function scans a string according to Sect. 7 of RFC 7159. While scanning, bytes are escaped and copied into buffer token_buffer. Then the function returns successfully, token_buffer is not null-terminated (as it may contain \0 bytes), and token_buffer.size() is the number of bytes in the string.
Definition at line 6125 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::add(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::error_message, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_codepoint(), nlohmann::detail::JSON_ASSERT(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_byte_in_range(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::reset().
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan().
|
inline |
skip the UTF-8 byte order mark
Definition at line 7357 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget().
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan().
|
inline |
Definition at line 7371 of file json.h.
References nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get().
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan().
|
inlineprivate |
unget current character (read it again on next get)
We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will behave as if the unget character is read again.
Definition at line 7245 of file json.h.
References nlohmann::detail::position_t::chars_read_current_line, nlohmann::detail::position_t::chars_read_total, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::current, nlohmann::detail::JSON_ASSERT(), nlohmann::detail::position_t::lines_read, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_unget, nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::position, and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::token_string.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_comment(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::skip_bom().
|
private |
the current character
Definition at line 7476 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_codepoint(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_byte_in_range(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::reset(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::skip_whitespace(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget().
|
private |
the decimal point
Definition at line 7499 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number().
|
private |
a description of occurred lexer errors
Definition at line 7491 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::for(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_error_message(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::next_byte_in_range(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_comment(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_string().
|
private |
input adapter
Definition at line 7470 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get().
|
private |
whether comments should be ignored (true) or signaled as errors (false)
Definition at line 7473 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan().
|
private |
|
private |
whether the next get() call should just return current
Definition at line 7479 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget().
|
private |
|
private |
the start position of the current token
Definition at line 7482 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_position(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget().
|
private |
|
private |
|
private |
Definition at line 6783 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::JSON_HEDLEY_NON_NULL().
|
private |
buffer for variable-length tokens (numbers, strings)
Definition at line 7488 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::add(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_string(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::reset(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number().
|
private |
raw input token string (for error messages)
Definition at line 7485 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_token_string(), nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::reset(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::unget().
|
private |
Definition at line 7496 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_number_float(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number().
|
private |
Definition at line 7494 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_number_integer(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number().
|
private |
Definition at line 7495 of file json.h.
Referenced by nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::get_number_unsigned(), and nlohmann::detail::lexer< BasicJsonType, InputAdapterType >::scan_number().